s3-backup.py

I use bacula to do the backup to file storage volumes and then I use a python program which I hacked together using the BitBucket library to query the bacula mysql database for full volumes and then it looks into the directory where bacula dumps the volumes and if it finds them there it uploads them to S3. When the upload finishes it deletes the volume from the local disk. I cron this program to run every 5 minutes checking for stuff to upload. This turns out to be a pretty slick solution and is working really well so far.

#!/usr/bin/python
"""

Get list of full volumes from the bacula database, upload the volumes
to S3, delete the volumes from the local system.

Tracy Reed
March 15, 2007

"""

import mimetypes
import os.path
import sys
import MySQLdb
import bitbucket
import time
import fcntl

AWS_ACCESS_KEY_ID = 'myaccesskey'
AWS_SECRET_ACCESS_KEY = 'mysecretkey'
BUCKET_NAME = 'mybucket'
BACKUP_DIR = '/home/treed/media/backups'

def daemonlock(progname, timeout=2):

    """Test lock for daemons to prevent multiple instances
 
    @param progname: The name of the caller. This name is used for the lock and
        should be unique on the system. Alternatively, this may be an absolute
        path for the lock file, which is useful for programs that do not run
        as root.
    @param timeout: Seconds to wait for the lock.
    @return: 
         0 - success
        -1 - another daemon is still active
        -2 - some other error
 
    The lock will be released when the caller terminates. No unlock is
    required.

    This is much slicker than the old way of writing our pid out to a
    file and then reading in that file and parsing ps output to see if
    that pid still exists.
    """
    
    if progname.startswith(os.path.sep):
        lockfile = progname
    else:
        lockfile = "/var/run/%s.pid" % (progname.replace(os.path.sep, "_"))
 
    try: 
        fd = os.open(lockfile, os.O_CREAT | os.O_RDWR, 0644)
    except OSError, e:
        print e
        return -2
 
    timed_out = True              # Assume failure
    t_end = time.time() + timeout
    while time.time() < t_end:
        try:
            fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
            timed_out = False
            break
        except IOError, e:
            pass
        time.sleep(0.3)         # Wait a little to minimise CPU load
        
    if timed_out:
        return -1
    else:
        # Got lock, write the program's current pid
        os.write(fd, "%d\n" % (os.getpid()))
        return 0

def get_full_volumes():

    """ Connect to the database and get a list of all of the full
    volumes. Return the list. """
    
    db = MySQLdb.connect(host="localhost", user="root", passwd="mypassword", db="bacula")
    cursor = db.cursor()
    cursor.execute("select VolumeName from Media where VolStatus='Full'")
    result = cursor.fetchall()
    return result

def upload_volumes(full_volumes):

    """ Use bitbucket to establish a connection to S3. Iterate over
    full_volumes and upload each one.  Bitbucket is cool in that it
    makes the files to upload keys in a dictionary. The bucket is the
    dictionary, the keys are the files, the values associated with the
    keys are the contents. So to put data into file on S3 we just
    assign the data to a key and it gets uploaded. It automatically
    checks if the contents are the same using md5sum in the
    implementation of the bucket object and if they are the same
    already it does not upload the data.  """

    connection = bitbucket.connect(access_key=AWS_ACCESS_KEY_ID, secret_key=AWS_SECRET_ACCESS_KEY)
    bucket = connection.get_bucket(BUCKET_NAME)
    for record in full_volumes:
        file=("%s/%s" % (BACKUP_DIR, record[0]))
        print file
        if os.path.exists(file):

            # This while causes us to loop which retries the
            # upload. We increment attempts each time through and if
            # attempts is less than 3 we raise ValueError which gets
            # caught with a pass which causes us to hit the bottom of
            # the loop so we start over. If attempts is 3 or greater
            # we don't raise ValueError which causes us to hit the
            # else clause which contains a break which gets us out of
            # the loop.

            attempts = 0
            while 1:
                # This try for implementing the retry mechanism
                try:

                    # This try for ignoring empty files
                    try:
                        if bucket.has_key(record[0]):
                            bits = bucket[record[0]]
                            bits.filename = file
                        else:
                            bits = bitbucket.Bits(filename=file)
                            # Here's where we assign bits (read in
                            # from the file) to record[0] (the
                            # filename in S3) to a key in the bucket.
                            bucket[record[0]] = bits
                        os.unlink("%s/%s" % (BACKUP_DIR, record[0]))
                    except bitbucket.BitBucketEmptyError:
                        print 'sync_dir: Empty File - Ignored %s' % fullpath
                    attempts = attempts + 1
                    if attempts < 3:
                        raise ValueError

                    
                except ValueError:
                    print "Retrying...Retry number ", attempts+1
                    pass # retry
                else:
                    break # done


if __name__ == "__main__":

    # Check our lockfile and see if another copy is already running
    if daemonlock("s3-upload") != 0:
        sys.exit()
    # Connect to the bacula db and get our list of full volumes to
    # upload
    full_volumes = get_full_volumes()
    # Do the uploads to S3
    upload_volumes(full_volumes)