# $Id$ ############################################################################# # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # ############################################################################## import os import S3 import time from datetime import datetime, timedelta from s3syncitem import SyncItem, SyncResponse def listdirs(base, filepaths): if not os.path.exists(base): return filepaths names = os.listdir(base) for name in names: fullpath = os.path.join(base, name) filepaths.append(fullpath) if os.path.isdir(fullpath): listdirs(fullpath, filepaths) return filepaths def makedirs(path): if not os.path.exists(path): os.makedirs(path) def get_timestamp(filepath): return os.stat(filepath)[8] + time.timezone def set_timestamp(filepath, timestamp): t = int(timestamp) t -= time.timezone os.utime(filepath, (t, t)) class S3Sync(object): def __init__(self, s3_access_key, s3_secret_key, base_dir, bucket_name): self._s3_access_key = s3_access_key self._s3_secret_key = s3_secret_key self._bucket_name = bucket_name self._base_dir = base_dir conn = S3.AWSAuthConnection(s3_access_key, s3_secret_key) conn.create_bucket(self._bucket_name) self._conn = conn self._infos = {} self.make_local_info() self.make_server_info() def make_local_info(self): filepaths = listdirs(self._base_dir, []) for filepath in filepaths: key = filepath.replace(self._base_dir, '').replace(os.sep, '/')[1:] info = self._infos.get(key, SyncItem(key)) info.set_localpath(filepath) self._infos[key] = info def make_server_info(self): objects = self._conn.list_bucket(self._bucket_name).entries for object in objects: key = object.key info = self._infos.get(key, SyncItem(key)) info.set_s3object(object) self._infos[key] = info def upload_item(self, info): timestamp = str(get_timestamp(info.get_localpath())) if info.is_dir(): filedata = 'this is directory holder.' itemtype = 'directory' else: filedata = file(info.get_localpath(), 'rb').read() itemtype = 'file' if info.is_changed(): elapse = time.time() result = self._conn.put(self._bucket_name, info.key, S3.S3Object(filedata), {'Content-Type': info.get_mimetype(), S3.METADATA_PREFIX + 'itemtype': itemtype, S3.METADATA_PREFIX + 'timestamp': timestamp, }, ) elapse = time.time() - elapse response = SyncResponse(result.http_response.status, result.http_response.reason, elapse, info.get_size(), ) else: response = SyncResponse(304, 'Not Modified', 0, ) return response def download_item(self, info): key = info.key elapse = time.time() result = self._conn.get(self._bucket_name, key, ) elapse = time.time() - elapse response = SyncResponse(result.http_response.status, result.http_response.reason, elapse, len(result.object.data), ) itemtype = result.object.metadata.get('itemtype') timestamp = result.object.metadata.get('timestamp') path = key.split('/') if itemtype == 'file': dirpath = os.path.join(self._base_dir, *path[:-1]) makedirs(dirpath) filepath = os.path.join(dirpath, path[-1]) f = file(filepath, 'wb') f.write(result.object.data) f.close() set_timestamp(filepath, timestamp) elif itemtype == 'directory': dirpath = os.path.join(self._base_dir, *path) makedirs(dirpath) else: raise ValueError,'Unknown ItemType' return response def sync(self, out): keys = self._infos.keys() keys.sort() for key in keys: info = self._infos[key] self.sync_one(info, out) def sync_one(self, info, out): if info.is_local_exist(): response = self.upload_item(info) else: response = self.download_item(info)