diff options
author | Boris Nagaev <bnagaev@gmail.com> | 2017-01-07 04:19:09 (GMT) |
---|---|---|
committer | Boris Nagaev <bnagaev@gmail.com> | 2017-01-07 04:19:09 (GMT) |
commit | 4a7b9526f08bfcf9bf84b68dbfb030f2227a6ce0 (patch) | |
tree | 419071b6c93b1bfb3169a48557de9df354c85735 /tools/backup_from_s3.py | |
parent | 0555e10e6f85e6066b6b210a23843967c8d4b394 (diff) | |
download | mxe-4a7b9526f08bfcf9bf84b68dbfb030f2227a6ce0.zip mxe-4a7b9526f08bfcf9bf84b68dbfb030f2227a6ce0.tar.gz mxe-4a7b9526f08bfcf9bf84b68dbfb030f2227a6ce0.tar.bz2 |
add tools to create and update backup of packages
Diffstat (limited to 'tools/backup_from_s3.py')
-rwxr-xr-x | tools/backup_from_s3.py | 74 |
1 files changed, 74 insertions, 0 deletions
diff --git a/tools/backup_from_s3.py b/tools/backup_from_s3.py new file mode 100755 index 0000000..7f51413 --- /dev/null +++ b/tools/backup_from_s3.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python + +""" Download MXE packages from https://s3.amazonaws.com/mxe-pkg/ +""" + +import argparse +import hashlib +import os +import urllib +try: + import urllib2 +except: + # Python 3 + import urllib.request as urllib2 +import xml.etree.ElementTree + +def get_files(): + x = xml.etree.ElementTree.fromstring( + urllib2.urlopen("https://s3.amazonaws.com/mxe-pkg/").read() + ) + for e in x: + if not e.tag.endswith('Contents'): + continue + filename = None + md5 = None + for child in e.getchildren(): + if child.tag.endswith('Key'): + filename = child.text + if child.tag.endswith('ETag'): + md5 = child.text.replace('"', '') + if '-' in md5: + md5 = None + yield { + 'filename': filename, + 'md5': md5, + } + +def download_files(backup_dir, files): + for f in files: + url = "https://s3.amazonaws.com/mxe-pkg/%s" % ( + urllib.quote(f['filename']) + ) + data = urllib2.urlopen(url).read() + if f['md5']: + md5 = hashlib.md5(data).hexdigest() + if md5 != f['md5']: + raise Exception("md5 mismatch for " + f['filename']) + sha256 = hashlib.sha256(data).hexdigest() + name = f['filename'] + '_' + sha256 + full_name = os.path.join(backup_dir, name) + if os.path.exists(full_name): + print("File %s is already backuped" % name) + continue + print("Backup file %s" % name) + with open(full_name, 'w') as f: + f.write(data) + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + '--backup-dir', + type=str, + help='Path to backup', + required=True, + ) + args = parser.parse_args() + files = get_files() + download_files(args.backup_dir, files) + +if __name__ == '__main__': + main() |