#!/usr/bin/env python # -*- coding: utf-8 -*- import errno import glob import hashlib import json import os import shutil import tarfile import tempfile import urllib.request, urllib.parse, urllib.error import urllib.request, urllib.error, urllib.parse from distutils.version import LooseVersion OK = '\033[92m' FAIL = '\033[91m' END = '\033[0m' DISTRIBUTION = "setuptools" class SetuptoolsOldReleasesWithoutZip(object): """docstring for SetuptoolsOldReleases""" def __init__(self): super(SetuptoolsOldReleasesWithoutZip, self).__init__() self.dirpath = tempfile.mkdtemp(prefix=DISTRIBUTION) print("Downloading %s releases..." % DISTRIBUTION) print("All releases will be downloaded to %s" % self.dirpath) self.data_json_setuptools = self.get_json_data(DISTRIBUTION) self.valid_releases_numbers = sorted([ release for release in self.data_json_setuptools['releases'] # This condition is motivated by 13.0 release, which # comes as "13.0": [], in the json if self.data_json_setuptools['releases'][release] ], key=LooseVersion) self.total_downloaded_ok = 0 def get_json_data(self, package_name): """ "releases": { "0.7.2": [ { "has_sig": false, "upload_time": "2013-06-09T16:10:00", "comment_text": "", "python_version": "source", "url": "https://pypi.python.org/packages/source/s/setuptools/setuptools-0.7.2.tar.gz", # NOQA "md5_digest": "de44cd90f8a1c713d6c2bff67bbca65d", "downloads": 159014, "filename": "setuptools-0.7.2.tar.gz", "packagetype": "sdist", "size": 633077 } ], "0.7.3": [ { "has_sig": false, "upload_time": "2013-06-18T21:08:56", "comment_text": "", "python_version": "source", "url": "https://pypi.python.org/packages/source/s/setuptools/setuptools-0.7.3.tar.gz", # NOQA "md5_digest": "c854adacbf9067d330a847f06f7a8eba", "downloads": 30594, "filename": "setuptools-0.7.3.tar.gz", "packagetype": "sdist", "size": 751152 } ], "12.3": [ { "has_sig": false, "upload_time": "2015-02-26T19:15:51", "comment_text": "", "python_version": "3.4", "url": "https://pypi.python.org/packages/3.4/s/setuptools/setuptools-12.3-py2.py3-none-any.whl", # NOQA "md5_digest": "31f51a38497a70efadf5ce8d4c2211ab", "downloads": 288451, "filename": "setuptools-12.3-py2.py3-none-any.whl", "packagetype": "bdist_wheel", "size": 501904 }, { "has_sig": false, "upload_time": "2015-02-26T19:15:43", "comment_text": "", "python_version": "source", "url": "https://pypi.python.org/packages/source/s/setuptools/setuptools-12.3.tar.gz", # NOQA "md5_digest": "67614b6d560fa4f240e99cd553ec7f32", "downloads": 110109, "filename": "setuptools-12.3.tar.gz", "packagetype": "sdist", "size": 635025 }, { "has_sig": false, "upload_time": "2015-02-26T19:15:47", "comment_text": "", "python_version": "source", "url": "https://pypi.python.org/packages/source/s/setuptools/setuptools-12.3.zip", # NOQA "md5_digest": "abc799e7db6e7281535bf342bfc41a12", "downloads": 67539, "filename": "setuptools-12.3.zip", "packagetype": "sdist", "size": 678783 } ], """ url = "https://pypi.python.org/pypi/%s/json" % (package_name,) data = json.load(urllib.request.urlopen(urllib.request.Request(url))) # Mainly for debug. json_filename = "%s/%s.json" % (self.dirpath, DISTRIBUTION) with open(json_filename, 'w') as outfile: json.dump( data, outfile, sort_keys=True, indent=4, separators=(',', ': '), ) return data def get_setuptools_releases_without_zip_counterpart(self): # Get set(all_valid_releases) - set(releases_with_zip), so now we have # the releases without zip. return set(self.valid_releases_numbers) - set([ release for release in self.valid_releases_numbers for same_version_release_dict in self.data_json_setuptools['releases'][release] # NOQA if 'zip' in same_version_release_dict['filename'] ]) def download_setuptools_releases_without_zip_counterpart(self): try: releases_without_zip = self.get_setuptools_releases_without_zip_counterpart() # NOQA failed_md5_releases = [] # This is a "strange" loop, going through all releases and # testing only the release I need to download, but I thought it # would be mouch more readable than trying to iterate through # releases I need and get into traverse hell values inside dicts # inside dicts of the json to get the distribution's url to # download. for release in self.valid_releases_numbers: if release in releases_without_zip: for same_version_release_dict in self.data_json_setuptools['releases'][release]: # NOQA if 'tar.gz' in same_version_release_dict['filename']: print("Downloading %s..." % release) local_file = '%s/%s' % ( self.dirpath, same_version_release_dict["filename"] ) urllib.request.urlretrieve( same_version_release_dict["url"], local_file ) targz = open(local_file, 'rb').read() hexdigest = hashlib.md5(targz).hexdigest() if (hexdigest != same_version_release_dict['md5_digest']): # NOQA print(FAIL + "FAIL: md5 for %s didn't match!" % release + END) # NOQA failed_md5_releases.append(release) else: self.total_downloaded_ok += 1 print('Total releases without zip: %s' % len(releases_without_zip)) print('Total downloaded: %s' % self.total_downloaded_ok) if failed_md5_releases: msg = FAIL + ( "FAIL: these releases %s failed the md5 check!" % ','.join(failed_md5_releases) ) + END raise Exception(msg) elif self.total_downloaded_ok != len(releases_without_zip): msg = FAIL + ( "FAIL: Unknown error occured. Please check the logs." ) + END raise Exception(msg) else: print(OK + "All releases downloaded and md5 checked." + END) except OSError as e: if e.errno != errno.EEXIST: raise e def convert_targz_to_zip(self): print("Converting the tar.gz to zip...") files = glob.glob('%s/*.tar.gz' % self.dirpath) total_converted = 0 for targz in sorted(files, key=LooseVersion): # Extract and remove tar. tar = tarfile.open(targz) tar.extractall(path=self.dirpath) tar.close() os.remove(targz) # Zip the extracted tar. setuptools_folder_path = targz.replace('.tar.gz', '') setuptools_folder_name = setuptools_folder_path.split("/")[-1] print(setuptools_folder_name) shutil.make_archive( setuptools_folder_path, 'zip', self.dirpath, setuptools_folder_name ) # Exclude extracted tar folder. shutil.rmtree(setuptools_folder_path.replace('.zip', '')) total_converted += 1 print('Total converted: %s' % total_converted) if self.total_downloaded_ok != total_converted: msg = FAIL + ( "FAIL: Total number of downloaded releases is different" " from converted ones. Please check the logs." ) + END raise Exception(msg) print("Done with the tar.gz->zip. Check folder %s." % main.dirpath) def upload_zips_to_pypi(self): print('Uploading to pypi...') zips = sorted(glob.glob('%s/*.zip' % self.dirpath), key=LooseVersion) for zips in glob.glob('%s/*.zip' % self.dirpath): # Put the twine upload code here pass if __name__ == '__main__': main = SetuptoolsOldReleasesWithoutZip() main.download_setuptools_releases_without_zip_counterpart() main.convert_targz_to_zip() main.upload_zips_to_pypi()