amprolla

devuan's apt repo merger
git clone git://parazyd.cf/amprolla.git
Log | Files | Refs | README.md | LICENSE

commit cf78750a6a7d40fe6025a81a2738f71a19a2a042
parent 89497bf3ee81e2b724808fed0f409aa3c41c2444
Author: parazyd <parazyd@dyne.org>
Date:   Tue,  6 Jun 2017 20:25:16 +0200

parallelize Release generation

Diffstat:
amprolla_merge.py | 62++++++++++++++++++++++++++++++++++----------------------------
1 file changed, 34 insertions(+), 28 deletions(-)

diff --git a/amprolla_merge.py b/amprolla_merge.py @@ -102,35 +102,33 @@ def merge(packages_list): write_packages(new_pkgs, new_out) -def gen_release(s): +def gen_release(suite): """ - Generates a Release file for a given main suite (jessie/ascii/unstable) + Generates a Release file for a given suite (jessie/ascii/unstable) """ - for suite in suites[s]: - filelist = [] - print('Crawling %s' % suite) - rootdir = join(mergedir, mergesubdir, suite) - for cat in categories: - for arch in arches: - if arch == 'source': - flist = srcfiles - else: - flist = pkgfiles + filelist = [] + print('Crawling %s' % suite) + rootdir = join(mergedir, mergesubdir, suite) + for cat in categories: + for arch in arches: + if arch == 'source': + flist = srcfiles + else: + flist = pkgfiles - for fl in flist: - filelist.append(join(rootdir, cat, arch, fl)) - if arch != 'source': - filelist.append(join(rootdir, cat, - 'debian-installer', arch, fl)) + for i in flist: + filelist.append(join(rootdir, cat, arch, i)) + if arch != 'source': + filelist.append(join(rootdir, cat, + 'debian-installer', arch, i)) - newrfl = join(rootdir, 'Release') - oldrfl = newrfl.replace(join(mergedir, mergesubdir), - join(spooldir, repos['devuan']['dists'])) + newrfl = join(rootdir, 'Release') + oldrfl = newrfl.replace(join(mergedir, mergesubdir), + join(spooldir, repos['devuan']['dists'])) - print('Writing Release') - write_release(oldrfl, newrfl, filelist, rootdir) - # break + print('Writing Release') + write_release(oldrfl, newrfl, filelist, rootdir) def main_merge(packages_file): @@ -167,11 +165,19 @@ def main(): pkg.append(join(j, i, mrgfile)) # pprint(pkg) - p = Pool(4) # Set it to the number of CPUs you want to use - p.map(main_merge, pkg) - - for st in suites: - gen_release(st) + mrgpool = Pool(4) # Set it to the number of CPUs you want to use + mrgpool.map(main_merge, pkg) + mrgpool.close() + + rel_list = [] + for i in suites: + for j in suites[i]: + rel_list.append(j) + # gen_release(j) + + relpool = Pool(4) # Set it to the number of CPUs you want to use + relpool.map(gen_release, rel_list) + relpool.close() if __name__ == '__main__':