#!/bin/env python ''' This script downloads and repacks official rust language builds with the necessary tool and target support for the Firefox build environment. ''' from __future__ import absolute_import, print_function import argparse import os.path import sys import requests import subprocess import toml def log(msg): print('repack: %s' % msg) def fetch_file(url): '''Download a file from the given url if it's not already present.''' filename = os.path.basename(url) if os.path.exists(filename): return r = requests.get(url, stream=True) r.raise_for_status() with open(filename, 'wb') as fd: for chunk in r.iter_content(4096): fd.write(chunk) def sha256sum(): '''Return the command for verifying SHA-2 256-bit checksums.''' if sys.platform.startswith('darwin'): return 'shasum' else: return 'sha256sum' def fetch(url): '''Download and verify a package url.''' base = os.path.basename(url) log('Fetching %s...' % base) fetch_file(url + '.asc') fetch_file(url) fetch_file(url + '.sha256') log('Verifying %s...' % base) shasum = sha256sum() subprocess.check_call([shasum, '-c', base + '.sha256']) subprocess.check_call(['gpg', '--verify', base + '.asc', base]) if True: subprocess.check_call([ 'keybase', 'pgp', 'verify', '-d', base + '.asc', '-i', base, ]) def install(filename, target): '''Run a package's installer script against the given target directory.''' log('Unpacking %s...' % filename) subprocess.check_call(['tar', 'xf', filename]) basename = filename.split('.tar')[0] log('Installing %s...' % basename) install_cmd = [os.path.join(basename, 'install.sh')] install_cmd += ['--prefix=' + os.path.abspath(target)] install_cmd += ['--disable-ldconfig'] subprocess.check_call(install_cmd) log('Cleaning %s...' % basename) subprocess.check_call(['rm', '-rf', basename]) def package(manifest, pkg, target): '''Pull out the package dict for a particular package and target from the given manifest.''' version = manifest['pkg'][pkg]['version'] info = manifest['pkg'][pkg]['target'][target] return (version, info) def fetch_package(manifest, pkg, host): version, info = package(manifest, pkg, host) log('%s %s\n %s\n %s' % (pkg, version, info['url'], info['hash'])) if not info['available']: log('%s marked unavailable for %s' % (pkg, host)) raise AssertionError fetch(info['url']) return info def fetch_std(manifest, targets): stds = [] for target in targets: info = fetch_package(manifest, 'rust-std', target) stds.append(info) return stds def tar_for_host(host): if 'linux' in host: tar_options = 'cJf' tar_ext = '.tar.xz' else: tar_options = 'cjf' tar_ext = '.tar.bz2' return tar_options, tar_ext def fetch_manifest(channel='stable'): url = 'https://static.rust-lang.org/dist/channel-rust-' + channel + '.toml' req = requests.get(url) req.raise_for_status() manifest = toml.loads(req.content) if manifest['manifest-version'] != '2': raise NotImplementedError('Unrecognized manifest version %s.' % manifest['manifest-version']) return manifest def repack(host, targets, channel='stable', suffix='', cargo_channel=None): log("Repacking rust for %s..." % host) manifest = fetch_manifest(channel) log('Using manifest for rust %s as of %s.' % (channel, manifest['date'])) if cargo_channel == channel: cargo_manifest = manifest else: cargo_manifest = fetch_manifest(cargo_channel) log('Using manifest for cargo %s as of %s.' % (cargo_channel, cargo_manifest['date'])) log('Fetching packages...') rustc = fetch_package(manifest, 'rustc', host) cargo = fetch_package(cargo_manifest, 'cargo', host) stds = fetch_std(manifest, targets) log('Installing packages...') tar_basename = 'rustc-' + host if suffix: tar_basename += '-' + suffix tar_basename += '-repack' install_dir = 'rustc' subprocess.check_call(['rm', '-rf', install_dir]) install(os.path.basename(rustc['url']), install_dir) install(os.path.basename(cargo['url']), install_dir) for std in stds: install(os.path.basename(std['url']), install_dir) pass log('Tarring %s...' % tar_basename) tar_options, tar_ext = tar_for_host(host) subprocess.check_call( ['tar', tar_options, tar_basename + tar_ext, install_dir]) subprocess.check_call(['rm', '-rf', install_dir]) def repack_cargo(host, channel='nightly'): log('Repacking cargo for %s...' % host) # Cargo doesn't seem to have a .toml manifest. base_url = 'https://static.rust-lang.org/cargo-dist/' req = requests.get(os.path.join(base_url, 'channel-cargo-' + channel)) req.raise_for_status() file = '' for line in req.iter_lines(): if line.find(host) != -1: file = line.strip() if not file: log('No manifest entry for %s!' % host) return manifest = { 'date': req.headers['Last-Modified'], 'pkg': { 'cargo': { 'version': channel, 'target': { host: { 'url': os.path.join(base_url, file), 'hash': None, 'available': True, }, }, }, }, } log('Using manifest for cargo %s.' % channel) log('Fetching packages...') cargo = fetch_package(manifest, 'cargo', host) log('Installing packages...') install_dir = 'cargo' subprocess.check_call(['rm', '-rf', install_dir]) install(os.path.basename(cargo['url']), install_dir) tar_basename = 'cargo-%s-repack' % host log('Tarring %s...' % tar_basename) tar_options, tar_ext = tar_for_host(host) subprocess.check_call( ['tar', tar_options, tar_basename + tar_ext, install_dir]) subprocess.check_call(['rm', '-rf', install_dir]) # rust platform triples android = "armv7-linux-androideabi" android_x86 = "i686-linux-android" android_aarch64 = "aarch64-linux-android" linux64 = "x86_64-unknown-linux-gnu" linux32 = "i686-unknown-linux-gnu" mac64 = "x86_64-apple-darwin" mac32 = "i686-apple-darwin" win64 = "x86_64-pc-windows-msvc" win32 = "i686-pc-windows-msvc" mingw32 = "i686-pc-windows-gnu" def args(): '''Read command line arguments and return options.''' parser = argparse.ArgumentParser() parser.add_argument('--channel', help='Release channel to use: ' 'stable, beta, or nightly', default='stable') parser.add_argument('--cargo-channel', help='Release channel to use for cargo: ' 'stable, beta, or nightly.' 'Defaults to the same as --channel.') args = parser.parse_args() if not args.cargo_channel: args.cargo_channel = args.channel return args if __name__ == '__main__': args = vars(args()) repack(mac64, [mac64], **args) repack(win32, [win32], **args) repack(win64, [win64], **args) repack(linux64, [linux64, linux32], **args) repack(linux64, [linux64, mac64], suffix='mac-cross', **args) repack(linux64, [linux64, android, android_x86, android_aarch64], suffix='android-cross', **args) repack(linux64, [linux64, win32, mingw32], suffix='mingw32-cross', **args)