#!/usr/bin/env python # ***** BEGIN LICENSE BLOCK ***** # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. # ***** END LICENSE BLOCK ***** """mobile_l10n.py This currently supports nightly and release single locale repacks for Android. This also creates nightly updates. """ import glob import os import re import subprocess import sys import time import shlex try: import simplejson as json assert json except ImportError: import json # load modules from parent dir sys.path.insert(1, os.path.dirname(sys.path[0])) from mozharness.base.errors import BaseErrorList, MakefileErrorList from mozharness.base.log import OutputParser from mozharness.base.transfer import TransferMixin from mozharness.mozilla.buildbot import BuildbotMixin from mozharness.mozilla.purge import PurgeMixin from mozharness.mozilla.release import ReleaseMixin from mozharness.mozilla.signing import MobileSigningMixin from mozharness.mozilla.tooltool import TooltoolMixin from mozharness.base.vcs.vcsbase import MercurialScript from mozharness.mozilla.l10n.locales import LocalesMixin from mozharness.mozilla.mock import MockMixin from mozharness.mozilla.updates.balrog import BalrogMixin from mozharness.base.python import VirtualenvMixin from mozharness.mozilla.taskcluster_helper import Taskcluster # MobileSingleLocale {{{1 class MobileSingleLocale(MockMixin, LocalesMixin, ReleaseMixin, MobileSigningMixin, TransferMixin, TooltoolMixin, BuildbotMixin, PurgeMixin, MercurialScript, BalrogMixin, VirtualenvMixin): config_options = [[ ['--locale', ], {"action": "extend", "dest": "locales", "type": "string", "help": "Specify the locale(s) to sign and update" } ], [ ['--locales-file', ], {"action": "store", "dest": "locales_file", "type": "string", "help": "Specify a file to determine which locales to sign and update" } ], [ ['--tag-override', ], {"action": "store", "dest": "tag_override", "type": "string", "help": "Override the tags set for all repos" } ], [ ['--user-repo-override', ], {"action": "store", "dest": "user_repo_override", "type": "string", "help": "Override the user repo path for all repos" } ], [ ['--release-config-file', ], {"action": "store", "dest": "release_config_file", "type": "string", "help": "Specify the release config file to use" } ], [ ['--key-alias', ], {"action": "store", "dest": "key_alias", "type": "choice", "default": "nightly", "choices": ["nightly", "release"], "help": "Specify the signing key alias" } ], [ ['--this-chunk', ], {"action": "store", "dest": "this_locale_chunk", "type": "int", "help": "Specify which chunk of locales to run" } ], [ ['--total-chunks', ], {"action": "store", "dest": "total_locale_chunks", "type": "int", "help": "Specify the total number of chunks of locales" } ], [ ["--disable-mock"], {"dest": "disable_mock", "action": "store_true", "help": "do not run under mock despite what gecko-config says", } ], [ ['--revision', ], {"action": "store", "dest": "revision", "type": "string", "help": "Override the gecko revision to use (otherwise use buildbot supplied" " value, or en-US revision) "} ]] def __init__(self, require_config_file=True): buildscript_kwargs = { 'all_actions': [ "clobber", "pull", "clone-locales", "list-locales", "setup", "repack", "validate-repacks-signed", "upload-repacks", "create-virtualenv", "taskcluster-upload", "submit-to-balrog", "summary", ], 'config': { 'taskcluster_credentials_file': 'oauth.txt', 'virtualenv_modules': [ 'requests==2.8.1', 'PyHawk-with-a-single-extra-commit==0.1.5', 'taskcluster==0.0.26', ], 'virtualenv_path': 'venv', }, } LocalesMixin.__init__(self) MercurialScript.__init__( self, config_options=self.config_options, require_config_file=require_config_file, **buildscript_kwargs ) self.base_package_name = None self.buildid = None self.make_ident_output = None self.repack_env = None self.enUS_revision = None self.revision = None self.upload_env = None self.version = None self.upload_urls = {} self.locales_property = {} # Helper methods {{{2 def query_repack_env(self): if self.repack_env: return self.repack_env c = self.config replace_dict = {} if c.get('release_config_file'): rc = self.query_release_config() replace_dict = { 'version': rc['version'], 'buildnum': rc['buildnum'] } repack_env = self.query_env(partial_env=c.get("repack_env"), replace_dict=replace_dict) if c.get('base_en_us_binary_url') and c.get('release_config_file'): rc = self.query_release_config() repack_env['EN_US_BINARY_URL'] = c['base_en_us_binary_url'] % replace_dict if 'MOZ_SIGNING_SERVERS' in os.environ: repack_env['MOZ_SIGN_CMD'] = subprocess.list2cmdline(self.query_moz_sign_cmd(formats=['jar'])) self.repack_env = repack_env return self.repack_env def query_l10n_env(self): return self.query_env() def query_upload_env(self): if self.upload_env: return self.upload_env c = self.config replace_dict = { 'buildid': self.query_buildid(), 'version': self.query_version(), } replace_dict.update(c) # Android l10n builds use a non-standard location for l10n files. Other # builds go to 'mozilla-central-l10n', while android builds add part of # the platform name as well, like 'mozilla-central-android-api-15-l10n'. # So we override the branch with something that contains the platform # name. replace_dict['branch'] = c['upload_branch'] replace_dict['post_upload_extra'] = ' '.join(c.get('post_upload_extra', [])) upload_env = self.query_env(partial_env=c.get("upload_env"), replace_dict=replace_dict) if 'MOZ_SIGNING_SERVERS' in os.environ: upload_env['MOZ_SIGN_CMD'] = subprocess.list2cmdline(self.query_moz_sign_cmd()) if self.query_is_release_or_beta(): upload_env['MOZ_PKG_VERSION'] = '%(version)s' % replace_dict self.upload_env = upload_env return self.upload_env def _query_make_ident_output(self): """Get |make ident| output from the objdir. Only valid after setup is run. """ if self.make_ident_output: return self.make_ident_output env = self.query_repack_env() dirs = self.query_abs_dirs() output = self.get_output_from_command_m(["make", "ident"], cwd=dirs['abs_locales_dir'], env=env, silent=True, halt_on_failure=True) parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=MakefileErrorList) parser.add_lines(output) self.make_ident_output = output return output def query_buildid(self): """Get buildid from the objdir. Only valid after setup is run. """ if self.buildid: return self.buildid r = re.compile("buildid (\d+)") output = self._query_make_ident_output() for line in output.splitlines(): m = r.match(line) if m: self.buildid = m.groups()[0] return self.buildid def query_revision(self): """ Get the gecko revision in this order of precedence * cached value * command line arg --revision (development, taskcluster) * buildbot properties (try with buildbot forced build) * buildbot change (try with buildbot scheduler) * from the en-US build (m-c & m-a) This will fail the last case if the build hasn't been pulled yet. """ if self.revision: return self.revision self.read_buildbot_config() config = self.config revision = None if config.get("revision"): revision = config["revision"] elif 'revision' in self.buildbot_properties: revision = self.buildbot_properties['revision'] elif (self.buildbot_config and self.buildbot_config.get('sourcestamp', {}).get('revision')): revision = self.buildbot_config['sourcestamp']['revision'] elif self.buildbot_config and self.buildbot_config.get('revision'): revision = self.buildbot_config['revision'] elif config.get("update_gecko_source_to_enUS", True): revision = self._query_enUS_revision() if not revision: self.fatal("Can't determine revision!") self.revision = str(revision) return self.revision def _query_enUS_revision(self): """Get revision from the objdir. Only valid after setup is run. """ if self.enUS_revision: return self.enUS_revision r = re.compile(r"^gecko_revision ([0-9a-f]+\+?)$") output = self._query_make_ident_output() for line in output.splitlines(): match = r.match(line) if match: self.enUS_revision = match.groups()[0] return self.enUS_revision def _query_make_variable(self, variable, make_args=None): make = self.query_exe('make') env = self.query_repack_env() dirs = self.query_abs_dirs() if make_args is None: make_args = [] # TODO error checking output = self.get_output_from_command_m( [make, "echo-variable-%s" % variable] + make_args, cwd=dirs['abs_locales_dir'], silent=True, env=env ) parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=MakefileErrorList) parser.add_lines(output) return output.strip() def query_base_package_name(self): """Get the package name from the objdir. Only valid after setup is run. """ if self.base_package_name: return self.base_package_name self.base_package_name = self._query_make_variable( "PACKAGE", make_args=['AB_CD=%(locale)s'] ) return self.base_package_name def query_version(self): """Get the package name from the objdir. Only valid after setup is run. """ if self.version: return self.version c = self.config if c.get('release_config_file'): rc = self.query_release_config() self.version = rc['version'] else: self.version = self._query_make_variable("MOZ_APP_VERSION") return self.version def query_upload_url(self, locale): if locale in self.upload_urls: return self.upload_urls[locale] else: self.error("Can't determine the upload url for %s!" % locale) def query_abs_dirs(self): if self.abs_dirs: return self.abs_dirs abs_dirs = super(MobileSingleLocale, self).query_abs_dirs() dirs = { 'abs_tools_dir': os.path.join(abs_dirs['base_work_dir'], 'tools'), 'build_dir': os.path.join(abs_dirs['base_work_dir'], 'build'), } abs_dirs.update(dirs) self.abs_dirs = abs_dirs return self.abs_dirs def add_failure(self, locale, message, **kwargs): self.locales_property[locale] = "Failed" prop_key = "%s_failure" % locale prop_value = self.query_buildbot_property(prop_key) if prop_value: prop_value = "%s %s" % (prop_value, message) else: prop_value = message self.set_buildbot_property(prop_key, prop_value, write_to_file=True) MercurialScript.add_failure(self, locale, message=message, **kwargs) def summary(self): MercurialScript.summary(self) # TODO we probably want to make this configurable on/off locales = self.query_locales() for locale in locales: self.locales_property.setdefault(locale, "Success") self.set_buildbot_property("locales", json.dumps(self.locales_property), write_to_file=True) # Actions {{{2 def clobber(self): self.read_buildbot_config() dirs = self.query_abs_dirs() c = self.config objdir = os.path.join(dirs['abs_work_dir'], c['mozilla_dir'], c['objdir']) super(MobileSingleLocale, self).clobber(always_clobber_dirs=[objdir]) def pull(self): c = self.config dirs = self.query_abs_dirs() repos = [] replace_dict = {} if c.get("user_repo_override"): replace_dict['user_repo_override'] = c['user_repo_override'] # this is OK so early because we get it from buildbot, or # the command line for local dev replace_dict['revision'] = self.query_revision() for repository in c['repos']: current_repo = {} for key, value in repository.iteritems(): try: current_repo[key] = value % replace_dict except TypeError: # pass through non-interpolables, like booleans current_repo[key] = value except KeyError: self.error('not all the values in "{0}" can be replaced. Check your configuration'.format(value)) raise repos.append(current_repo) self.info("repositories: %s" % repos) self.vcs_checkout_repos(repos, parent_dir=dirs['abs_work_dir'], tag_override=c.get('tag_override')) def clone_locales(self): self.pull_locale_source() # list_locales() is defined in LocalesMixin. def _setup_configure(self, buildid=None): c = self.config dirs = self.query_abs_dirs() env = self.query_repack_env() make = self.query_exe("make") if self.run_command_m([make, "-f", "client.mk", "configure"], cwd=dirs['abs_mozilla_dir'], env=env, error_list=MakefileErrorList): self.fatal("Configure failed!") # Run 'make export' in objdir/config to get nsinstall self.run_command_m([make, 'export'], cwd=os.path.join(dirs['abs_objdir'], 'config'), env=env, error_list=MakefileErrorList, halt_on_failure=True) # Run 'make buildid.h' in objdir/ to get the buildid.h file cmd = [make, 'buildid.h'] if buildid: cmd.append('MOZ_BUILD_DATE=%s' % str(buildid)) self.run_command_m(cmd, cwd=dirs['abs_objdir'], env=env, error_list=MakefileErrorList, halt_on_failure=True) def setup(self): c = self.config dirs = self.query_abs_dirs() mozconfig_path = os.path.join(dirs['abs_mozilla_dir'], '.mozconfig') self.copyfile(os.path.join(dirs['abs_work_dir'], c['mozconfig']), mozconfig_path) # TODO stop using cat cat = self.query_exe("cat") make = self.query_exe("make") self.run_command_m([cat, mozconfig_path]) env = self.query_repack_env() if self.config.get("tooltool_config"): self.tooltool_fetch( self.config['tooltool_config']['manifest'], output_dir=self.config['tooltool_config']['output_dir'] % self.query_abs_dirs(), ) self._setup_configure() self.run_command_m([make, "wget-en-US"], cwd=dirs['abs_locales_dir'], env=env, error_list=MakefileErrorList, halt_on_failure=True) self.run_command_m([make, "unpack"], cwd=dirs['abs_locales_dir'], env=env, error_list=MakefileErrorList, halt_on_failure=True) # on try we want the source we already have, otherwise update to the # same as the en-US binary if self.config.get("update_gecko_source_to_enUS", True): revision = self._query_enUS_revision() if not revision: self.fatal("Can't determine revision!") hg = self.query_exe("hg") # TODO do this through VCSMixin instead of hardcoding hg self.run_command_m([hg, "update", "-r", revision], cwd=dirs["abs_mozilla_dir"], env=env, error_list=BaseErrorList, halt_on_failure=True) self.set_buildbot_property('revision', revision, write_to_file=True) # Configure again since the hg update may have invalidated it. buildid = self.query_buildid() self._setup_configure(buildid=buildid) def repack(self): # TODO per-locale logs and reporting. dirs = self.query_abs_dirs() locales = self.query_locales() make = self.query_exe("make") repack_env = self.query_repack_env() success_count = total_count = 0 for locale in locales: total_count += 1 self.enable_mock() result = self.run_compare_locales(locale) self.disable_mock() if result: self.add_failure(locale, message="%s failed in compare-locales!" % locale) continue if self.run_command_m([make, "installers-%s" % locale], cwd=dirs['abs_locales_dir'], env=repack_env, error_list=MakefileErrorList, halt_on_failure=False): self.add_failure(locale, message="%s failed in make installers-%s!" % (locale, locale)) continue success_count += 1 self.summarize_success_count(success_count, total_count, message="Repacked %d of %d binaries successfully.") def validate_repacks_signed(self): c = self.config dirs = self.query_abs_dirs() locales = self.query_locales() base_package_name = self.query_base_package_name() base_package_dir = os.path.join(dirs['abs_objdir'], 'dist') repack_env = self.query_repack_env() success_count = total_count = 0 for locale in locales: total_count += 1 signed_path = os.path.join(base_package_dir, base_package_name % {'locale': locale}) # We need to wrap what this function does with mock, since # MobileSigningMixin doesn't know about mock self.enable_mock() status = self.verify_android_signature( signed_path, script=c['signature_verification_script'], env=repack_env, key_alias=c['key_alias'], ) self.disable_mock() if status: self.add_failure(locale, message="Errors verifying %s binary!" % locale) # No need to rm because upload is per-locale continue success_count += 1 self.summarize_success_count(success_count, total_count, message="Validated signatures on %d of %d binaries successfully.") def taskcluster_upload(self): auth = os.path.join(os.getcwd(), self.config['taskcluster_credentials_file']) credentials = {} execfile(auth, credentials) client_id = credentials.get('taskcluster_clientId') access_token = credentials.get('taskcluster_accessToken') if not client_id or not access_token: self.warning('Skipping S3 file upload: No taskcluster credentials.') return self.activate_virtualenv() dirs = self.query_abs_dirs() locales = self.query_locales() make = self.query_exe("make") upload_env = self.query_upload_env() cwd = dirs['abs_locales_dir'] branch = self.config['branch'] revision = self.query_revision() repo = self.query_l10n_repo() pushinfo = self.vcs_query_pushinfo(repo, revision, vcs='hg') pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(pushinfo.pushdate)) routes_json = os.path.join(self.query_abs_dirs()['abs_mozilla_dir'], 'testing/mozharness/configs/routes.json') with open(routes_json) as routes_file: contents = json.load(routes_file) templates = contents['l10n'] for locale in locales: output = self.get_output_from_command_m( "%s echo-variable-UPLOAD_FILES AB_CD=%s" % (make, locale), cwd=cwd, env=upload_env, ) files = shlex.split(output) abs_files = [os.path.abspath(os.path.join(cwd, f)) for f in files] routes = [] fmt = { 'index': self.config.get('taskcluster_index', 'index.garbage.staging'), 'project': branch, 'head_rev': revision, 'pushdate': pushdate, 'year': pushdate[0:4], 'month': pushdate[4:6], 'day': pushdate[6:8], 'build_product': self.config['stage_product'], 'build_name': self.query_build_name(), 'build_type': self.query_build_type(), 'locale': locale, } for template in templates: routes.append(template.format(**fmt)) self.info('Using routes: %s' % routes) tc = Taskcluster(branch, pushinfo.pushdate, # Use pushdate as the rank client_id, access_token, self.log_obj, ) task = tc.create_task(routes) tc.claim_task(task) for upload_file in abs_files: tc.create_artifact(task, upload_file) tc.report_completed(task) def upload_repacks(self): c = self.config dirs = self.query_abs_dirs() locales = self.query_locales() make = self.query_exe("make") base_package_name = self.query_base_package_name() version = self.query_version() upload_env = self.query_upload_env() success_count = total_count = 0 buildnum = None if c.get('release_config_file'): rc = self.query_release_config() buildnum = rc['buildnum'] for locale in locales: if self.query_failure(locale): self.warning("Skipping previously failed locale %s." % locale) continue total_count += 1 if c.get('base_post_upload_cmd'): upload_env['POST_UPLOAD_CMD'] = c['base_post_upload_cmd'] % {'version': version, 'locale': locale, 'buildnum': str(buildnum), 'post_upload_extra': ' '.join(c.get('post_upload_extra', []))} output = self.get_output_from_command_m( # Ugly hack to avoid |make upload| stderr from showing up # as get_output_from_command errors "%s upload AB_CD=%s 2>&1" % (make, locale), cwd=dirs['abs_locales_dir'], env=upload_env, silent=True ) parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=MakefileErrorList) parser.add_lines(output) if parser.num_errors: self.add_failure(locale, message="%s failed in make upload!" % (locale)) continue package_name = base_package_name % {'locale': locale} r = re.compile("(http.*%s)" % package_name) for line in output.splitlines(): m = r.match(line) if m: self.upload_urls[locale] = m.groups()[0] self.info("Found upload url %s" % self.upload_urls[locale]) # XXX Move the files to a SIMPLE_NAME format until we can enable # Simple names in the build system if self.config.get("simple_name_move"): # Assume an UPLOAD PATH upload_target = self.config["upload_env"]["UPLOAD_PATH"] target_path = os.path.join(upload_target, locale) self.mkdir_p(target_path) glob_name = "*.%s.android-arm.*" % locale for f in glob.glob(os.path.join(upload_target, glob_name)): glob_extension = f[f.rfind('.'):] self.move(os.path.join(f), os.path.join(target_path, "target%s" % glob_extension)) self.log("Converted uploads for %s to simple names" % locale) success_count += 1 self.summarize_success_count(success_count, total_count, message="Make Upload for %d of %d locales successful.") def checkout_tools(self): dirs = self.query_abs_dirs() # We need hg.m.o/build/tools checked out self.info("Checking out tools") repos = [{ 'repo': self.config['tools_repo'], 'vcs': "hg", 'branch': "default", 'dest': dirs['abs_tools_dir'], }] rev = self.vcs_checkout(**repos[0]) self.set_buildbot_property("tools_revision", rev, write_to_file=True) def query_apkfile_path(self,locale): dirs = self.query_abs_dirs() apkdir = os.path.join(dirs['abs_objdir'], 'dist') r = r"(\.)" + re.escape(locale) + r"(\.*)" apks = [] for f in os.listdir(apkdir): if f.endswith(".apk") and re.search(r, f): apks.append(f) if len(apks) == 0: self.fatal("Found no apks files in %s, don't know what to do:\n%s" % (apkdir, apks), exit_code=1) return os.path.join(apkdir, apks[0]) def query_is_release_or_beta(self): return bool(self.config.get("is_release_or_beta")) def submit_to_balrog(self): if not self.query_is_nightly() and not self.query_is_release_or_beta(): self.info("Not a nightly or release build, skipping balrog submission.") return self.checkout_tools() dirs = self.query_abs_dirs() locales = self.query_locales() if not self.config.get('taskcluster_nightly'): balrogReady = True for locale in locales: apk_url = self.query_upload_url(locale) if not apk_url: self.add_failure(locale, message="Failed to detect %s url in make upload!" % (locale)) balrogReady = False continue if not balrogReady: return self.fatal(message="Not all repacks successful, abort without submitting to balrog") env = self.query_upload_env() for locale in locales: apkfile = self.query_apkfile_path(locale) if self.config.get('taskcluster_nightly'): # Taskcluster needs stage_platform self.set_buildbot_property("stage_platform", self.config.get("stage_platform")) self.set_buildbot_property("branch", self.config.get("branch")) else: apk_url = self.query_upload_url(locale) self.set_buildbot_property("completeMarUrl", apk_url) # The Balrog submitter translates this platform into a build target # via https://github.com/mozilla/build-tools/blob/master/lib/python/release/platforms.py#L23 self.set_buildbot_property( "platform", self.buildbot_config["properties"]["platform"]) #TODO: Is there a better way to get this? # Set other necessary properties for Balrog submission. None need to # be passed back to buildbot, so we won't write them to the properties #files. self.set_buildbot_property("locale", locale) self.set_buildbot_property("appVersion", self.query_version()) self.set_buildbot_property("appName", "Fennec") # TODO: don't hardcode self.set_buildbot_property("hashType", "sha512") self.set_buildbot_property("completeMarSize", self.query_filesize(apkfile)) self.set_buildbot_property("completeMarHash", self.query_sha512sum(apkfile)) self.set_buildbot_property("isOSUpdate", False) self.set_buildbot_property("buildid", self.query_buildid()) if self.config.get('taskcluster_nightly'): props_path = os.path.join(env["UPLOAD_PATH"], locale, 'balrog_props.json') self.generate_balrog_props(props_path) else: if not self.config.get("balrog_servers"): self.info("balrog_servers not set; skipping balrog submission.") return if self.query_is_nightly(): self.submit_balrog_updates(release_type="nightly") else: self.submit_balrog_updates(release_type="release") if not self.query_is_nightly(): self.submit_balrog_release_pusher(dirs) # main {{{1 if __name__ == '__main__': single_locale = MobileSingleLocale() single_locale.run_and_exit()