aecb7edab9
No builds other than vanilla opt and debug builds are supported by the artifact code currently, so prevent variant builds from being replaced by artifact builds by modifying mozharness' replacement logic to replace a build with an artifact build only when it is a regular opt or debug build or when specified by a config. MozReview-Commit-ID: KUUgrbga53l
261 lines
11 KiB
Python
Executable File
261 lines
11 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# ***** BEGIN LICENSE BLOCK *****
|
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
|
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
# ***** END LICENSE BLOCK *****
|
|
"""fx_desktop_build.py.
|
|
|
|
script harness to build nightly firefox within Mozilla's build environment
|
|
and developer machines alike
|
|
|
|
author: Jordan Lund
|
|
|
|
"""
|
|
|
|
import copy
|
|
import pprint
|
|
import sys
|
|
import os
|
|
|
|
# load modules from parent dir
|
|
sys.path.insert(1, os.path.dirname(sys.path[0]))
|
|
|
|
import mozharness.base.script as script
|
|
from mozharness.mozilla.building.buildbase import BUILD_BASE_CONFIG_OPTIONS, \
|
|
BuildingConfig, BuildOptionParser, BuildScript
|
|
from mozharness.base.config import parse_config_file
|
|
from mozharness.mozilla.testing.try_tools import TryToolsMixin, try_config_options
|
|
|
|
|
|
class FxDesktopBuild(BuildScript, TryToolsMixin, object):
|
|
def __init__(self):
|
|
buildscript_kwargs = {
|
|
'config_options': BUILD_BASE_CONFIG_OPTIONS + copy.deepcopy(try_config_options),
|
|
'all_actions': [
|
|
'get-secrets',
|
|
'clobber',
|
|
'clone-tools',
|
|
'checkout-sources',
|
|
'setup-mock',
|
|
'build',
|
|
'upload-files', # upload from BB to TC
|
|
'sendchange',
|
|
'check-test',
|
|
'valgrind-test',
|
|
'package-source',
|
|
'generate-source-signing-manifest',
|
|
'multi-l10n',
|
|
'generate-build-stats',
|
|
'update',
|
|
],
|
|
'require_config_file': True,
|
|
# Default configuration
|
|
'config': {
|
|
'is_automation': True,
|
|
"pgo_build": False,
|
|
"debug_build": False,
|
|
"pgo_platforms": ['linux', 'linux64', 'win32', 'win64'],
|
|
# nightly stuff
|
|
"nightly_build": False,
|
|
'balrog_credentials_file': 'oauth.txt',
|
|
'taskcluster_credentials_file': 'oauth.txt',
|
|
'periodic_clobber': 168,
|
|
# hg tool stuff
|
|
"tools_repo": "https://hg.mozilla.org/build/tools",
|
|
# Seed all clones with mozilla-unified. This ensures subsequent
|
|
# jobs have a minimal `hg pull`.
|
|
"clone_upstream_url": "https://hg.mozilla.org/mozilla-unified",
|
|
"repo_base": "https://hg.mozilla.org",
|
|
'tooltool_url': 'https://api.pub.build.mozilla.org/tooltool/',
|
|
"graph_selector": "/server/collect.cgi",
|
|
# only used for make uploadsymbols
|
|
'old_packages': [
|
|
"%(objdir)s/dist/firefox-*",
|
|
"%(objdir)s/dist/fennec*",
|
|
"%(objdir)s/dist/seamonkey*",
|
|
"%(objdir)s/dist/thunderbird*",
|
|
"%(objdir)s/dist/install/sea/*.exe"
|
|
],
|
|
'stage_product': 'firefox',
|
|
'platform_supports_post_upload_to_latest': True,
|
|
'build_resources_path': '%(abs_src_dir)s/obj-firefox/.mozbuild/build_resources.json',
|
|
'nightly_promotion_branches': ['mozilla-central', 'mozilla-aurora'],
|
|
|
|
# try will overwrite these
|
|
'clone_with_purge': False,
|
|
'clone_by_revision': False,
|
|
'tinderbox_build_dir': None,
|
|
'to_tinderbox_dated': True,
|
|
'release_to_try_builds': False,
|
|
'include_post_upload_builddir': False,
|
|
'use_clobberer': True,
|
|
|
|
'stage_username': 'ffxbld',
|
|
'stage_ssh_key': 'ffxbld_rsa',
|
|
'virtualenv_modules': [
|
|
'requests==2.8.1',
|
|
'PyHawk-with-a-single-extra-commit==0.1.5',
|
|
'taskcluster==0.0.26',
|
|
],
|
|
'virtualenv_path': 'venv',
|
|
#
|
|
|
|
},
|
|
'ConfigClass': BuildingConfig,
|
|
}
|
|
super(FxDesktopBuild, self).__init__(**buildscript_kwargs)
|
|
|
|
def _pre_config_lock(self, rw_config):
|
|
"""grab buildbot props if we are running this in automation"""
|
|
super(FxDesktopBuild, self)._pre_config_lock(rw_config)
|
|
c = self.config
|
|
if c['is_automation']:
|
|
# parse buildbot config and add it to self.config
|
|
self.info("We are running this in buildbot, grab the build props")
|
|
self.read_buildbot_config()
|
|
###
|
|
if c.get('stage_platform'):
|
|
platform_for_log_url = c['stage_platform']
|
|
if c.get('pgo_build'):
|
|
platform_for_log_url += '-pgo'
|
|
# postrun.py uses stage_platform buildbot prop as part of the log url
|
|
self.set_buildbot_property('stage_platform',
|
|
platform_for_log_url,
|
|
write_to_file=True)
|
|
else:
|
|
self.fatal("'stage_platform' not determined and is required in your config")
|
|
|
|
if self.try_message_has_flag('artifact'):
|
|
# Not all jobs that look like builds can be made into artifact
|
|
# builds (for example, various SAN builds will not make sense as
|
|
# artifact builds). By default, only a vanilla debug or opt build
|
|
# will be replaced by an artifact build.
|
|
#
|
|
# In addition, some jobs want to specify their artifact equivalent.
|
|
# Use `artifact_flag_build_variant_in_try` to specify that variant.
|
|
#
|
|
# This is temporary, until we find a way to introduce an "artifact
|
|
# build dimension" like "opt"/"debug" into the CI configurations.
|
|
self.info('Artifact build requested in try syntax.')
|
|
|
|
variant = None
|
|
|
|
if 'artifact_flag_build_variant_in_try' in c:
|
|
variant = c['artifact_flag_build_variant_in_try']
|
|
if not variant:
|
|
self.info('Build variant has falsy `artifact_flag_build_variant_in_try`; '
|
|
'ignoring artifact build request and performing original build.')
|
|
return
|
|
self.info('Build variant has `artifact_build_variant_in_try`: "%s".' % variant)
|
|
else:
|
|
if not c.get('build_variant'):
|
|
variant = 'artifact'
|
|
elif c.get('build_variant') in ['debug', 'cross-debug']:
|
|
variant = 'debug-artifact'
|
|
|
|
if variant:
|
|
self.info('Using artifact build variant "%s".' % variant)
|
|
self._update_build_variant(rw_config, variant)
|
|
|
|
# helpers
|
|
def _update_build_variant(self, rw_config, variant='artifact'):
|
|
""" Intended for use in _pre_config_lock """
|
|
c = self.config
|
|
variant_cfg_path, _ = BuildOptionParser.find_variant_cfg_path(
|
|
'--custom-build-variant-cfg',
|
|
variant,
|
|
rw_config.config_parser
|
|
)
|
|
if not variant_cfg_path:
|
|
self.fatal('Could not find appropriate config file for variant %s' % variant)
|
|
# Update other parts of config to keep dump-config accurate
|
|
# Only dump-config is affected because most config info is set during
|
|
# initial parsing
|
|
variant_cfg_dict = parse_config_file(variant_cfg_path)
|
|
rw_config.all_cfg_files_and_dicts.append((variant_cfg_path, variant_cfg_dict))
|
|
c.update({
|
|
'build_variant': variant,
|
|
'config_files': c['config_files'] + [variant_cfg_path]
|
|
})
|
|
|
|
self.info("Updating self.config with the following from {}:".format(variant_cfg_path))
|
|
self.info(pprint.pformat(variant_cfg_dict))
|
|
c.update(variant_cfg_dict)
|
|
c['forced_artifact_build'] = True
|
|
# Bug 1231320 adds MOZHARNESS_ACTIONS in TaskCluster tasks to override default_actions
|
|
# We don't want that when forcing an artifact build.
|
|
if rw_config.volatile_config['actions']:
|
|
self.info("Updating volatile_config to include default_actions "
|
|
"from {}.".format(variant_cfg_path))
|
|
# add default actions in correct order
|
|
combined_actions = []
|
|
for a in rw_config.all_actions:
|
|
if a in c['default_actions'] or a in rw_config.volatile_config['actions']:
|
|
combined_actions.append(a)
|
|
rw_config.volatile_config['actions'] = combined_actions
|
|
self.info("Actions in volatile_config are now: {}".format(
|
|
rw_config.volatile_config['actions'])
|
|
)
|
|
# replace rw_config as well to set actions as in BaseScript
|
|
rw_config.set_config(c, overwrite=True)
|
|
rw_config.update_actions()
|
|
self.actions = tuple(rw_config.actions)
|
|
self.all_actions = tuple(rw_config.all_actions)
|
|
|
|
|
|
def query_abs_dirs(self):
|
|
if self.abs_dirs:
|
|
return self.abs_dirs
|
|
c = self.config
|
|
abs_dirs = super(FxDesktopBuild, self).query_abs_dirs()
|
|
if not c.get('app_ini_path'):
|
|
self.fatal('"app_ini_path" is needed in your config for this '
|
|
'script.')
|
|
|
|
dirs = {
|
|
# BuildFactories in factory.py refer to a 'build' dir on the slave.
|
|
# This contains all the source code/objdir to compile. However,
|
|
# there is already a build dir in mozharness for every mh run. The
|
|
# 'build' that factory refers to I named: 'src' so
|
|
# there is a seperation in mh. for example, rather than having
|
|
# '{mozharness_repo}/build/build/', I have '{
|
|
# mozharness_repo}/build/src/'
|
|
'abs_src_dir': os.path.join(abs_dirs['abs_work_dir'],
|
|
'src'),
|
|
'abs_obj_dir': os.path.join(abs_dirs['abs_work_dir'],
|
|
'src',
|
|
self._query_objdir()),
|
|
'abs_tools_dir': os.path.join(abs_dirs['abs_work_dir'], 'tools'),
|
|
'abs_app_ini_path': c['app_ini_path'] % {
|
|
'obj_dir': os.path.join(abs_dirs['abs_work_dir'],
|
|
'src',
|
|
self._query_objdir())
|
|
},
|
|
}
|
|
abs_dirs.update(dirs)
|
|
self.abs_dirs = abs_dirs
|
|
return self.abs_dirs
|
|
|
|
# Actions {{{2
|
|
# read_buildbot_config in BuildingMixin
|
|
# clobber in BuildingMixin -> PurgeMixin
|
|
# if Linux config:
|
|
# reset_mock in BuildingMixing -> MockMixin
|
|
# setup_mock in BuildingMixing (overrides MockMixin.mock_setup)
|
|
|
|
def set_extra_try_arguments(self, action, success=None):
|
|
""" Override unneeded method from TryToolsMixin """
|
|
pass
|
|
|
|
@script.PreScriptRun
|
|
def suppress_windows_modal_dialogs(self, *args, **kwargs):
|
|
if self._is_windows():
|
|
# Suppress Windows modal dialogs to avoid hangs
|
|
import ctypes
|
|
ctypes.windll.kernel32.SetErrorMode(0x8001)
|
|
|
|
if __name__ == '__main__':
|
|
fx_desktop_build = FxDesktopBuild()
|
|
fx_desktop_build.run_and_exit()
|