d611d1ef40
Backed out changeset 8786eabb61a4 (bug 1438688) Backed out changeset e05ec1e08b46 (bug 1438688) Backed out changeset 4c437ba9d984 (bug 1438688) Backed out changeset 2f243bca1af3 (bug 1438688) Backed out changeset 4da0e1839353 (bug 1438688) Backed out changeset 186f916dcc7a (bug 1438688) Backed out changeset 08b1a5f904e4 (bug 1438688)
343 lines
12 KiB
Python
343 lines
12 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
from mozpack.packager.formats import (
|
|
FlatFormatter,
|
|
JarFormatter,
|
|
OmniJarFormatter,
|
|
)
|
|
from mozpack.packager import (
|
|
preprocess_manifest,
|
|
preprocess,
|
|
Component,
|
|
SimpleManifestSink,
|
|
)
|
|
from mozpack.files import (
|
|
GeneratedFile,
|
|
FileFinder,
|
|
File,
|
|
)
|
|
from mozpack.copier import (
|
|
FileCopier,
|
|
Jarrer,
|
|
)
|
|
from mozpack.errors import errors
|
|
from mozpack.files import ExecutableFile
|
|
from mozpack.mozjar import JAR_BROTLI
|
|
import mozpack.path as mozpath
|
|
import buildconfig
|
|
from argparse import ArgumentParser
|
|
from createprecomplete import generate_precomplete
|
|
import os
|
|
from StringIO import StringIO
|
|
import subprocess
|
|
import mozinfo
|
|
|
|
# List of libraries to shlibsign.
|
|
SIGN_LIBS = [
|
|
'softokn3',
|
|
'nssdbm3',
|
|
'freebl3',
|
|
'freeblpriv3',
|
|
'freebl_32fpu_3',
|
|
'freebl_32int_3',
|
|
'freebl_32int64_3',
|
|
'freebl_64fpu_3',
|
|
'freebl_64int_3',
|
|
]
|
|
|
|
|
|
class ToolLauncher(object):
|
|
'''
|
|
Helper to execute tools like xpcshell with the appropriate environment.
|
|
launcher = ToolLauncher()
|
|
launcher.tooldir = '/path/to/tools'
|
|
launcher.launch(['xpcshell', '-e', 'foo.js'])
|
|
'''
|
|
def __init__(self):
|
|
self.tooldir = None
|
|
|
|
def launch(self, cmd, extra_linker_path=None, extra_env={}):
|
|
'''
|
|
Launch the given command, passed as a list. The first item in the
|
|
command list is the program name, without a path and without a suffix.
|
|
These are determined from the tooldir member and the BIN_SUFFIX value.
|
|
An extra_linker_path may be passed to give an additional directory
|
|
to add to the search paths for the dynamic linker.
|
|
An extra_env dict may be passed to give additional environment
|
|
variables to export when running the command.
|
|
'''
|
|
assert self.tooldir
|
|
cmd[0] = os.path.join(self.tooldir, 'bin',
|
|
cmd[0] + buildconfig.substs['BIN_SUFFIX'])
|
|
if not extra_linker_path:
|
|
extra_linker_path = os.path.join(self.tooldir, 'bin')
|
|
env = dict(os.environ)
|
|
for p in ['LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH']:
|
|
if p in env:
|
|
env[p] = extra_linker_path + ':' + env[p]
|
|
else:
|
|
env[p] = extra_linker_path
|
|
for e in extra_env:
|
|
env[e] = extra_env[e]
|
|
|
|
# For VC12+, make sure we can find the right bitness of pgort1x0.dll
|
|
if not buildconfig.substs.get('HAVE_64BIT_BUILD'):
|
|
for e in ('VS140COMNTOOLS', 'VS120COMNTOOLS'):
|
|
if e not in env:
|
|
continue
|
|
|
|
vcdir = os.path.abspath(os.path.join(env[e], '../../VC/bin'))
|
|
if os.path.exists(vcdir):
|
|
env['PATH'] = '%s;%s' % (vcdir, env['PATH'])
|
|
break
|
|
|
|
# Work around a bug in Python 2.7.2 and lower where unicode types in
|
|
# environment variables aren't handled by subprocess.
|
|
for k, v in env.items():
|
|
if isinstance(v, unicode):
|
|
env[k] = v.encode('utf-8')
|
|
|
|
print >>errors.out, 'Executing', ' '.join(cmd)
|
|
errors.out.flush()
|
|
return subprocess.call(cmd, env=env)
|
|
|
|
def can_launch(self):
|
|
return self.tooldir is not None
|
|
|
|
launcher = ToolLauncher()
|
|
|
|
|
|
class LibSignFile(File):
|
|
'''
|
|
File class for shlibsign signatures.
|
|
'''
|
|
def copy(self, dest, skip_if_older=True):
|
|
assert isinstance(dest, basestring)
|
|
# os.path.getmtime returns a result in seconds with precision up to the
|
|
# microsecond. But microsecond is too precise because shutil.copystat
|
|
# only copies milliseconds, and seconds is not enough precision.
|
|
if os.path.exists(dest) and skip_if_older and \
|
|
int(os.path.getmtime(self.path) * 1000) <= \
|
|
int(os.path.getmtime(dest) * 1000):
|
|
return False
|
|
if launcher.launch(['shlibsign', '-v', '-o', dest, '-i', self.path]):
|
|
errors.fatal('Error while signing %s' % self.path)
|
|
|
|
|
|
class RemovedFiles(GeneratedFile):
|
|
'''
|
|
File class for removed-files. Is used as a preprocessor parser.
|
|
'''
|
|
def __init__(self, copier):
|
|
self.copier = copier
|
|
GeneratedFile.__init__(self, '')
|
|
|
|
def handle_line(self, str):
|
|
f = str.strip()
|
|
if not f:
|
|
return
|
|
if self.copier.contains(f):
|
|
errors.error('Removal of packaged file(s): %s' % f)
|
|
self.content += f + '\n'
|
|
|
|
|
|
def split_define(define):
|
|
'''
|
|
Give a VAR[=VAL] string, returns a (VAR, VAL) tuple, where VAL defaults to
|
|
1. Numeric VALs are returned as ints.
|
|
'''
|
|
if '=' in define:
|
|
name, value = define.split('=', 1)
|
|
try:
|
|
value = int(value)
|
|
except ValueError:
|
|
pass
|
|
return (name, value)
|
|
return (define, 1)
|
|
|
|
|
|
class NoPkgFilesRemover(object):
|
|
'''
|
|
Formatter wrapper to handle NO_PKG_FILES.
|
|
'''
|
|
def __init__(self, formatter, has_manifest):
|
|
assert 'NO_PKG_FILES' in os.environ
|
|
self._formatter = formatter
|
|
self._files = os.environ['NO_PKG_FILES'].split()
|
|
if has_manifest:
|
|
self._error = errors.error
|
|
self._msg = 'NO_PKG_FILES contains file listed in manifest: %s'
|
|
else:
|
|
self._error = errors.warn
|
|
self._msg = 'Skipping %s'
|
|
|
|
def add_base(self, base, *args):
|
|
self._formatter.add_base(base, *args)
|
|
|
|
def add(self, path, content):
|
|
if not any(mozpath.match(path, spec) for spec in self._files):
|
|
self._formatter.add(path, content)
|
|
else:
|
|
self._error(self._msg % path)
|
|
|
|
def add_manifest(self, entry):
|
|
self._formatter.add_manifest(entry)
|
|
|
|
def add_interfaces(self, path, content):
|
|
self._formatter.add_interfaces(path, content)
|
|
|
|
def contains(self, path):
|
|
return self._formatter.contains(path)
|
|
|
|
|
|
def main():
|
|
parser = ArgumentParser()
|
|
parser.add_argument('-D', dest='defines', action='append',
|
|
metavar="VAR[=VAL]", help='Define a variable')
|
|
parser.add_argument('--format', default='omni',
|
|
help='Choose the chrome format for packaging ' +
|
|
'(omni, jar or flat ; default: %(default)s)')
|
|
parser.add_argument('--removals', default=None,
|
|
help='removed-files source file')
|
|
parser.add_argument('--ignore-errors', action='store_true', default=False,
|
|
help='Transform errors into warnings.')
|
|
parser.add_argument('--minify', action='store_true', default=False,
|
|
help='Make some files more compact while packaging')
|
|
parser.add_argument('--minify-js', action='store_true',
|
|
help='Minify JavaScript files while packaging.')
|
|
parser.add_argument('--js-binary',
|
|
help='Path to js binary. This is used to verify '
|
|
'minified JavaScript. If this is not defined, '
|
|
'minification verification will not be performed.')
|
|
parser.add_argument('--jarlog', default='', help='File containing jar ' +
|
|
'access logs')
|
|
parser.add_argument('--optimizejars', action='store_true', default=False,
|
|
help='Enable jar optimizations')
|
|
parser.add_argument('--compress', choices=('none', 'deflate', 'brotli'),
|
|
default='deflate',
|
|
help='Use given jar compression (default: deflate)')
|
|
parser.add_argument('manifest', default=None, nargs='?',
|
|
help='Manifest file name')
|
|
parser.add_argument('source', help='Source directory')
|
|
parser.add_argument('destination', help='Destination directory')
|
|
parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
|
|
default=[],
|
|
help='Extra files not to be considered as resources')
|
|
args = parser.parse_args()
|
|
|
|
defines = dict(buildconfig.defines['ALLDEFINES'])
|
|
if args.ignore_errors:
|
|
errors.ignore_errors()
|
|
|
|
if args.defines:
|
|
for name, value in [split_define(d) for d in args.defines]:
|
|
defines[name] = value
|
|
|
|
compress = {
|
|
'none': False,
|
|
'deflate': True,
|
|
'brotli': JAR_BROTLI,
|
|
}[args.compress]
|
|
|
|
copier = FileCopier()
|
|
if args.format == 'flat':
|
|
formatter = FlatFormatter(copier)
|
|
elif args.format == 'jar':
|
|
formatter = JarFormatter(copier, compress=compress, optimize=args.optimizejars)
|
|
elif args.format == 'omni':
|
|
formatter = OmniJarFormatter(copier,
|
|
buildconfig.substs['OMNIJAR_NAME'],
|
|
compress=compress,
|
|
optimize=args.optimizejars,
|
|
non_resources=args.non_resource)
|
|
else:
|
|
errors.fatal('Unknown format: %s' % args.format)
|
|
|
|
# Adjust defines according to the requested format.
|
|
if isinstance(formatter, OmniJarFormatter):
|
|
defines['MOZ_OMNIJAR'] = 1
|
|
elif 'MOZ_OMNIJAR' in defines:
|
|
del defines['MOZ_OMNIJAR']
|
|
|
|
respath = ''
|
|
if 'RESPATH' in defines:
|
|
respath = SimpleManifestSink.normalize_path(defines['RESPATH'])
|
|
while respath.startswith('/'):
|
|
respath = respath[1:]
|
|
|
|
if not buildconfig.substs['CROSS_COMPILE']:
|
|
launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist')
|
|
|
|
with errors.accumulate():
|
|
finder_args = dict(
|
|
minify=args.minify,
|
|
minify_js=args.minify_js,
|
|
)
|
|
if args.js_binary:
|
|
finder_args['minify_js_verify_command'] = [
|
|
args.js_binary,
|
|
os.path.join(os.path.abspath(os.path.dirname(__file__)),
|
|
'js-compare-ast.js')
|
|
]
|
|
finder = FileFinder(args.source, find_executables=True,
|
|
**finder_args)
|
|
if 'NO_PKG_FILES' in os.environ:
|
|
sinkformatter = NoPkgFilesRemover(formatter,
|
|
args.manifest is not None)
|
|
else:
|
|
sinkformatter = formatter
|
|
sink = SimpleManifestSink(finder, sinkformatter)
|
|
if args.manifest:
|
|
preprocess_manifest(sink, args.manifest, defines)
|
|
else:
|
|
sink.add(Component(''), 'bin/*')
|
|
sink.close(args.manifest is not None)
|
|
|
|
if args.removals:
|
|
removals_in = StringIO(open(args.removals).read())
|
|
removals_in.name = args.removals
|
|
removals = RemovedFiles(copier)
|
|
preprocess(removals_in, removals, defines)
|
|
copier.add(mozpath.join(respath, 'removed-files'), removals)
|
|
|
|
# shlibsign libraries
|
|
if launcher.can_launch():
|
|
if not mozinfo.isMac and buildconfig.substs.get('COMPILE_ENVIRONMENT'):
|
|
for lib in SIGN_LIBS:
|
|
libbase = mozpath.join(respath, '%s%s') \
|
|
% (buildconfig.substs['DLL_PREFIX'], lib)
|
|
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
|
|
if copier.contains(libname):
|
|
copier.add(libbase + '.chk',
|
|
LibSignFile(os.path.join(args.destination,
|
|
libname)))
|
|
|
|
# Include pdb files for llvm-symbolizer to resolve symbols.
|
|
if buildconfig.substs.get('LLVM_SYMBOLIZER') and mozinfo.isWin:
|
|
for p, f in copier:
|
|
if isinstance(f, ExecutableFile):
|
|
pdbname = os.path.splitext(f.inputs()[0])[0] + '.pdb'
|
|
if os.path.exists(pdbname):
|
|
copier.add(os.path.basename(pdbname), File(pdbname))
|
|
|
|
# Setup preloading
|
|
if args.jarlog and os.path.exists(args.jarlog):
|
|
from mozpack.mozjar import JarLog
|
|
log = JarLog(args.jarlog)
|
|
for p, f in copier:
|
|
if not isinstance(f, Jarrer):
|
|
continue
|
|
key = JarLog.canonicalize(os.path.join(args.destination, p))
|
|
if key in log:
|
|
f.preload(log[key])
|
|
|
|
copier.copy(args.destination)
|
|
generate_precomplete(os.path.normpath(os.path.join(args.destination,
|
|
respath)))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|