Bug 1425406 - Add a linux64 clang 6 (pre) toolchain with the macosx64 native sanitizer dylibs. r=froydnj

MozReview-Commit-ID: Ig9xpBDcjNu

build/build-clang/build-clang.py

@@ -213,9 +213,10 @@ def build_one_stage(cc, cxx, asm, ld, ar, ranlib, libtool,
                        "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER",
                        "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY",
                        "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY",
-                       "-DCMAKE_MACOSX_RPATH=@executable_path",
+                       "-DCMAKE_MACOSX_RPATH=ON",
                        "-DCMAKE_OSX_ARCHITECTURES=x86_64",
                        "-DDARWIN_osx_ARCHS=x86_64",
+                       "-DDARWIN_osx_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
                        "-DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-apple-darwin11"]
     build_package(build_dir, cmake_args)
 
@@ -371,6 +372,9 @@ if __name__ == "__main__":
     parser.add_argument('--clean', required=False,
                         action='store_true',
                         help="Clean the build directory")
+    parser.add_argument('--skip-tar', required=False,
+                        action='store_true',
+                        help="Skip tar packaging stage")
 
     args = parser.parse_args()
     config = json.load(args.config)
@@ -595,5 +599,6 @@ if __name__ == "__main__":
         prune_final_dir_for_clang_tidy(os.path.join(final_stage_dir, "clang"))
         package_name = "clang-tidy"
 
+    if not args.skip_tar:
         ext = "bz2" if is_darwin() or is_windows() else "xz"
         build_tar_package("tar", "%s.tar.%s" % (package_name, ext), final_stage_dir, "clang")

build/build-clang/clang-6-pre-macosx64.json

@@ -0,0 +1,27 @@
+{
+    "llvm_revision": "317840",
+    "stages": "1",
+    "build_libcxx": true,
+    "build_type": "Release",
+    "assertions": false,
+    "osx_cross_compile": true,
+    "llvm_repo": "https://llvm.org/svn/llvm-project/llvm/trunk",
+    "clang_repo": "https://llvm.org/svn/llvm-project/cfe/trunk",
+    "compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/trunk",
+    "libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/trunk",
+    "libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/trunk",
+    "python_path": "/usr/bin/python2.7",
+    "gcc_dir": "/builds/worker/workspace/build/src/gcc",
+    "cc": "/builds/worker/workspace/build/src/clang/bin/clang",
+    "cxx": "/builds/worker/workspace/build/src/clang/bin/clang++",
+    "as": "/builds/worker/workspace/build/src/clang/bin/clang",
+    "ar": "/builds/worker/workspace/build/src/cctools/bin/x86_64-apple-darwin11-ar",
+    "ranlib": "/builds/worker/workspace/build/src/cctools/bin/x86_64-apple-darwin11-ranlib",
+    "libtool": "/builds/worker/workspace/build/src/cctools/bin/x86_64-apple-darwin11-libtool",
+    "ld": "/builds/worker/workspace/build/src/clang/bin/clang",
+    "patches": [
+      "compiler-rt-cross-compile.patch",
+      "compiler-rt-no-codesign.patch",
+      "r321543.patch"
+    ]
+}

build/build-clang/compiler-rt-no-codesign.patch

@@ -1,3 +1,5 @@
+Disable codesign for macosx cross-compile toolchain. Codesign only works on OSX.
+
 Index: cmake/Modules/AddCompilerRT.cmake
 ===================================================================
 --- a/compiler-rt/cmake/Modules/AddCompilerRT.cmake	(revision 312553)

build/build-clang/r321543.patch

@@ -0,0 +1,33 @@
+From: Yi Kong <yikong@google.com>
+Date: Thu, 28 Dec 2017 23:06:24 +0000
+Subject: [PATCH] Ignore the DISPATCH_NOESCAPE if not defined
+
+This macro is only defined after XCode 8, causing build breakage for
+build systems with prior versions. Ignore DISPATCH_NOESCAPE if not
+defined.
+
+Differential Revision: https://reviews.llvm.org/D41601
+
+
+git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@321543 91177308-0d34-0410-b5e6-96231b3b80d8
+---
+ lib/tsan/rtl/tsan_libdispatch_mac.cc | 5 +++++
+ 1 file changed, 5 insertions(+)
+
+diff --git a/lib/tsan/rtl/tsan_libdispatch_mac.cc b/lib/tsan/rtl/tsan_libdispatch_mac.cc
+index eb22e4baa..d6c1ca662 100644
+--- a/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc
++++ b/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc
+@@ -25,6 +25,11 @@
+ #include <dispatch/dispatch.h>
+ #include <pthread.h>
+ 
++// DISPATCH_NOESCAPE is not defined prior to XCode 8.
++#ifndef DISPATCH_NOESCAPE
++#define DISPATCH_NOESCAPE
++#endif
++
+ typedef long long_t;  // NOLINT
+ 
+ namespace __tsan {
+

taskcluster/ci/toolchain/linux.yml

@@ -91,6 +91,32 @@ linux64-clang-6-pre:
     toolchains:
         - linux64-gcc-4.9
 
+linux64-clang-6-pre-macosx-cross:
+    description: "Clang 6 Pre toolchain build with MacOS Compiler RT libs"
+    treeherder:
+        kind: build
+        platform: toolchains/opt
+        symbol: TL(clang6p-macosx-cross)
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        max-run-time: 36000
+        env:
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
+    run:
+        using: toolchain-script
+        script: build-clang-6-pre-linux-macosx-cross.sh
+        resources:
+            - 'build/build-clang/build-clang.py'
+            - 'build/build-clang/clang-6-pre-macosx64.json'
+            - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/build/clang.tar.xz
+        tooltool-downloads: internal
+    toolchains:
+        - linux64-cctools-port
+        - linux64-clang-6-pre
+        - linux64-gcc-4.9
+
 linux64-clang-tidy:
     description: "Clang-tidy build"
     index:

taskcluster/scripts/misc/build-clang-6-pre-linux-macosx-cross.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+set -x -e -v
+
+# This script is for building clang for Mac OS X targets on a Linux host,
+# including native Mac OS X Compiler-RT libraries and llvm-symbolizer.
+WORKSPACE=$HOME/workspace
+HOME_DIR=$WORKSPACE/build
+UPLOAD_DIR=$HOME/artifacts
+
+cd $HOME_DIR/src
+
+. taskcluster/scripts/misc/tooltool-download.sh
+
+# ld needs libLTO.so from llvm
+export LD_LIBRARY_PATH=$HOME_DIR/src/clang/lib
+# these variables are used in build-clang.py
+export CROSS_CCTOOLS_PATH=$HOME_DIR/src/cctools
+export CROSS_SYSROOT=$HOME_DIR/src/MacOSX10.11.sdk
+# cmake doesn't allow us to specify a path to lipo on the command line.
+export PATH=$PATH:$CROSS_CCTOOLS_PATH/bin
+ln -sf $CROSS_CCTOOLS_PATH/bin/x86_64-apple-darwin11-lipo $CROSS_CCTOOLS_PATH/bin/lipo
+
+# gets a bit too verbose here
+set +x
+
+cd build/build-clang
+# |mach python| sets up a virtualenv for us!
+../../mach python ./build-clang.py -c clang-6-pre-macosx64.json --skip-tar
+
+# We now have a native macosx64 toolchain.
+# What we want is a native linux64 toolchain which can target macosx64 and use the sanitizer dylibs.
+# Overlay the linux64 toolchain that we used for this build (except llvm-symbolizer).
+(
+cd "$WORKSPACE/moz-toolchain/build/stage1"
+# Need the macosx64 native llvm-symbolizer since this gets shipped with sanitizer builds
+mv clang/bin/llvm-symbolizer $HOME_DIR/src/clang/bin/
+cp --remove-destination -lr $HOME_DIR/src/clang/* clang/
+tar -c -J -f $HOME_DIR/src/build/build-clang/clang.tar.xz clang
+)
+
+set -x
+
+# Put a tarball in the artifacts dir
+mkdir -p $UPLOAD_DIR
+cp clang.tar.* $UPLOAD_DIR