Bug 1188330 - Use intree mozharness for taskcluster tests. r=dustin,garndt,wcosta

2015-09-04 17:01:36 +08:00
parent c07da59a78
commit cd1347d243
18 changed files with 43 additions and 48 deletions
--- a/testing/docker/tester/VERSION
+++ b/testing/docker/tester/VERSION
@@ -1 +1 @@
-0.3.8
+0.3.9
--- a/testing/docker/tester/bin/entrypoint
+++ b/testing/docker/tester/bin/entrypoint
@@ -1,8 +1,5 @@
 #! /bin/bash -ex

-test $MOZHARNESS_REPOSITORY # mozharness repository
-test $MOZHARNESS_REV # mozharness revision
-
 # This file is intended to be used as the "entrypoint" or first command in the
 # docker image. It uses "eval" so it takes place of the usual /bin/bash -c
 # <stuff> conventions you need when running a docker image. Primarily it
@@ -14,13 +11,11 @@ export DISPLAY=:0

 pulseaudio --start

-# First check if the mozharness directory is available. This is intended to be
-# used locally in development to test mozharness changes:
-#
-#   $ docker -v your_mozharness:/home/worker/mozharness ...
-#
-if [ ! -d mozharness ]; then
-  buildbot_step 'Checkout mozharness' tc-vcs checkout mozharness $MOZHARNESS_REPOSITORY $MOZHARNESS_REPOSITORY $MOZHARNESS_REV
+# Download mozharness only when $MOZHARNESS_URL is defined
+if [ ! -z "$MOZHARNESS_URL" ]; then
+  buildbot_step 'Download mozharness' wget -c $MOZHARNESS_URL && \
+    unzip -q mozharness.zip && \
+    rm mozharness.zip
 fi

 buildbot_step 'Running tests' $@
--- a/testing/docker/tester/tester.env
+++ b/testing/docker/tester/tester.env
@@ -1,5 +1,3 @@
-MOZHARNESS_REPOSITORY=http://hg.mozilla.org/build/mozharness
-MOZHARNESS_REV=tip
 GAIA_REV=tip
 GAIA_REF=tip
 GAIA_BASE_REPOSITORY=https://hg.mozilla.org/integration/gaia-central
--- a/testing/mozharness/configs/b2g/mulet_config.py
+++ b/testing/mozharness/configs/b2g/mulet_config.py
@@ -10,16 +10,20 @@ config = {
        'run-tests',
    ],
    # testsuite options
-    "reftest_options": [
-        "--mulet",
-        "--profile=%(gaia_profile)s",
-        "--appname=%(application)s",
-        "--total-chunks=%(total_chunks)s",
-        "--this-chunk=%(this_chunk)s",
-        "--symbols-path=%(symbols_path)s",
-        "--enable-oop",
-        "%(test_manifest)s"
-    ],
+    "suite_definitions": {
+        "reftest": {
+            "options": [
+                "--mulet",
+                "--profile=%(gaia_profile)s",
+                "--appname=%(application)s",
+                "--total-chunks=%(total_chunks)s",
+                "--this-chunk=%(this_chunk)s",
+                "--symbols-path=%(symbols_path)s",
+                "--enable-oop",
+                "%(test_manifest)s"
+            ],
+        }
+    },
    "run_file_names": {
        "reftest": "runreftestb2g.py",
    },
--- a/testing/mozharness/mozharness.json
+++ b/testing/mozharness/mozharness.json
@@ -1,4 +0,0 @@
-{
-    "repo": "https://hg.mozilla.org/build/mozharness",
-    "revision": "e218c0e82735"
-}
--- a/testing/mozharness/mozharness/mozilla/testing/gaia_test.py
+++ b/testing/mozharness/mozharness/mozilla/testing/gaia_test.py
@@ -25,7 +25,7 @@ from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_opt
 from mozharness.mozilla.proxxy import Proxxy


-class GaiaTest(TestingMixin, MercurialScript, TransferMixin, GaiaMixin, BlobUploadMixin):
+class GaiaTest(MercurialScript, TestingMixin, BlobUploadMixin, GaiaMixin, TransferMixin):
    config_options = [[
        ["--application"],
        {"action": "store",
--- a/testing/taskcluster/mach_commands.py
+++ b/testing/taskcluster/mach_commands.py
@@ -21,7 +21,6 @@ from mach.decorators import (
 ROOT = os.path.dirname(os.path.realpath(__file__))
 GECKO = os.path.realpath(os.path.join(ROOT, '..', '..'))
 DOCKER_ROOT = os.path.join(ROOT, '..', 'docker')
-MOZHARNESS_CONFIG = os.path.join(GECKO, 'testing', 'mozharness', 'mozharness.json')

 # XXX: If/when we have the taskcluster queue use construct url instead
 ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
@@ -40,10 +39,6 @@ DEFAULT_JOB_PATH = os.path.join(
    ROOT, 'tasks', 'branches', 'base_jobs.yml'
 )

-def load_mozharness_info():
-    with open(MOZHARNESS_CONFIG) as content:
-        return json.load(content)
-
 def docker_image(name):
    ''' Determine the docker tag/revision from an in tree docker file '''
    repository_path = os.path.join(DOCKER_ROOT, name, 'REGISTRY')
@@ -322,9 +317,6 @@ class Graph(object):

        job_graph = parse_commit(message, jobs)

-        # once everything uses in-tree mozharness (bug 1187706), this can go away.
-        mozharness = load_mozharness_info()
-
        cmdline_interactive = params.get('interactive', False)

        # Template parameters used when expanding the graph
@@ -341,9 +333,6 @@ class Graph(object):
            'owner': params['owner'],
            'from_now': json_time_from_now,
            'now': current_json_time(),
-            'mozharness_repository': mozharness['repo'],
-            'mozharness_rev': mozharness['revision'],
-            'mozharness_ref':mozharness.get('reference', mozharness['revision']),
            'revision_hash': params['revision_hash']
        }.items())

@@ -399,7 +388,7 @@ class Graph(object):
            taskcluster_graph.build_task.validate(build_task)
            graph['tasks'].append(build_task)

-            test_packages_url, tests_url = None, None
+            test_packages_url, tests_url, mozharness_url = None, None, None

            if 'test_packages' in build_task['task']['extra']['locations']:
                test_packages_url = ARTIFACT_URL.format(
@@ -413,6 +402,12 @@ class Graph(object):
                    build_task['task']['extra']['locations']['tests']
                )

+            if 'mozharness' in build_task['task']['extra']['locations']:
+                mozharness_url = ARTIFACT_URL.format(
+                    build_parameters['build_slugid'],
+                    build_task['task']['extra']['locations']['mozharness']
+                )
+
            build_url = ARTIFACT_URL.format(
                build_parameters['build_slugid'],
                build_task['task']['extra']['locations']['build']
@@ -470,6 +465,8 @@ class Graph(object):
                    test_parameters['tests_url'] = tests_url
                if test_packages_url:
                    test_parameters['test_packages_url'] = test_packages_url
+                if mozharness_url:
+                    test_parameters['mozharness_url'] = mozharness_url
                test_definition = templates.load(test['task'], {})['task']
                chunk_config = test_definition['extra']['chunks']

--- a/testing/taskcluster/scripts/builder/build-b2g-desktop.sh
+++ b/testing/taskcluster/scripts/builder/build-b2g-desktop.sh
@@ -41,6 +41,7 @@ for name in common cppunittest reftest mochitest xpcshell web-platform; do
 done
 mv test_packages_tc.json        $HOME/artifacts/test_packages.json
 mv *.crashreporter-symbols.zip  $HOME/artifacts/target.crashreporter-symbols.zip
+mv mozharness.zip               $HOME/artifacts/mozharness.zip

 # If the simulator does not exist don't fail
 mv fxos-simulator*              $HOME/artifacts/fxos-simulator.xpi || :
--- a/testing/taskcluster/scripts/builder/build-emulator-x86.sh
+++ b/testing/taskcluster/scripts/builder/build-emulator-x86.sh
@@ -63,5 +63,6 @@ done
 mv $gecko_objdir/dist/test_packages_tc.json $HOME/artifacts/test_packages.json
 mv $WORKSPACE/B2G/out/emulator.tar.gz $HOME/artifacts/emulator.tar.gz
 mv $gecko_objdir/dist/b2g-*.crashreporter-symbols.zip $HOME/artifacts/b2g-crashreporter-symbols.zip
+mv $gecko_objdir/dist/mozharness.zip $HOME/artifacts/mozharness.zip

 ccache -s
--- a/testing/taskcluster/scripts/builder/build-emulator.sh
+++ b/testing/taskcluster/scripts/builder/build-emulator.sh
@@ -65,5 +65,6 @@ done
 mv $gecko_objdir/dist/test_packages_tc.json $HOME/artifacts/test_packages.json
 mv $WORKSPACE/B2G/out/emulator.tar.gz $HOME/artifacts/emulator.tar.gz
 mv $gecko_objdir/dist/b2g-*.crashreporter-symbols.zip $HOME/artifacts/b2g-crashreporter-symbols.zip
+mv $gecko_objdir/dist/mozharness.zip $HOME/artifacts/mozharness.zip

 ccache -s
--- a/testing/taskcluster/scripts/builder/build-mulet-linux.sh
+++ b/testing/taskcluster/scripts/builder/build-mulet-linux.sh
@@ -36,6 +36,7 @@ done
 mv test_packages_tc.json        $HOME/artifacts/test_packages.json
 mv *.crashreporter-symbols.zip  $HOME/artifacts/target.crashreporter-symbols.zip
 mv jsshell-linux-x86_64.zip     $HOME/artifacts/jsshell-linux-x86_64.zip
+mv mozharness.zip               $HOME/artifacts/mozharness.zip

 ccache -s

--- a/testing/taskcluster/tasks/builds/b2g_base.yml
+++ b/testing/taskcluster/tasks/builds/b2g_base.yml
@@ -2,3 +2,8 @@ $inherits:
  from: 'tasks/build.yml'
  variables:
    build_product: 'b2g'
+task:
+  extra:
+    locations:
+      test_packages: 'public/build/test_packages.json'
+      mozharness: 'public/build/mozharness.zip'
--- a/testing/taskcluster/tasks/builds/b2g_desktop_base.yml
+++ b/testing/taskcluster/tasks/builds/b2g_desktop_base.yml
@@ -34,4 +34,3 @@ task:
    # files are located.
    locations:
      build: 'public/build/target.linux-x86_64.tar.bz2'
-      test_packages: 'public/build/test_packages.json'
--- a/testing/taskcluster/tasks/builds/b2g_emulator_base.yml
+++ b/testing/taskcluster/tasks/builds/b2g_emulator_base.yml
@@ -33,6 +33,5 @@ task:
    # files are located.
    locations:
      build: 'public/build/emulator.tar.gz'
-      test_packages: 'public/build/test_packages.json'
      symbols: 'public/build/b2g-crashreporter-symbols.zip'
      sources: 'public/build/sources.xml'
--- a/testing/taskcluster/tasks/builds/b2g_emulator_x86_base.yml
+++ b/testing/taskcluster/tasks/builds/b2g_emulator_x86_base.yml
@@ -26,7 +26,6 @@ task:
    # files are located.
    locations:
      build: 'public/build/emulator.tar.gz'
-      test_packages: 'public/build/test_packages.json'
      tests: 'public/build/b2g-tests.zip'
      symbols: 'public/build/b2g-crashreporter-symbols.zip'
      sources: 'public/build/sources.xml'
--- a/testing/taskcluster/tasks/phone_test.yml
+++ b/testing/taskcluster/tasks/phone_test.yml
@@ -27,8 +27,6 @@ task:
      GAIA_BASE_REPOSITORY: '{{{gaia_base_repository}}}'
      GAIA_REF: '{{{gaia_ref}}}'
      GAIA_REV: '{{{gaia_rev}}}'
-      MOZHARNESS_REPOSITORY: '{{mozharness_repository}}'
-      MOZHARNESS_REV: '{{mozharness_rev}}'

    # All builds share a common artifact directory for ease of uploading.
    artifacts:
--- a/testing/taskcluster/tasks/test.yml
+++ b/testing/taskcluster/tasks/test.yml
@@ -36,9 +36,8 @@ task:
      GAIA_BASE_REPOSITORY: '{{{gaia_base_repository}}}'
      GAIA_REF: '{{{gaia_ref}}}'
      GAIA_REV: '{{{gaia_rev}}}'
-      MOZHARNESS_REPOSITORY: '{{mozharness_repository}}'
-      MOZHARNESS_REV: '{{mozharness_rev}}'
      MOZILLA_BUILD_URL: '{{build_url}}'
+      MOZHARNESS_URL: '{{mozharness_url}}'

    # All builds share a common artifact directory for ease of uploading.
    artifacts:
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -430,7 +430,9 @@ package-tests:
 	$(NSINSTALL) -D $(DIST)/$(PKG_PATH)
 # Exclude harness specific directories when generating the common zip.
 	$(MKDIR) -p $(abspath $(DIST))/$(PKG_PATH) && \
-	cd $(PKG_STAGE) && \
+	cd $(topsrcdir)/testing/ && \
+	  zip -rq9D $(abspath $(DIST))/$(PKG_PATH)mozharness.zip mozharness && \
+	cd $(abspath $(PKG_STAGE)) && \
 	  zip -rq9D '$(abspath $(DIST))/$(PKG_PATH)$(TEST_PACKAGE)' \
 	  * -x \*/.mkdir.done \*.pyc $(foreach name,$(TEST_PKGS),$(name)\*) && \
 	$(foreach name,$(TEST_PKGS),rm -f '$(DIST)/$(PKG_PATH)$(PKG_BASENAME).'$(name)'.tests.zip' && \
@@ -1 +1 @@
 .3.8
 .3.9