diff --git a/python/mozperftest/mozperftest/test/mochitest.py b/python/mozperftest/mozperftest/test/mochitest.py index 655ccb963b0f..0db82f994e9d 100644 --- a/python/mozperftest/mozperftest/test/mochitest.py +++ b/python/mozperftest/mozperftest/test/mochitest.py @@ -7,7 +7,9 @@ from contextlib import redirect_stdout from pathlib import Path from mozperftest.layers import Layer -from mozperftest.test.functionaltestrunner import FunctionalTestRunner +from mozperftest.test.functionaltestrunner import ( + FunctionalTestRunner, +) from mozperftest.utils import ( METRICS_MATCHER, ON_TRY, @@ -195,6 +197,18 @@ class Mochitest(Layer): f"--setenv=MOZ_HOST_BIN={os.environ['MOZ_HOST_BIN']}", ] ) + else: + os.environ["MOZ_HOST_BIN"] = str( + Path(os.getenv("MOZ_FETCHES_DIR"), "hostutils") + ) + mochitest_android_args.extend( + [ + f"--setenv=MOZ_HOST_BIN={os.environ['MOZ_HOST_BIN']}", + f"--remote-webserver={os.environ['HOST_IP']}", + "--http-port=8854", + "--ssl-port=4454", + ] + ) return mochitest_android_args @@ -213,6 +227,7 @@ class Mochitest(Layer): def remote_run(self, test, metadata): """Run tests in CI.""" import runtests + import runtestsremote from manifestparser import TestManifest from mochitest_options import MochitestArgumentParser @@ -247,18 +262,29 @@ class Mochitest(Layer): args.topobjdir = self.topobjdir args.topsrcdir = self.topsrcdir args.flavor = manifest_flavor - args.app = self.get_arg("mochitest_binary") fetch_dir = os.getenv("MOZ_FETCHES_DIR") - args.utilityPath = str(Path(fetch_dir, "bin")) - args.extraProfileFiles.append(str(Path(fetch_dir, "bin", "plugins"))) - args.testingModulesDir = str(Path(fetch_dir, "modules")) - args.symbolsPath = str(Path(fetch_dir, "crashreporter-symbols")) - args.certPath = str(Path(fetch_dir, "certs")) + if self.get_arg("android"): + args.utilityPath = str(Path(fetch_dir, "hostutils")) + args.xrePath = str(Path(fetch_dir, "hostutils")) + args.extraProfileFiles.append(str(Path(fetch_dir, "bin", "plugins"))) + args.testingModulesDir = str(Path(fetch_dir, "modules")) + args.symbolsPath = str(Path(fetch_dir, "crashreporter-symbols")) + args.certPath = str(Path(fetch_dir, "certs")) + else: + args.app = self.get_arg("mochitest_binary") + args.utilityPath = str(Path(fetch_dir, "bin")) + args.extraProfileFiles.append(str(Path(fetch_dir, "bin", "plugins"))) + args.testingModulesDir = str(Path(fetch_dir, "modules")) + args.symbolsPath = str(Path(fetch_dir, "crashreporter-symbols")) + args.certPath = str(Path(fetch_dir, "certs")) log_processor = LogProcessor(METRICS_MATCHER) with redirect_stdout(log_processor): - result = runtests.run_test_harness(parser, args) + if self.get_arg("android"): + result = runtestsremote.run_test_harness(parser, args) + else: + result = runtests.run_test_harness(parser, args) return result, log_processor diff --git a/python/mozperftest/mozperftest/tests/test_mochitest.py b/python/mozperftest/mozperftest/tests/test_mochitest.py index 88132922c5c3..24f56cbf1d4b 100644 --- a/python/mozperftest/mozperftest/tests/test_mochitest.py +++ b/python/mozperftest/mozperftest/tests/test_mochitest.py @@ -307,6 +307,75 @@ def test_mochitest_ci_metrics(formatter_mock, run_test_harness_mock): assert results[0]["values"] == [0] +@mock.patch("mozperftest.system.VersionProducer.run", new=fake_version_producer) +@mock.patch("mozperftest.system.android.ADBLoggedDevice", new=FakeDevice) +@mock.patch( + # This mock.patch actually patches the mochitest run_test_harness function + "runtestsremote.run_test_harness" +) +@mock.patch( + # This mock.patch causes mochitest's runtests to be imported instead of + # others in the remote_run + "mochitest.runtestsremote.run_test_harness", + new=mock.MagicMock(), +) +@mock.patch( + "mozperftest.test.functionaltestrunner.mozlog.formatters.MachFormatter.__new__" +) +@mock.patch( + "mozperftest.test.mochitest.install_requirements_file", new=mock.MagicMock() +) +@mock.patch( + "mozperftest.test.functionaltestrunner.load_class_from_path", new=mock.MagicMock() +) +@mock.patch("moztest.resolve.TestResolver", new=mock.MagicMock()) +@mock.patch("mozperftest.test.mochitest.ON_TRY", new=True) +@mock.patch("mozperftest.utils.ON_TRY", new=True) +@mock.patch("mochitest.mochitest_options.MochitestArgumentParser", new=mock.MagicMock()) +@mock.patch("manifestparser.TestManifest", new=mock.MagicMock()) +def test_mochitest_android_ci_metrics(formatter_mock, run_test_harness_mock): + if not os.getenv("MOZ_FETCHES_DIR"): + os.environ["MOZ_FETCHES_DIR"] = "fake-path" + if not os.getenv("HOST_IP"): + os.environ["HOST_IP"] = "fake-ip" + + mach_cmd, metadata, env = running_env( + tests=[str(EXAMPLE_MOCHITEST_TEST)], + mochitest_extra_args=[], + mochitest_manifest="fake.ini", + mochitest_manifest_flavor="mocha", + android=True, + app="geckoview", + android_install_apk=["this.apk"], + android_capture_adb="stdout", + android_activity="GeckoViewActivity", + ) + + system = env.layers[SYSTEM] + mochitest = env.layers[TEST] + + formatter_mock.return_value = lambda x: x + + def test_print(*args, **kwargs): + print('perfMetrics | { "fake": 0 }') + return 0 + + run_test_harness_mock.side_effect = test_print + try: + with system as s, mochitest as m: + m(s(metadata)) + finally: + shutil.rmtree(mach_cmd._mach_context.state_dir) + + res = metadata.get_results() + assert len(res) == 1 + assert res[0]["name"] == "test_mochitest.html" + results = res[0]["results"] + + assert results[0]["name"] == "fake" + assert results[0]["values"] == [0] + + @mock.patch( # This mock.patch actually patches the mochitest run_test_harness function "runtests.run_test_harness" diff --git a/python/mozperftest/mozperftest/utils.py b/python/mozperftest/mozperftest/utils.py index 7d40f61800e8..a618c1708fdf 100644 --- a/python/mozperftest/mozperftest/utils.py +++ b/python/mozperftest/mozperftest/utils.py @@ -30,7 +30,7 @@ MULTI_REVISION_ROOT = f"{API_ROOT}/namespaces" MULTI_TASK_ROOT = f"{API_ROOT}/tasks" ON_TRY = "MOZ_AUTOMATION" in os.environ DOWNLOAD_TIMEOUT = 30 -METRICS_MATCHER = re.compile(r"(perfMetrics\s.*)") +METRICS_MATCHER = re.compile(r"(perfMetrics.*)") PRETTY_APP_NAMES = { "org.mozilla.fenix": "fenix", "org.mozilla.firefox": "fenix", @@ -87,10 +87,8 @@ class LogProcessor: continue self.stdout.write(data.strip("\n") + "\n") - # Check if a temporary commit wa created - match = self.matcher.match(data) + match = self.matcher.search(data) if match: - # Last line found is the revision we want self._match.append(match.group(1)) def flush(self): diff --git a/python/mozperftest/perfdocs/writing.rst b/python/mozperftest/perfdocs/writing.rst index 6e4be9115983..6abeeef29808 100644 --- a/python/mozperftest/perfdocs/writing.rst +++ b/python/mozperftest/perfdocs/writing.rst @@ -124,6 +124,23 @@ To run your test in CI, you may need to modify the ``_TRY_MAPPING`` variable `fo The Mochitest test that is written can also be run as a unit test, however, if this is not desired, set the `disabled = reason` flag in the test TOML file to prevent it from running there. `See here for an example `_. +Mochitest Android Tests in CI +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For Mochitest Android tests in CI, everything that applies to desktop tests also applies here. When writing a new task in the ``android.yml``, ensure that there are the following fetches applied to the task:: + + build: + - artifact: geckoview_example.apk + extract: false + - artifact: en-US/target.perftests.tests.tar.gz + - artifact: en-US/target.condprof.tests.tar.gz + - artifact: en-US/target.common.tests.tar.gz + - artifact: en-US/target.mochitest.tests.tar.gz + toolchain: + - linux64-hostutils + +Ensure that the ``runner.py`` script is also running from ``MOZ_FETCHES_DIR`` instead of the ``GECKO_PATH`` like other android MozPerftest tests. Everything else is the same as other android mozperftest tests. Note that ``--android-install-apk`` needs to be specified to point to the ``geckoview_example.apk`` that was obtained from the build task. Fenix is not currently supported in CI for Mochitest (see `bug 1902535 `_). + Custom Script ------------- diff --git a/testing/perfdocs/generated/writing.rst b/testing/perfdocs/generated/writing.rst index 6e4be9115983..6abeeef29808 100644 --- a/testing/perfdocs/generated/writing.rst +++ b/testing/perfdocs/generated/writing.rst @@ -124,6 +124,23 @@ To run your test in CI, you may need to modify the ``_TRY_MAPPING`` variable `fo The Mochitest test that is written can also be run as a unit test, however, if this is not desired, set the `disabled = reason` flag in the test TOML file to prevent it from running there. `See here for an example `_. +Mochitest Android Tests in CI +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For Mochitest Android tests in CI, everything that applies to desktop tests also applies here. When writing a new task in the ``android.yml``, ensure that there are the following fetches applied to the task:: + + build: + - artifact: geckoview_example.apk + extract: false + - artifact: en-US/target.perftests.tests.tar.gz + - artifact: en-US/target.condprof.tests.tar.gz + - artifact: en-US/target.common.tests.tar.gz + - artifact: en-US/target.mochitest.tests.tar.gz + toolchain: + - linux64-hostutils + +Ensure that the ``runner.py`` script is also running from ``MOZ_FETCHES_DIR`` instead of the ``GECKO_PATH`` like other android MozPerftest tests. Everything else is the same as other android mozperftest tests. Note that ``--android-install-apk`` needs to be specified to point to the ``geckoview_example.apk`` that was obtained from the build task. Fenix is not currently supported in CI for Mochitest (see `bug 1902535 `_). + Custom Script -------------