corysanin/tubestation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug 1944559 - Enable mochitest android tests in CI for mozperftest. r=perftest-reviewers,kshampur

Browse Source 
This patch adds some changes needed to run mochitest android tests in CI for mozperftest. The main change is that we use `runtestsremote.run_test_harness` instead of `runtests.run_test_harness` for running android tests.

Additionally, the `LogProcessor` is modified to use the `re.search` method for finding the `perfMetrics` output and remove the requirement for there to be a space after `perfMetrics`. This is done because the output received from mochitest android is always prefixed with something which broke `re.match`. The removal of the space requirement was done to remove a common pitfall that was seen with others writing mozperftest mochitest tests.

Differential Revision: https://phabricator.services.mozilla.com/D245766
This commit is contained in:
Greg Mierzwinski
2025-05-07 11:48:35 +00:00
committed by gmierz2@outlook.com
parent 0d6b825223
commit 75364ccec8
5 changed files with 139 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
python/mozperftest/mozperftest/test/mochitest.py
View File

@@ -7,7 +7,9 @@ from contextlib import redirect_stdout
from pathlib import Path
from mozperftest.layers import Layer
from mozperftest.test.functionaltestrunner import FunctionalTestRunner
from mozperftest.test.functionaltestrunner import (
FunctionalTestRunner,
)
from mozperftest.utils import (
METRICS_MATCHER,
ON_TRY,
@@ -195,6 +197,18 @@ class Mochitest(Layer):
f"--setenv=MOZ_HOST_BIN={os.environ['MOZ_HOST_BIN']}",
]
)
else:
os.environ["MOZ_HOST_BIN"] = str(
Path(os.getenv("MOZ_FETCHES_DIR"), "hostutils")
)
mochitest_android_args.extend(
[
f"--setenv=MOZ_HOST_BIN={os.environ['MOZ_HOST_BIN']}",
f"--remote-webserver={os.environ['HOST_IP']}",
"--http-port=8854",
"--ssl-port=4454",
]
)
return mochitest_android_args
@@ -213,6 +227,7 @@ class Mochitest(Layer):
def remote_run(self, test, metadata):
"""Run tests in CI."""
import runtests
import runtestsremote
from manifestparser import TestManifest
from mochitest_options import MochitestArgumentParser
@@ -247,18 +262,29 @@ class Mochitest(Layer):
args.topobjdir = self.topobjdir
args.topsrcdir = self.topsrcdir
args.flavor = manifest_flavor
args.app = self.get_arg("mochitest_binary")
fetch_dir = os.getenv("MOZ_FETCHES_DIR")
args.utilityPath = str(Path(fetch_dir, "bin"))
args.extraProfileFiles.append(str(Path(fetch_dir, "bin", "plugins")))
args.testingModulesDir = str(Path(fetch_dir, "modules"))
args.symbolsPath = str(Path(fetch_dir, "crashreporter-symbols"))
args.certPath = str(Path(fetch_dir, "certs"))
if self.get_arg("android"):
args.utilityPath = str(Path(fetch_dir, "hostutils"))
args.xrePath = str(Path(fetch_dir, "hostutils"))
args.extraProfileFiles.append(str(Path(fetch_dir, "bin", "plugins")))
args.testingModulesDir = str(Path(fetch_dir, "modules"))
args.symbolsPath = str(Path(fetch_dir, "crashreporter-symbols"))
args.certPath = str(Path(fetch_dir, "certs"))
else:
args.app = self.get_arg("mochitest_binary")
args.utilityPath = str(Path(fetch_dir, "bin"))
args.extraProfileFiles.append(str(Path(fetch_dir, "bin", "plugins")))
args.testingModulesDir = str(Path(fetch_dir, "modules"))
args.symbolsPath = str(Path(fetch_dir, "crashreporter-symbols"))
args.certPath = str(Path(fetch_dir, "certs"))
log_processor = LogProcessor(METRICS_MATCHER)
with redirect_stdout(log_processor):
result = runtests.run_test_harness(parser, args)
if self.get_arg("android"):
result = runtestsremote.run_test_harness(parser, args)
else:
result = runtests.run_test_harness(parser, args)
return result, log_processor

69
python/mozperftest/mozperftest/tests/test_mochitest.py
View File

@@ -307,6 +307,75 @@ def test_mochitest_ci_metrics(formatter_mock, run_test_harness_mock):
assert results[0]["values"] == [0]
@mock.patch("mozperftest.system.VersionProducer.run", new=fake_version_producer)
@mock.patch("mozperftest.system.android.ADBLoggedDevice", new=FakeDevice)
@mock.patch(
# This mock.patch actually patches the mochitest run_test_harness function
"runtestsremote.run_test_harness"
)
@mock.patch(
# This mock.patch causes mochitest's runtests to be imported instead of
# others in the remote_run
"mochitest.runtestsremote.run_test_harness",
new=mock.MagicMock(),
)
@mock.patch(
"mozperftest.test.functionaltestrunner.mozlog.formatters.MachFormatter.__new__"
)
@mock.patch(
"mozperftest.test.mochitest.install_requirements_file", new=mock.MagicMock()
)
@mock.patch(
"mozperftest.test.functionaltestrunner.load_class_from_path", new=mock.MagicMock()
)
@mock.patch("moztest.resolve.TestResolver", new=mock.MagicMock())
@mock.patch("mozperftest.test.mochitest.ON_TRY", new=True)
@mock.patch("mozperftest.utils.ON_TRY", new=True)
@mock.patch("mochitest.mochitest_options.MochitestArgumentParser", new=mock.MagicMock())
@mock.patch("manifestparser.TestManifest", new=mock.MagicMock())
def test_mochitest_android_ci_metrics(formatter_mock, run_test_harness_mock):
if not os.getenv("MOZ_FETCHES_DIR"):
os.environ["MOZ_FETCHES_DIR"] = "fake-path"
if not os.getenv("HOST_IP"):
os.environ["HOST_IP"] = "fake-ip"
mach_cmd, metadata, env = running_env(
tests=[str(EXAMPLE_MOCHITEST_TEST)],
mochitest_extra_args=[],
mochitest_manifest="fake.ini",
mochitest_manifest_flavor="mocha",
android=True,
app="geckoview",
android_install_apk=["this.apk"],
android_capture_adb="stdout",
android_activity="GeckoViewActivity",
)
system = env.layers[SYSTEM]
mochitest = env.layers[TEST]
formatter_mock.return_value = lambda x: x
def test_print(*args, **kwargs):
print('perfMetrics | { "fake": 0 }')
return 0
run_test_harness_mock.side_effect = test_print
try:
with system as s, mochitest as m:
m(s(metadata))
finally:
shutil.rmtree(mach_cmd._mach_context.state_dir)
res = metadata.get_results()
assert len(res) == 1
assert res[0]["name"] == "test_mochitest.html"
results = res[0]["results"]
assert results[0]["name"] == "fake"
assert results[0]["values"] == [0]
@mock.patch(
# This mock.patch actually patches the mochitest run_test_harness function
"runtests.run_test_harness"

6
python/mozperftest/mozperftest/utils.py
View File

@@ -30,7 +30,7 @@ MULTI_REVISION_ROOT = f"{API_ROOT}/namespaces"
MULTI_TASK_ROOT = f"{API_ROOT}/tasks"
ON_TRY = "MOZ_AUTOMATION" in os.environ
DOWNLOAD_TIMEOUT = 30
METRICS_MATCHER = re.compile(r"(perfMetrics\s.*)")
METRICS_MATCHER = re.compile(r"(perfMetrics.*)")
PRETTY_APP_NAMES = {
"org.mozilla.fenix": "fenix",
"org.mozilla.firefox": "fenix",
@@ -87,10 +87,8 @@ class LogProcessor:
continue
self.stdout.write(data.strip("\n") + "\n")
# Check if a temporary commit wa created
match = self.matcher.match(data)
match = self.matcher.search(data)
if match:
# Last line found is the revision we want
self._match.append(match.group(1))
def flush(self):

17
python/mozperftest/perfdocs/writing.rst
View File

@@ -124,6 +124,23 @@ To run your test in CI, you may need to modify the ``_TRY_MAPPING`` variable `fo
The Mochitest test that is written can also be run as a unit test, however, if this is not desired, set the `disabled = reason` flag in the test TOML file to prevent it from running there. `See here for an example <https://searchfox.org/mozilla-central/rev/7d1b5c88343879056168aa710a9ee743392604c0/toolkit/components/ml/tests/browser/perftest.toml#7>`_.
Mochitest Android Tests in CI
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
For Mochitest Android tests in CI, everything that applies to desktop tests also applies here. When writing a new task in the ``android.yml``, ensure that there are the following fetches applied to the task::
build:
- artifact: geckoview_example.apk
extract: false
- artifact: en-US/target.perftests.tests.tar.gz
- artifact: en-US/target.condprof.tests.tar.gz
- artifact: en-US/target.common.tests.tar.gz
- artifact: en-US/target.mochitest.tests.tar.gz
toolchain:
- linux64-hostutils
Ensure that the ``runner.py`` script is also running from ``MOZ_FETCHES_DIR`` instead of the ``GECKO_PATH`` like other android MozPerftest tests. Everything else is the same as other android mozperftest tests. Note that ``--android-install-apk`` needs to be specified to point to the ``geckoview_example.apk`` that was obtained from the build task. Fenix is not currently supported in CI for Mochitest (see `bug 1902535 <https://bugzilla.mozilla.org/show_bug.cgi?id=1902535>`_).
Custom Script
-------------

17
testing/perfdocs/generated/writing.rst
View File

@@ -124,6 +124,23 @@ To run your test in CI, you may need to modify the ``_TRY_MAPPING`` variable `fo
The Mochitest test that is written can also be run as a unit test, however, if this is not desired, set the `disabled = reason` flag in the test TOML file to prevent it from running there. `See here for an example <https://searchfox.org/mozilla-central/rev/7d1b5c88343879056168aa710a9ee743392604c0/toolkit/components/ml/tests/browser/perftest.toml#7>`_.
Mochitest Android Tests in CI
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
For Mochitest Android tests in CI, everything that applies to desktop tests also applies here. When writing a new task in the ``android.yml``, ensure that there are the following fetches applied to the task::
build:
- artifact: geckoview_example.apk
extract: false
- artifact: en-US/target.perftests.tests.tar.gz
- artifact: en-US/target.condprof.tests.tar.gz
- artifact: en-US/target.common.tests.tar.gz
- artifact: en-US/target.mochitest.tests.tar.gz
toolchain:
- linux64-hostutils
Ensure that the ``runner.py`` script is also running from ``MOZ_FETCHES_DIR`` instead of the ``GECKO_PATH`` like other android MozPerftest tests. Everything else is the same as other android mozperftest tests. Note that ``--android-install-apk`` needs to be specified to point to the ``geckoview_example.apk`` that was obtained from the build task. Fenix is not currently supported in CI for Mochitest (see `bug 1902535 <https://bugzilla.mozilla.org/show_bug.cgi?id=1902535>`_).
Custom Script
-------------