Bug 1715900 - Add a mach test-interventions command for testing webcompat interventions; r=jgraham,mhentges

Differential Revision: https://phabricator.services.mozilla.com/D138384

build/mach_initialize.py

@@ -359,6 +359,9 @@ def initialize(topsrcdir):
         "power": MachCommandReference("tools/power/mach_commands.py"),
         "try": MachCommandReference("tools/tryselect/mach_commands.py"),
         "import-pr": MachCommandReference("tools/vcs/mach_commands.py"),
+        "test-interventions": MachCommandReference(
+            "testing/webcompat/mach_commands.py"
+        ),
     }
 
     # Set a reasonable limit to the number of open files.

build/webcompat_virtualenv_packages.txt

@@ -0,0 +1,5 @@
+pth:testing/webcompat
+pypi:pytest==4.6.6
+pypi:selenium==3.141.0
+vendored:testing/web-platform/tests/tools/webdriver
+vendored:testing/web-platform/tests/tools/third_party/websockets/src

taskcluster/ci/config.yml

@@ -201,6 +201,7 @@ treeherder:
         'perftest-http3': 'Performance tests with HTTP/3'
         'l10n': 'Localization checks'
         'fxrec': 'Desktop startup recorder (fxrecord)'
+        'wc': 'webcompat'
 
 index:
     products:

taskcluster/ci/source-test/kind.yml

@@ -31,6 +31,7 @@ jobs-from:
     - shadow-scheduler.yml
     - taskgraph.yml
     - webidl.yml
+    - webcompat.yml
     - wpt-manifest.yml
     - wpt-metadata.yml
 

taskcluster/ci/source-test/webcompat.yml

@@ -0,0 +1,37 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+job-defaults:
+    platform: linux1804-64/opt
+    require-build:
+        by-project:
+            try:
+                linux1804-64/opt: build-linux64/opt
+            default:
+                linux1804-64/opt: build-linux64-shippable/opt
+    fetches:
+        build:
+            - target.tar.bz2
+        toolchain:
+            - linux64-geckodriver
+    run-on-projects: []
+    treeherder:
+        kind: test
+    worker-type: t-linux-xlarge-source
+    worker:
+        docker-image: {in-tree: ubuntu1804-test}
+        max-run-time: 1800
+    optimization:
+        skip-unless-expanded: null
+
+interventions:
+    description: webcompat intervention tests
+    treeherder:
+        symbol: wc(I)
+        tier: 3
+    python-version: [3]
+    run:
+        using: mach
+        # Need to start Xvfb if we remove --headless
+        mach: test-interventions --headless --binary $MOZ_FETCHES_DIR/firefox/firefox --webdriver-binary $MOZ_FETCHES_DIR/geckodriver --log-tbpl -

testing/webcompat/config.json.sample

@@ -0,0 +1,5 @@
+{
+  "bug number 1": {"username": "name", "password": "pass"}],
+  "bug number 2": {"username": "name", "password": "pass"}]
+}
+

testing/webcompat/interventions/conftest.py

@@ -0,0 +1,149 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import re
+import subprocess
+
+import pytest
+from selenium.webdriver import Remote
+
+
+def pytest_generate_tests(metafunc):
+    """Generate tests based on markers."""
+
+    if "session" not in metafunc.fixturenames:
+        return
+
+    marks = [mark.name for mark in metafunc.function.pytestmark]
+
+    argvalues = []
+    ids = []
+
+    skip_platforms = []
+    if "skip_platforms" in marks:
+        for mark in metafunc.function.pytestmark:
+            if mark.name == "skip_platforms":
+                skip_platforms = mark.args
+
+    if "with_interventions" in marks:
+        argvalues.append([{"interventions": True, "skip_platforms": skip_platforms}])
+        ids.append("with_interventions")
+
+    if "without_interventions" in marks:
+        argvalues.append([{"interventions": False, "skip_platforms": skip_platforms}])
+        ids.append("without_interventions")
+
+    metafunc.parametrize(["session"], argvalues, ids=ids, indirect=True)
+
+
+class WebDriver:
+    def __init__(self, config):
+        self.browser_binary = config.getoption("browser_binary")
+        self.webdriver_binary = config.getoption("webdriver_binary")
+        self.port = config.getoption("webdriver_port")
+        self.headless = config.getoption("headless")
+        self.proc = None
+
+    def command_line_driver(self):
+        raise NotImplementedError
+
+    def capabilities(self, use_interventions):
+        raise NotImplementedError
+
+    def __enter__(self):
+        assert self.proc is None
+        self.proc = subprocess.Popen(self.command_line_driver())
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        self.proc.kill()
+
+
+class FirefoxWebDriver(WebDriver):
+    def command_line_driver(self):
+        return [self.webdriver_binary, "--port", str(self.port), "-vv"]
+
+    def capabilities(self, use_interventions):
+        fx_options = {"binary": self.browser_binary}
+
+        interventions_prefs = [
+            "perform_injections",
+            "perform_ua_overrides",
+            "enable_shims",
+            "enable_picture_in_picture_overrides",
+        ]
+        fx_options["prefs"] = {
+            f"extensions.webcompat.{pref}": use_interventions
+            for pref in interventions_prefs
+        }
+        if self.headless:
+            fx_options["args"] = ["--headless"]
+
+        return {
+            "pageLoadStrategy": "normal",
+            "moz:firefoxOptions": fx_options,
+        }
+
+
+@pytest.fixture(scope="session")
+def config_file(request):
+    path = request.config.getoption("config")
+    if not path:
+        return None
+    with open(path) as f:
+        return json.load(f)
+
+
+@pytest.fixture
+def credentials(request, config_file):
+    bug_number = re.findall("\d+", str(request.fspath.basename))[0]
+
+    if not config_file:
+        pytest.skip(f"login info required for bug #{bug_number}")
+        return None
+
+    try:
+        credentials = config_file[bug_number]
+    except KeyError:
+        pytest.skip(f"no login for bug #{bug_number} found")
+        return
+
+    return {"username": credentials["username"], "password": credentials["password"]}
+
+
+@pytest.fixture(scope="session")
+def driver(pytestconfig):
+    if pytestconfig.getoption("browser") == "firefox":
+        cls = FirefoxWebDriver
+    else:
+        assert False
+
+    with cls(pytestconfig) as driver_instance:
+        yield driver_instance
+
+
+@pytest.fixture(scope="session")
+def session(request, driver):
+    use_interventions = request.param.get("interventions")
+    print(f"use_interventions {use_interventions}")
+    if use_interventions is None:
+        raise ValueError(
+            "Missing intervention marker in %s:%s"
+            % (request.fspath, request.function.__name__)
+        )
+    capabilities = driver.capabilities(use_interventions)
+    print(capabilities)
+
+    url = f"http://localhost:{driver.port}"
+    with Remote(command_executor=url, desired_capabilities=capabilities) as session:
+        yield session
+
+
+@pytest.fixture(autouse=True)
+def skip_platforms(request, session):
+    platform = session.capabilities["platformName"]
+    if request.node.get_closest_marker("skip_platforms"):
+        if request.node.get_closest_marker("skip_platforms").args[0] == platform:
+            pytest.skip(f"Skipped on platform: {platform}")

testing/webcompat/interventions/pytest.ini

@@ -0,0 +1,6 @@
+[pytest]
+console_output_style = classic
+markers =
+  skip_platforms: skip tests on specific platforms
+  with_interventions: enable web-compat interventions
+  without_interventions: disable web-compat interventions

testing/webcompat/mach_commands.py

@@ -0,0 +1,109 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import os
+import sys
+
+from mach.decorators import Command
+from mozbuild.base import MozbuildObject
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+def create_parser_interventions():
+    from mozlog import commandline
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--binary", help="Path to browser binary")
+    parser.add_argument("--webdriver-binary", help="Path to webdriver binary")
+    parser.add_argument("--bug", help="Bug to run tests for")
+    parser.add_argument(
+        "--config", help="Path to JSON file containing logins and other settings"
+    )
+    parser.add_argument(
+        "--debug", action="store_true", default=False, help="Debug failing tests"
+    )
+    parser.add_argument(
+        "--headless",
+        action="store_true",
+        default=False,
+        help="Run firefox in headless mode",
+    )
+    parser.add_argument(
+        "--interventions",
+        action="store",
+        default="both",
+        choices=["enabled", "disabled", "both"],
+        help="Enable webcompat interventions",
+    )
+    commandline.add_logging_group(parser)
+    return parser
+
+
+class InterventionTest(MozbuildObject):
+    def set_default_kwargs(self, kwargs):
+        if kwargs["binary"] is None:
+            kwargs["binary"] = self.get_binary_path()
+
+        if kwargs["webdriver_binary"] is None:
+            kwargs["webdriver_binary"] = self.get_binary_path(
+                "geckodriver", validate_exists=False
+            )
+
+    def get_capabilities(self, kwargs):
+        return {"moz:firefoxOptions": {"binary": kwargs["binary"]}}
+
+    def run(self, **kwargs):
+        import runner
+        import mozlog
+
+        mozlog.commandline.setup_logging(
+            "test-interventions", kwargs, {"mach": sys.stdout}
+        )
+        logger = mozlog.get_default_logger("test-interventions")
+        status_handler = mozlog.handlers.StatusHandler()
+        logger.add_handler(status_handler)
+
+        self.set_default_kwargs(kwargs)
+
+        interventions = (
+            ["enabled", "disabled"]
+            if kwargs["interventions"] == "both"
+            else [kwargs["interventions"]]
+        )
+
+        for interventions_setting in interventions:
+            runner.run(
+                logger,
+                os.path.join(here, "interventions"),
+                kwargs["binary"],
+                kwargs["webdriver_binary"],
+                bug=kwargs["bug"],
+                debug=kwargs["debug"],
+                interventions=interventions_setting,
+                config=kwargs["config"],
+                headless=kwargs["headless"],
+            )
+
+        summary = status_handler.summarize()
+        passed = (
+            summary.unexpected_statuses == 0
+            and summary.log_level_counts.get("ERROR", 0) == 0
+            and summary.log_level_counts.get("CRITICAL", 0) == 0
+        )
+        return passed
+
+
+@Command(
+    "test-interventions",
+    category="testing",
+    description="Test the webcompat interventions",
+    parser=create_parser_interventions,
+    virtualenv_name="webcompat",
+)
+def test_interventions(command_context, **params):
+    command_context.activate_virtualenv()
+    intervention_test = command_context._spawn(InterventionTest)
+    return 0 if intervention_test.run(**params) else 1

testing/webcompat/requirements.txt

@@ -0,0 +1,3 @@
+pytest==4.6.6
+selenium==3.141.0
+urllib3==1.26

testing/webcompat/runner.py

@@ -0,0 +1,183 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import errno
+import os
+import shutil
+import tempfile
+
+import pytest
+
+
+def run(
+    logger,
+    path,
+    browser_binary,
+    webdriver_binary,
+    environ=None,
+    bug=None,
+    debug=False,
+    interventions=None,
+    config=None,
+    headless=False,
+):
+    """"""
+    old_environ = os.environ.copy()
+    try:
+        with TemporaryDirectory() as cache:
+            if environ:
+                os.environ.update(environ)
+
+            config_plugin = WDConfig()
+            result_recorder = ResultRecorder(logger)
+
+            args = [
+                "--strict",  # turn warnings into errors
+                "-vv",  # show each individual subtest and full failure logs
+                "--capture",
+                "no",  # enable stdout/stderr from tests
+                "--basetemp",
+                cache,  # temporary directory
+                "--showlocals",  # display contents of variables in local scope
+                "-p",
+                "no:mozlog",  # use the WPT result recorder
+                "--disable-warnings",
+                "-rfEs",
+                "-p",
+                "no:cacheprovider",  # disable state preservation across invocations
+                "-o=console_output_style=classic",  # disable test progress bar
+                "--browser",
+                "firefox",
+                "--browser-binary",
+                browser_binary,
+                "--webdriver-binary",
+                webdriver_binary,
+            ]
+
+            if debug:
+                args.append("--pdb")
+
+            if headless:
+                args.append("--headless")
+
+            if config:
+                args.append("--config")
+                args.append(config)
+
+            if interventions == "enabled":
+                args.extend(["-m", "with_interventions"])
+            elif interventions == "disabled":
+                args.extend(["-m", "without_interventions"])
+            elif interventions is not None:
+                raise ValueError(f"Invalid value for interventions {interventions}")
+            else:
+                raise ValueError("Must provide interventions argument")
+
+            if bug is not None:
+                args.extend(["-k", bug])
+
+            args.append(path)
+            try:
+                logger.suite_start([], name="webcompat-interventions")
+                pytest.main(args, plugins=[config_plugin, result_recorder])
+            except Exception as e:
+                logger.critical(str(e))
+            finally:
+                logger.suite_end()
+
+    finally:
+        os.environ = old_environ
+
+
+class WDConfig:
+    def pytest_addoption(self, parser):
+        parser.addoption(
+            "--browser-binary", action="store", help="Path to browser binary"
+        )
+        parser.addoption(
+            "--webdriver-binary", action="store", help="Path to webdriver binary"
+        )
+        parser.addoption(
+            "--webdriver-port",
+            action="store",
+            default=4444,
+            help="Port on which to run WebDriver",
+        )
+        parser.addoption(
+            "--browser", action="store", choices=["firefox"], help="Name of the browser"
+        )
+        parser.addoption("--bug", action="store", help="Bug number to run tests for")
+        parser.addoption(
+            "--config",
+            action="store",
+            help="Path to JSON file containing logins and other settings",
+        )
+        parser.addoption(
+            "--headless", action="store_true", help="Run browser in headless mode"
+        )
+
+
+class ResultRecorder(object):
+    def __init__(self, logger):
+        self.logger = logger
+
+    def pytest_runtest_logreport(self, report):
+        if report.passed and report.when == "call":
+            self.record_pass(report)
+        elif report.failed:
+            if report.when != "call":
+                self.record_error(report)
+            else:
+                self.record_fail(report)
+        elif report.skipped:
+            self.record_skip(report)
+
+    def record_pass(self, report):
+        self.record(report.nodeid, "PASS")
+
+    def record_fail(self, report):
+        # pytest outputs the stacktrace followed by an error message prefixed
+        # with "E   ", e.g.
+        #
+        #        def test_example():
+        #  >         assert "fuu" in "foobar"
+        #  > E       AssertionError: assert 'fuu' in 'foobar'
+        message = ""
+        for line in report.longreprtext.splitlines():
+            if line.startswith("E   "):
+                message = line[1:].strip()
+                break
+
+        self.record(report.nodeid, "FAIL", message=message, stack=report.longrepr)
+
+    def record_error(self, report):
+        # error in setup/teardown
+        if report.when != "call":
+            message = "%s error" % report.when
+        self.record(report.nodeid, "ERROR", message, report.longrepr)
+
+    def record_skip(self, report):
+        self.record(report.nodeid, "SKIP")
+
+    def record(self, test, status, message=None, stack=None):
+        if stack is not None:
+            stack = str(stack)
+        self.logger.test_start(test)
+        self.logger.test_end(
+            test=test, status=status, expected="PASS", message=message, stack=stack
+        )
+
+
+class TemporaryDirectory(object):
+    def __enter__(self):
+        self.path = tempfile.mkdtemp(prefix="pytest-")
+        return self.path
+
+    def __exit__(self, *args):
+        try:
+            shutil.rmtree(self.path)
+        except OSError as e:
+            # no such file or directory
+            if e.errno != errno.ENOENT:
+                raise

tools/lint/license.yml

@@ -34,6 +34,7 @@ license:
         - mobile/android/geckoview/src/main/AndroidManifest_overlay.jinja
         - mobile/android/geckoview/src/main/res/drawable/ic_generic_file.xml
         - mobile/android/geckoview_example/src/main
+        - testing/webcompat/interventions/
         # might not work with license
         - mobile/android/gradle/dotgradle-offline/gradle.properties
         # might not work with license