Bug 1869592 - Automate chromedriver (stable) updates. r=perftest-reviewers,aglavic,taskgraph-reviewers,bhearsum

This patch adds logic to the fetch chromedriver scripts to now enable
selectively choosing what channel (e.g. Stable & Canary in our case).

Additionally, we now have the ability to pin specific major version in
case of any platform/OS/hardware issues.

By default if no pinned version is specified, the N-1 stable version is
fetched as a backup, along with the latest stable version. This is due
to instances where not all hardware machines update at the same time/day
and this should prevent perma failures. Otherwise this `backup` also
serves as a way to select a specific version.

Differential Revision: https://phabricator.services.mozilla.com/D233063

taskcluster/gecko_taskgraph/transforms/fetch.py

@@ -397,6 +397,12 @@ def create_chromium_fetch_task(config, name, fetch):
         Required("platform"): str,
         # The name to give to the generated artifact.
         Required("artifact-name"): str,
+        # The chrome channel to download from.
+        Optional("channel"): str,
+        # Determine if we are fetching a backup (stable version - 1) driver.
+        Optional("backup"): bool,
+        # Pin a stable version of chrome to download from. To be used together with `backup`.
+        Optional("version"): str,
     },
 )
 def create_cft_canary_fetch_task(config, name, fetch):
@@ -405,8 +411,19 @@ def create_cft_canary_fetch_task(config, name, fetch):
     workdir = "/builds/worker"
 
     platform = fetch.get("platform")
+    channel = fetch.get("channel")
+    version = fetch.get("version")
+    backup = fetch.get("backup", False)
 
     args = "--platform " + shell_quote(platform)
+    if channel:
+        args += " --channel " + shell_quote(channel)
+
+    if backup:
+        args += " --backup"
+        # only allow pinning version with backup
+        if version:
+            args += " --version " + shell_quote(version)
 
     cmd = [
         "bash",

taskcluster/gecko_taskgraph/transforms/test/other.py

@@ -399,32 +399,33 @@ def setup_browsertime(config, tasks):
 
         cd_fetches = {
             "android.*": [
-                "linux64-chromedriver-130",
+                "linux64-cft-cd-backup",
-                "linux64-chromedriver-131",
+                "linux64-cft-cd-stable",
             ],
             "linux.*": [
-                "linux64-chromedriver-130",
+                "linux64-cft-cd-backup",
-                "linux64-chromedriver-131",
+                "linux64-cft-cd-stable",
             ],
             "macosx1470.*": [
-                "mac64-chromedriver-130",
+                "mac-cft-cd-backup",
-                "mac64-chromedriver-131",
+                "mac-cft-cd-stable",
             ],
             "macosx1400.*": [
-                "mac-arm-chromedriver-130",
+                "mac-cft-cd-arm-backup",
-                "mac-arm-chromedriver-131",
+                "mac-cft-cd-arm-stable",
             ],
             "windows.*-64.*": [
-                "win64-chromedriver-130",
+                "win64-cft-cd-backup",
-                "win64-chromedriver-131",
+                "win64-cft-cd-stable",
             ],
         }
 
         chromium_fetches = {
-            "linux.*": ["linux64-cft-chromedriver"],
+            "linux.*": ["linux64-cft-cd-canary"],
-            "macosx1400.*": ["mac-cft-chromedriver-arm"],
+            "macosx1400.*": ["mac-cft-cd-arm-canary"],
-            "windows.*-64.*": ["win64-cft-chromedriver"],
+            "macosx1470.*": ["mac-cft-cd-canary"],
-            "android.*": ["linux64-cft-chromedriver"],
+            "windows.*-64.*": ["win64-cft-cd-canary"],
+            "android.*": ["linux64-cft-cd-canary"],
         }
 
         cd_extracted_name = {

taskcluster/kinds/fetch/cft-chromedriver-fetch.yml

@@ -7,26 +7,110 @@ task-defaults:
         type: cft-chromedriver-fetch
         script: /builds/worker/bin/fetch-cft-chromedriver.py
 
-linux64-cft-chromedriver:
+linux64-cft-cd-canary:
-    description: 'Linux64 Chrome-for-Testing Chromedriver Fetch'
+    description: 'Linux64 Chrome-for-Testing Chromedriver Fetch (canary channel)'
     attributes:
         cached_task: false
     fetch:
         platform: linux
         artifact-name: cft-cd-linux.tar.bz2
+        channel: Canary
 
-win64-cft-chromedriver:
+win64-cft-cd-canary:
-    description: 'Windows64 Chrome-for-Testing Chromedriver Fetch'
+    description: 'Windows64 Chrome-for-Testing Chromedriver Fetch (canary channel)'
     attributes:
         cached_task: false
     fetch:
         platform: win64
         artifact-name: cft-cd-win64.tar.bz2
+        channel: Canary
 
-mac-cft-chromedriver-arm:
+mac-cft-cd-canary:
-    description: 'MacOS (arm) Chrome-for-Testing Chromedriver fetch'
+    description: 'MacOSX Chrome-for-Testing Chromedriver Fetch (canary channel)'
+    attributes:
+        cached_task: false
+    fetch:
+        platform: mac
+        artifact-name: cft-cd-mac.tar.bz2
+        channel: Canary
+
+mac-cft-cd-arm-canary:
+    description: 'MacOS (arm) Chrome-for-Testing Chromedriver fetch (canary channel)'
     attributes:
         cached_task: false
     fetch:
         platform: mac-arm
         artifact-name: cft-cd-mac-arm.tar.bz2
+        channel: Canary
+
+linux64-cft-cd-stable:
+    description: 'Linux64 Chrome-for-Testing Chromedriver Fetch (stable channel)'
+    attributes:
+        cached_task: false
+    fetch:
+        platform: linux
+        artifact-name: cft-cd-linux.tar.bz2
+        channel: Stable
+
+win64-cft-cd-stable:
+    description: 'Windows64 Chrome-for-Testing Chromedriver Fetch (stable channel)'
+    attributes:
+        cached_task: false
+    fetch:
+        platform: win64
+        artifact-name: cft-cd-win64.tar.bz2
+        channel: Stable
+
+mac-cft-cd-stable:
+    description: 'MacOSX Chrome-for-Testing Chromedriver Fetch (stable channel)'
+    attributes:
+        cached_task: false
+    fetch:
+        platform: mac
+        artifact-name: cft-cd-mac.tar.bz2
+        channel: Stable
+
+mac-cft-cd-arm-stable:
+    description: 'MacOS (arm) Chrome-for-Testing Chromedriver fetch (stable channel)'
+    attributes:
+        cached_task: false
+    fetch:
+        platform: mac-arm
+        artifact-name: cft-cd-mac-arm.tar.bz2
+        channel: Stable
+
+linux64-cft-cd-backup:
+    description: 'Linux64 Chrome-for-Testing Chromedriver Fetch (backup/pinned version)'
+    attributes:
+        cached_task: false
+    fetch:
+        platform: linux
+        artifact-name: cft-cd-linux-backup.tar.bz2
+        backup: true
+
+win64-cft-cd-backup:
+    description: 'Windows64 Chrome-for-Testing Chromedriver Fetch (backup/pinned version)'
+    attributes:
+        cached_task: false
+    fetch:
+        platform: win64
+        artifact-name: cft-cd-win64-backup.tar.bz2
+        backup: true
+
+mac-cft-cd-backup:
+    description: 'MacOSX Chrome-for-Testing Chromedriver Fetch (backup/pinned version)'
+    attributes:
+        cached_task: false
+    fetch:
+        platform: mac
+        artifact-name: cft-cd-mac-backup.tar.bz2
+        backup: true
+
+mac-cft-cd-arm-backup:
+    description: 'MacOS (arm) Chrome-for-Testing Chromedriver Fetch (backup/pinned version)'
+    attributes:
+        cached_task: false
+    fetch:
+        platform: mac-arm
+        artifact-name: cft-cd-mac-arm-backup.tar.bz2
+        backup: true

taskcluster/scripts/misc/fetch-cft-chromedriver.py

@@ -30,29 +30,42 @@ CHROME_FOR_TESTING_INFO = {
         "platform": "linux64",
         "dir": "cft-chromedriver-linux",
         "result": "cft-cd-linux.tar.bz2",
+        "result_backup": "cft-cd-linux-backup.tar.bz2",
         "chromedriver": "chromedriver_linux64.zip",
     },
     "win64": {
         "platform": "win64",
         "dir": "cft-chromedriver-win64",
         "result": "cft-cd-win64.tar.bz2",
+        "result_backup": "cft-cd-win64-backup.tar.bz2",
         "chromedriver": "chromedriver_win32.zip",
     },
+    "mac": {
+        "platform": "mac-x64",
+        "dir": "cft-chromedriver-mac",
+        "result": "cft-cd-mac.tar.bz2",
+        "result_backup": "cft-cd-mac-backup.tar.bz2",
+        "chromedriver": "chromedriver_mac64.zip",
+    },
     "mac-arm": {
         "platform": "mac-arm64",
         "dir": "cft-chromedriver-mac",
         "result": "cft-cd-mac-arm.tar.bz2",
+        "result_backup": "cft-cd-mac-arm-backup.tar.bz2",
         "chromedriver": "chromedriver_mac64.zip",
     },
 }
 
-# Bug 1869592
-# Potentially add another JSON endpoint to grab more than 1 chromedriver
 LAST_GOOD_CFT_JSON = (
     "https://googlechromelabs.github.io/chrome-for-testing/"
     "last-known-good-versions-with-downloads.json"
 )
 
+MILESTONE_CFT_JSON = (
+    "https://googlechromelabs.github.io/chrome-for-testing/"
+    "latest-versions-per-milestone-with-downloads.json"
+)
+
 
 def log(msg):
     print("build-cft-chromedriver: %s" % msg)
@@ -77,30 +90,30 @@ def unzip(zippath, target):
     subprocess.check_call(unzip_command)
 
 
-def get_cft_metadata():
+def get_cft_metadata(endpoint=LAST_GOOD_CFT_JSON):
     """Send a request to the Chrome for Testing's last
     good json URL (default) and get the json payload which will have
     the download URLs that we need.
     """
-    res = requests.get(LAST_GOOD_CFT_JSON)
+    res = requests.get(endpoint)
     data = res.json()
 
     return data
 
 
-def get_cd_url(data, cft_platform):
+def get_cd_url(data, cft_platform, channel):
     """Given the json data, get the download URL's for
     the correct platform
     """
-    for p in data["channels"]["Canary"]["downloads"]["chromedriver"]:
+    for p in data["channels"][channel]["downloads"]["chromedriver"]:
         if p["platform"] == cft_platform:
             return p["url"]
     raise Exception("Platform not found")
 
 
-def get_chromedriver_revision(data):
+def get_chromedriver_revision(data, channel):
     """Grab revision metadata from payload"""
-    return data["channels"]["Canary"]["revision"]
+    return data["channels"][channel]["revision"]
 
 
 def fetch_chromedriver(download_url, cft_dir):
@@ -128,7 +141,43 @@ def fetch_chromedriver(download_url, cft_dir):
     shutil.copy(cd_path, cft_dir)
 
 
-def build_cft_archive(platform):
+def get_backup_chromedriver(version, cft_data, cft_platform):
+    """Download a backup chromedriver for the transitionary period of machine auto updates.
+
+    If no version is specified, by default grab the N-1 version of the latest Stable channel
+    chromedriver.
+
+    """
+    log("Grabbing a backup chromedriver...")
+    if not version:
+        log("No version specified")
+        # Get latest stable version and subtract 1
+        current_stable_version = cft_data["channels"]["Stable"]["version"].split(".")[0]
+        version = str(int(current_stable_version) - 1)
+        log("Fetching major version %s" % version)
+
+    milestone_metadata = get_cft_metadata(MILESTONE_CFT_JSON)
+    backup_revision = milestone_metadata["milestones"][version]["revision"]
+    backup_version = milestone_metadata["milestones"][version]["version"].split(".")[0]
+
+    backup_url = None
+    for p in milestone_metadata["milestones"][version]["downloads"]["chromedriver"]:
+        if p["platform"] == cft_platform:
+            backup_url = p["url"]
+
+    log("Found backup chromedriver")
+
+    if not backup_url:
+        raise Exception("Platform not found")
+
+    return backup_url, backup_revision, backup_version
+
+
+def get_version_from_json(data, channel):
+    return data["channels"][channel]["version"].split(".")[0]
+
+
+def build_cft_archive(platform, channel, backup, version):
     """Download and store a chromedriver for a given platform."""
     upload_dir = os.environ.get("UPLOAD_DIR")
     if upload_dir:
@@ -143,13 +192,25 @@ def build_cft_archive(platform):
     cft_platform = CHROME_FOR_TESTING_INFO[platform]["platform"]
 
     data = get_cft_metadata()
-    cft_chromedriver_url = get_cd_url(data, cft_platform)
+    if backup:
-    revision = get_chromedriver_revision(data)
+        cft_chromedriver_url, revision, payload_version = get_backup_chromedriver(
+            version, data, cft_platform
+        )
+        tar_file = CHROME_FOR_TESTING_INFO[platform]["result_backup"]
+    else:
+        cft_chromedriver_url = get_cd_url(data, cft_platform, channel)
+        revision = get_chromedriver_revision(data, channel)
+        payload_version = get_version_from_json(data, channel)
+        tar_file = CHROME_FOR_TESTING_INFO[platform]["result"]
     # Make a temporary location for the file
     tmppath = tempfile.mkdtemp()
 
     # Create the directory format expected for browsertime setup in taskgraph transform
     artifact_dir = CHROME_FOR_TESTING_INFO[platform]["dir"]
+    if backup or channel == "Stable":
+        # need to prepend the major version to the artifact dir due to how raptor browsertime
+        # ensures the correct version is used with chrome stable.
+        artifact_dir = payload_version + artifact_dir
     cft_dir = os.path.join(tmppath, artifact_dir)
     os.mkdir(cft_dir)
 
@@ -159,7 +220,6 @@ def build_cft_archive(platform):
     with open(revision_file, "w+") as f:
         f.write(str(revision))
 
-    tar_file = CHROME_FOR_TESTING_INFO[platform]["result"]
     tar_command = ["tar", "cjf", tar_file, "-C", tmppath, artifact_dir]
     log("Revision is %s" % revision)
     log("Added revision to %s file." % revision_file)
@@ -185,6 +245,25 @@ def parse_args():
         required=True,
     )
     # Bug 1869592 - Add optional flag to provide CfT channel e.g. Canary, Stable, etc.
+    parser.add_argument(
+        "--channel",
+        help="Corresponding channel of CfT chromedriver to fetch.",
+        required=False,
+        default="Canary",
+    )
+    parser.add_argument(
+        "--backup",
+        help="Determine if we are grabbing a backup chromedriver version.",
+        required=False,
+        default=False,
+        action="store_true",
+    )
+    parser.add_argument(
+        "--version",
+        help="Pin the revision if necessary for current platform",
+        required=False,
+        default="",
+    )
     return parser.parse_args()
 
 

testing/raptor/raptor/browsertime/base.py

@@ -186,16 +186,16 @@ class Browsertime(Perftest):
                 # setup once all chrome versions use the new artifact setup.
                 cd_extracted_names_115 = {
                     "windows": str(
-                        pathlib.Path("{}chromedriver-win64", "chromedriver.exe")
+                        pathlib.Path("{}cft-chromedriver-win64", "chromedriver.exe")
                     ),
                     "mac-x86_64": str(
-                        pathlib.Path("{}chromedriver-mac-x64", "chromedriver")
+                        pathlib.Path("{}cft-chromedriver-mac", "chromedriver")
                     ),
                     "mac-aarch64": str(
-                        pathlib.Path("{}chromedriver-mac-arm64", "chromedriver")
+                        pathlib.Path("{}cft-chromedriver-mac", "chromedriver")
                     ),
                     "default": str(
-                        pathlib.Path("{}chromedriver-linux64", "chromedriver")
+                        pathlib.Path("{}cft-chromedriver-linux", "chromedriver")
                     ),
                 }