Bug 1961341 - Add task routes based on the git commit when recorded in Mercurial changesets. r=taskgraph-reviewers,jcristau

After the git migration, the synchonization from github to hg.m.o will
add the git commit sha1s as "extra" data in the corresponding Mercurial
changesets.

To support things that use the index to pull artifacts from taskcluster,
like artifact builds off git clones without knowledge about Mercurial,
we need routes based on the git commit sha1s.

This adds such routes, in the same namespace as for the Mercurial
changeset sha1, because it simplifies what the other end needs to do
to accomodate. The chances that a Mercurial changeset sha1 conflict with
a git commit sha1 are pretty slim.

Differential Revision: https://phabricator.services.mozilla.com/D246026

taskcluster/docs/parameters.rst

@@ -47,6 +47,10 @@ Push Information
 ``head_rev``
    The revision to check out; this can be a short revision string
 
+``head_git_rev``
+   Optionally, the git commit corresponding to the head_rev when it's in a Mercurial
+   repository.
+
 ``base_ref``
    Reference where ``head_rev`` got merged into. It is usually a branch or a tag.
 

taskcluster/gecko_taskgraph/decision.py

@@ -38,7 +38,7 @@ from .try_option_syntax import parse_message
 from .util.backstop import ANDROID_PERFTEST_BACKSTOP_INDEX, BACKSTOP_INDEX, is_backstop
 from .util.bugbug import push_schedules
 from .util.chunking import resolver
-from .util.hg import get_hg_commit_message, get_hg_revision_branch
+from .util.hg import get_hg_commit_message, get_hg_revision_branch, get_hg_revision_info
 from .util.partials import populate_release_history
 from .util.taskcluster import insert_index
 from .util.taskgraph import find_decision_task, find_existing_tasks_from_previous_kinds
@@ -310,6 +310,11 @@ def get_decision_parameters(graph_config, options):
         env_prefix=_get_env_prefix(graph_config),
     )
 
+    if head_git_rev := get_hg_revision_info(
+        GECKO, revision=parameters["head_rev"], info="extras.git_commit"
+    ):
+        parameters["head_git_rev"] = head_git_rev
+
     # Define default filter list, as most configurations shouldn't need
     # custom filters.
     parameters["filters"] = [

taskcluster/gecko_taskgraph/parameters.py

@@ -89,6 +89,7 @@ gecko_parameters_schema = {
         Optional("routes"): [str],
     },
     Required("version"): str,
+    Optional("head_git_rev"): str,
 }
 
 

taskcluster/gecko_taskgraph/test/test_decision.py

@@ -71,6 +71,7 @@ def test_write_artifact_yml():
         decision.ARTIFACTS_DIR = "artifacts"
 
 
+@patch("gecko_taskgraph.decision.get_hg_revision_info")
 @patch("gecko_taskgraph.decision.get_hg_revision_branch")
 @patch("gecko_taskgraph.decision.get_hg_commit_message")
 @patch("gecko_taskgraph.decision._determine_more_accurate_base_rev")
@@ -91,6 +92,7 @@ def test_write_artifact_yml():
                 "try_mode": None,
                 "try_options": None,
                 "try_task_config": {},
+                "head_git_rev": "bcde",
             },
             id="simple_options",
         ),
@@ -131,6 +133,7 @@ def test_write_artifact_yml():
                     "use-artifact-builds": True,
                     "env": {},
                 },
+                "head_git_rev": "bcde",
             },
             id="try_options",
         ),
@@ -144,6 +147,7 @@ def test_write_artifact_yml():
                 "try_mode": "try_task_config",
                 "try_options": None,
                 "try_task_config": {"tasks": ["a", "b"]},
+                "head_git_rev": "bcde",
             },
             id="try_task_config",
         ),
@@ -154,12 +158,14 @@ def test_get_decision_parameters(
     mock_determine_more_accurate_base_rev,
     mock_get_hg_commit_message,
     mock_get_hg_revision_branch,
+    mock_get_hg_revision_info,
     options,
     extra_options,
     commit_msg,
     ttc,
     expected,
 ):
+    mock_get_hg_revision_info.return_value = "bcde"
     mock_get_hg_revision_branch.return_value = "default"
     mock_get_hg_commit_message.return_value = commit_msg or "commit message"
     mock_determine_more_accurate_base_rev.return_value = "baserev"

taskcluster/gecko_taskgraph/transforms/task.py

@@ -219,6 +219,7 @@ V2_ROUTE_TEMPLATES = [
     "index.{trust-domain}.v2.{project}.pushdate.{build_date}.latest.{product}.{job-name}",
     "index.{trust-domain}.v2.{project}.pushlog-id.{pushlog_id}.{product}.{job-name}",
     "index.{trust-domain}.v2.{project}.revision.{branch_rev}.{product}.{job-name}",
+    "index.{trust-domain}.v2.{project}.revision.{branch_git_rev}.{product}.{job-name}",
 ]
 
 # {central, inbound, autoland} write to a "trunk" index prefix. This facilitates
@@ -232,6 +233,7 @@ V2_SHIPPABLE_TEMPLATES = [
     "index.{trust-domain}.v2.{project}.shippable.{build_date}.revision.{branch_rev}.{product}.{job-name}",  # noqa - too long
     "index.{trust-domain}.v2.{project}.shippable.{build_date}.latest.{product}.{job-name}",
     "index.{trust-domain}.v2.{project}.shippable.revision.{branch_rev}.{product}.{job-name}",
+    "index.{trust-domain}.v2.{project}.shippable.revision.{branch_git_rev}.{product}.{job-name}",
 ]
 
 V2_SHIPPABLE_L10N_TEMPLATES = [
@@ -262,6 +264,12 @@ def get_branch_rev(config):
     ]
 
 
+def get_branch_git_rev(config):
+    return config.params[
+        "{}head_git_rev".format(config.graph_config["project-repo-param-prefix"])
+    ]
+
+
 def get_branch_repo(config):
     return config.params[
         "{}head_repository".format(
@@ -1719,11 +1727,19 @@ def add_generic_index_routes(config, task):
     subs["product"] = index["product"]
     subs["trust-domain"] = config.graph_config["trust-domain"]
     subs["branch_rev"] = get_branch_rev(config)
+    try:
+        subs["branch_git_rev"] = get_branch_git_rev(config)
+    except KeyError:
+        pass
 
     project = config.params.get("project")
 
     for tpl in V2_ROUTE_TEMPLATES:
-        routes.append(tpl.format(**subs))
+        try:
+            routes.append(tpl.format(**subs))
+        except KeyError:
+            # Ignore errors that arise from branch_git_rev not being set.
+            pass
 
     # Additionally alias all tasks for "trunk" repos into a common
     # namespace.
@@ -1752,9 +1768,17 @@ def add_shippable_index_routes(config, task):
     subs["product"] = index["product"]
     subs["trust-domain"] = config.graph_config["trust-domain"]
     subs["branch_rev"] = get_branch_rev(config)
+    try:
+        subs["branch_git_rev"] = get_branch_git_rev(config)
+    except KeyError:
+        pass
 
     for tpl in V2_SHIPPABLE_TEMPLATES:
-        routes.append(tpl.format(**subs))
+        try:
+            routes.append(tpl.format(**subs))
+        except KeyError:
+            # Ignore errors that arise from branch_git_rev not being set.
+            pass
 
     # Also add routes for en-US
     task = add_shippable_l10n_index_routes(config, task, force_locale="en-US")

taskcluster/gecko_taskgraph/util/hg.py

@@ -108,12 +108,16 @@ def get_json_pushchangedfiles(repository, revision):
 def get_hg_revision_branch(root, revision):
     """Given the parameters for a revision, find the hg_branch (aka
     relbranch) of the revision."""
+    return get_hg_revision_info(root, revision, "branch")
+
+
+def get_hg_revision_info(root, revision, info):
     return subprocess.check_output(
         [
             "hg",
             "identify",
             "-T",
-            "{branch}",
+            f"{{{info}}}",
             "--rev",
             revision,
         ],