diff --git a/taskcluster/docs/kinds.rst b/taskcluster/docs/kinds.rst index 12b5d9dad3c5..46b82beab078 100644 --- a/taskcluster/docs/kinds.rst +++ b/taskcluster/docs/kinds.rst @@ -216,13 +216,6 @@ Beetmover, takes specific artifacts, "Beets", and pushes them to a location outs of Taskcluster's task artifacts, (archive.mozilla.org as one place) and in the process determines the final location and a "pretty" name (versioned product name) -beetmover-integration ---------------------- -Beetmover, takes specific artifacts, "Beets", and pushes them to a location outside -of Taskcluster's task artifacts, (archive.mozilla.org as one place) and in the -process determines the final location and a "pretty" name (versioned product name) -This separate kind archives builds from the autoland branch. - beetmover-l10n -------------- diff --git a/taskcluster/gecko_taskgraph/manifests/firefox_integration.yml b/taskcluster/gecko_taskgraph/manifests/firefox_integration.yml deleted file mode 100644 index 984a1ccbf6c3..000000000000 --- a/taskcluster/gecko_taskgraph/manifests/firefox_integration.yml +++ /dev/null @@ -1,153 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. ---- -# This file contains exhaustive information about all the release artifacs that -# are needed within a type of release. -# -# Structure -# -------- -# `s3_bucket_paths` -- prefix to be used per product to correctly access our S3 buckets -# `default_locales` -- list of locales to be used when composing upstream artifacts or the list of -# destinations. If given an empty locale, it uses these locales instead. -# `tasktype_map` -- mapping between task reference and task type, particularly usefule when -# composing the upstreamArtifacts for scriptworker. -# `platform_names` -- various platform mappings used in reckoning artifacts or other paths -# `default` -- a default entry, which the mappings extend and override in such a way that -# final path full-destinations will be a concatenation of the following: -# `s3_bucket_paths`, `destinations`, `locale_prefix`, `pretty_name` -# `from` -- specifies the dependency(ies) from which to expect the particular artifact -# `all_locales` -- boolean argument to specify whether that particular artifact is to be expected -# for all locales or just the default one -# `description` -- brief summary of what that artifact is -# `locale_prefix` -- prefix to be used in the final destination paths, whether that's for default locale or not -# `source_path_modifier` -- any parent dir that might be used in between artifact prefix and filename at source location -# for example `public/build` vs `public/build/ach/`. -# `destinations` -- final list of directories where to push the artifacts in S3 -# `pretty_name` -- the final name the artifact will have at destination -# `checksums_path` -- the name to identify one artifact within the checksums file -# `not_for_platforms` -- filtering option to avoid associating an artifact with a specific platform -# `only_for_platforms` -- filtering option to exclusively include the association of an artifact for a specific platform -# `partials_only` -- filtering option to avoid associating an artifact unless this flag is present -# `update_balrog_manifest`-- flag needed downstream in beetmover jobs to reckon the balrog manifest -# `from_buildid` -- flag needed downstream in beetmover jobs to reckon the balrog manifest - -s3_bucket_paths: - - pub/firefox/integration -default_locales: - - en-US -tasktype_map: - build: build - signing: signing - repackage: repackage -platform_names: - filename_platform: - by-platform: - linux: 'linux-i686' - linux-shippable: 'linux-i686-shippable' - linux64: 'linux64-x86_64' - linux64-shippable: 'linux-x86_64-shippable' - linux64-aarch64-shippable: 'linux-aarch64' - linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' - macosx64: 'mac' - macosx64-shippable: 'mac-shippable' - macosx64-aarch64-shippable: 'mac-aarch64' - win32: 'win32' - win32-shippable: 'win32-shippable' - win64: 'win64' - win64-shippable: 'win64-shippable' - win64-aarch64: 'win64-aarch64' - win64-aarch64-shippable: 'win64-aarch64' - win64-asan-reporter-shippable: 'win64-asan-reporter' - -default: &default - from: - - build - all_locales: false - description: "TO_BE_OVERRIDDEN" - locale_prefix: '' - source_path_modifier: - by-locale: - default: '${locale}' - en-US: '' - destinations: - - ${branch}/${year}/${month}/${day}/${head_rev} - expiry: "2 years" - -mapping: - target.json: - <<: *default - description: "Various compile and moz_app flags baked together in a json file" - pretty_name: firefox-${version}.${locale}.${filename_platform}.json - checksums_path: firefox-${version}.${locale}.${filename_platform}.json - target.mozinfo.json: - <<: *default - description: "Various compile and moz_app flags baked together in a json file" - pretty_name: firefox-${version}.${locale}.${filename_platform}.mozinfo.json - checksums_path: firefox-${version}.${locale}.${filename_platform}.mozinfo.json - target.txt: - <<: *default - description: "File containing buildid and revision" - pretty_name: firefox-${version}.${locale}.${filename_platform}.txt - checksums_path: firefox-${version}.${locale}.${filename_platform}.txt - target_info.txt: - <<: *default - description: "File containing the buildID" - pretty_name: firefox-${version}.${locale}.${filename_platform}_info.txt - checksums_path: firefox-${version}.${locale}.${filename_platform}_info.txt - target.jsshell.zip: - <<: *default - description: "Set of shells to allow test snippets of Javascript code without needing to reload the page" - pretty_name: jsshell-${filename_platform}.zip - checksums_path: jsshell-${filename_platform}.zip - target.tar.xz: - <<: *default - description: "Main installer for Linux platforms" - only_for_platforms: - - linux - - linux-shippable - - linux64 - - linux64-shippable - - linux64-aarch64-shippable - - linux64-asan-reporter-shippable - pretty_name: firefox-${version}.${locale}.${filename_platform}.tar.xz - checksums_path: firefox-${version}.${locale}.${filename_platform}.tar.xz - target.dmg: - <<: *default - description: "Main package disk image for Mac OS X platforms" - # Attribution-l10n jobs don't have locale in the artifact path - source_path_modifier: "" - only_for_platforms: - - macosx64 - - macosx64-shippable - - macosx64-aarch64-shippable - pretty_name: firefox-${version}.${locale}.${filename_platform}.dmg - checksums_path: firefox-${version}.${locale}.${filename_platform}.dmg - target.zip: - <<: *default - description: "Main package installer for Windows platforms" - only_for_platforms: - - win32 - - win32-shippable - - win64 - - win64-shippable - - win64-aarch64 - - win64-aarch64-shippable - - win64-asan-reporter-shippable - pretty_name: firefox-${version}.${locale}.${filename_platform}.zip - checksums_path: firefox-${version}.${locale}.${filename_platform}.zip - target.installer.exe: - <<: *default - description: "Main installer for Windows platforms" - # Need to add the installer path since it's sourcing from build kind instead of repackage-signing - source_path_modifier: 'install/sea/' - only_for_platforms: - - win32 - - win32-shippable - - win64 - - win64-shippable - - win64-aarch64 - - win64-aarch64-shippable - - win64-asan-reporter-shippable - pretty_name: firefox-${version}.${locale}.${filename_platform}.installer.exe - checksums_path: firefox-${version}.${locale}.${filename_platform}.installer.exe diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_integration.py b/taskcluster/gecko_taskgraph/transforms/beetmover_integration.py deleted file mode 100644 index bd0de710e92d..000000000000 --- a/taskcluster/gecko_taskgraph/transforms/beetmover_integration.py +++ /dev/null @@ -1,123 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" -Transform the beetmover task into an actual task description. -""" - - -from copy import deepcopy - -from taskgraph.transforms.base import TransformSequence -from taskgraph.util.dependencies import get_primary_dependency -from taskgraph.util.schema import Schema -from voluptuous import Optional, Required - -from gecko_taskgraph.transforms.beetmover import craft_release_properties -from gecko_taskgraph.transforms.task import task_description_schema -from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job -from gecko_taskgraph.util.scriptworker import ( - generate_beetmover_artifact_map, - generate_beetmover_upstream_artifacts, - get_beetmover_action_scope, - get_beetmover_bucket_scope, -) - -beetmover_description_schema = Schema( - { - Required("label"): str, - Required("dependencies"): task_description_schema["dependencies"], - Required("if-dependencies"): task_description_schema["if-dependencies"], - Optional("treeherder"): task_description_schema["treeherder"], - Required("run-on-projects"): task_description_schema["run-on-projects"], - Optional("attributes"): task_description_schema["attributes"], - Optional("task-from"): task_description_schema["task-from"], - } -) - -transforms = TransformSequence() - - -@transforms.add -def remove_name(config, jobs): - for job in jobs: - if "name" in job: - del job["name"] - yield job - - -transforms.add_validate(beetmover_description_schema) - - -@transforms.add -def make_task_description(config, jobs): - for job in jobs: - dep_job = get_primary_dependency(config, job) - assert dep_job - - attributes = copy_attributes_from_dependent_job(dep_job) - attributes.update(job.get("attributes", {})) - - treeherder = job.get("treeherder", {}) - dep_th_platform = ( - dep_job.task.get("extra", {}) - .get("treeherder", {}) - .get("machine", {}) - .get("platform", "") - ) - treeherder.setdefault("platform", f"{dep_th_platform}/opt") - treeherder.setdefault("tier", 2) - treeherder.setdefault("kind", "build") - treeherder.setdefault("symbol", "BM-int") - label = job["label"] - description = ( - "Beetmover submission for autoland-" - "{build_platform}/{build_type}'".format( - build_platform=attributes.get("build_platform"), - build_type=attributes.get("build_type"), - ) - ) - - job["dependencies"].update(deepcopy(dep_job.dependencies)) - - if job.get("locale"): - attributes["locale"] = job["locale"] - - bucket_scope = get_beetmover_bucket_scope(config) - action_scope = get_beetmover_action_scope(config) - - task = { - "label": label, - "description": description, - "worker-type": "beetmover", - "scopes": [bucket_scope, action_scope], - "dependencies": job["dependencies"], - "if-dependencies": job.get("if-dependencies"), - "attributes": attributes, - "run-on-projects": job["run-on-projects"], - "treeherder": treeherder, - } - - yield task - - -@transforms.add -def make_task_worker(config, jobs): - for job in jobs: - locale = job["attributes"].get("locale") - platform = job["attributes"]["build_platform"] - - worker = { - "implementation": "beetmover", - "release-properties": craft_release_properties(config, job), - "upstream-artifacts": generate_beetmover_upstream_artifacts( - config, job, platform, locale - ), - "artifact-map": generate_beetmover_artifact_map( - config, job, platform=platform, locale=locale - ), - } - - job["worker"] = worker - - yield job diff --git a/taskcluster/gecko_taskgraph/util/scriptworker.py b/taskcluster/gecko_taskgraph/util/scriptworker.py index 19cf4dd4db06..c10f0c098e5e 100644 --- a/taskcluster/gecko_taskgraph/util/scriptworker.py +++ b/taskcluster/gecko_taskgraph/util/scriptworker.py @@ -711,9 +711,7 @@ def generate_beetmover_artifact_map(config, job, **kwargs): "build_number": config.params["build_number"], "year": upload_date.year, "month": upload_date.strftime("%m"), # zero-pad the month - "day": upload_date.strftime("%d"), "upload_date": upload_date.strftime("%Y-%m-%d-%H-%M-%S"), - "head_rev": config.params["head_rev"], } ) kwargs.update(**platforms) diff --git a/taskcluster/kinds/beetmover-integration/kind.yml b/taskcluster/kinds/beetmover-integration/kind.yml deleted file mode 100644 index f02b14425af8..000000000000 --- a/taskcluster/kinds/beetmover-integration/kind.yml +++ /dev/null @@ -1,32 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. ---- -loader: taskgraph.loader.transform:loader - -transforms: - - taskgraph.transforms.from_deps - - gecko_taskgraph.transforms.name_sanity - - gecko_taskgraph.transforms.beetmover_integration - - gecko_taskgraph.transforms.task - -kind-dependencies: - # Primary dependencies - - build - -only-for-build-platforms: - - linux/opt - - linux64/opt - - macosx64/opt - - win32/opt - - win64/opt - -tasks: - beetmover-integration: - from-deps: - group-by: platform - attributes: - artifact_map: taskcluster/gecko_taskgraph/manifests/firefox_integration.yml - if-dependencies: - - build - run-on-projects: [autoland]