diff --git a/taskcluster/ci/build/kind.yml b/taskcluster/ci/build/kind.yml index bf412f1fc..bd71b6a17 100644 --- a/taskcluster/ci/build/kind.yml +++ b/taskcluster/ci/build/kind.yml @@ -21,6 +21,7 @@ job-defaults: apk-artifact-template: type: file name: public/build/{abi}/target.apk + github-name: 'fenix-{version}-{abi}.apk' path: '/builds/worker/checkouts/src/app/build/outputs/apk/{gradle_build_type}/{fileName}' description: Build Fenix from source code. fetches: diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index 6981d4f26..bb332d2b0 100644 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -43,6 +43,11 @@ workers: implementation: scriptworker-signing os: scriptworker worker-type: mobile-t-signing + github: + provisioner: scriptworker-k8s + implementation: scriptworker-github + os: scriptworker + worker-type: 'mobile-{level}-github' images: provisioner: 'mobile-{level}' implementation: docker-worker diff --git a/taskcluster/ci/github-release/kind.yml b/taskcluster/ci/github-release/kind.yml new file mode 100644 index 000000000..8d7df3941 --- /dev/null +++ b/taskcluster/ci/github-release/kind.yml @@ -0,0 +1,44 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: fenix_taskgraph.loader.multi_dep:loader + +transforms: + - fenix_taskgraph.transforms.multi_dep:transforms + - fenix_taskgraph.transforms.github_release:transforms + - taskgraph.transforms.task:transforms + +kind-dependencies: + - signing + +primary-dependency: signing + +group-by: build-type + +only-for-build-types: + - beta + - release + +job-template: + description: Create/Update Github release and upload APKs + worker-type: github + worker: + action: release + github-project: + by-level: + '3': fenix + default: mock + is-prerelease: + by-build-type: + beta: true + release: false + default: true + release-name: + by-build-type: + beta: Firefox Beta {version} + release: Firefox {version} + + treeherder: + job-symbol: gh-r + kind: build diff --git a/taskcluster/fenix_taskgraph/transforms/browsertime.py b/taskcluster/fenix_taskgraph/transforms/browsertime.py index 02fefa48c..27bc2e52a 100644 --- a/taskcluster/fenix_taskgraph/transforms/browsertime.py +++ b/taskcluster/fenix_taskgraph/transforms/browsertime.py @@ -30,9 +30,10 @@ def add_variants(config, tasks): if build_type not in only_types: continue - for abi, apk_path in dep_task.attributes["apks"].items(): + for abi, apk_metadata in dep_task.attributes["apks"].items(): if abi not in only_abis: continue + apk_path = apk_metadata["name"] for test in tests: test = copy.deepcopy(test) attributes = copy.deepcopy(dep_task.attributes) diff --git a/taskcluster/fenix_taskgraph/transforms/build.py b/taskcluster/fenix_taskgraph/transforms/build.py index ebc4b94de..2c62117bf 100644 --- a/taskcluster/fenix_taskgraph/transforms/build.py +++ b/taskcluster/fenix_taskgraph/transforms/build.py @@ -121,7 +121,13 @@ def add_artifacts(config, tasks): **apk ), }) - apks[apk["abi"]] = apk_name + apks[apk["abi"]] = { + "name": apk_name, + "github-name": artifact_template["github-name"].format( + version=config.params["version"], + **apk + ) + } yield task diff --git a/taskcluster/fenix_taskgraph/transforms/github_release.py b/taskcluster/fenix_taskgraph/transforms/github_release.py new file mode 100644 index 000000000..fc32583fc --- /dev/null +++ b/taskcluster/fenix_taskgraph/transforms/github_release.py @@ -0,0 +1,64 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the jobs defined in the github_release +kind. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + for key in ("worker.github-project", "worker.is-prerelease", "worker.release-name"): + resolve_keyed_by( + task, + key, + item_name=task["name"], + **{ + 'build-type': task["attributes"]["build-type"], + 'level': config.params["level"], + } + ) + yield task + + +@transforms.add +def build_worker_definition(config, tasks): + for task in tasks: + worker_definition = { + "artifact-map": _build_artifact_map(task), + "git-tag": config.params["head_tag"].decode("utf-8"), + "git-revision": config.params["head_rev"].decode("utf-8"), + "release-name": task["worker"]["release-name"].format(version=config.params["version"]), + } + + task["worker"].update(worker_definition) + + yield task + +def _build_artifact_map(task): + artifact_map = [] + github_names_per_path = { + apk_metadata["name"]: apk_metadata["github-name"] + for apk_metadata in task["attributes"]["apks"].values() + } + + for upstream_artifact_metadata in task["worker"]["upstream-artifacts"]: + artifacts = {"paths": {}, "taskId": upstream_artifact_metadata["taskId"]} + for path in upstream_artifact_metadata["paths"]: + artifacts["paths"][path] = { + "destinations": [github_names_per_path[path]] + } + + artifact_map.append(artifacts) + + return artifact_map diff --git a/taskcluster/fenix_taskgraph/transforms/multi_dep.py b/taskcluster/fenix_taskgraph/transforms/multi_dep.py index 4924182a2..f1161dc9e 100644 --- a/taskcluster/fenix_taskgraph/transforms/multi_dep.py +++ b/taskcluster/fenix_taskgraph/transforms/multi_dep.py @@ -66,7 +66,10 @@ def build_upstream_artifacts(config, tasks): } for dep in _get_all_deps(task).values(): - paths = sorted(dep.attributes.get("apks", {}).values()) + paths = sorted([ + apk_metadata["name"] + for apk_metadata in dep.attributes.get("apks", {}).values() + ]) if paths: worker_definition["upstream-artifacts"].append({ "taskId": {"task-reference": "<{}>".format(dep.kind)}, diff --git a/taskcluster/fenix_taskgraph/transforms/raptor.py b/taskcluster/fenix_taskgraph/transforms/raptor.py index 6e7b38004..d087e9c87 100644 --- a/taskcluster/fenix_taskgraph/transforms/raptor.py +++ b/taskcluster/fenix_taskgraph/transforms/raptor.py @@ -30,9 +30,10 @@ def add_variants(config, tasks): if build_type not in only_types: continue - for abi, apk_path in dep_task.attributes["apks"].items(): + for abi, apk_metadata in dep_task.attributes["apks"].items(): if abi not in only_abis: continue + apk_path = apk_metadata["name"] for test in tests: test = copy.deepcopy(test) attributes = copy.deepcopy(dep_task.attributes) diff --git a/taskcluster/fenix_taskgraph/worker_types.py b/taskcluster/fenix_taskgraph/worker_types.py index 7dd9f8c95..1a8073e71 100644 --- a/taskcluster/fenix_taskgraph/worker_types.py +++ b/taskcluster/fenix_taskgraph/worker_types.py @@ -109,7 +109,7 @@ def build_push_apk_payload(config, task, task_def): Required("release-name"): text_type, }, ) -def build_push_apk_payload(config, task, task_def): +def build_shipit_payload(config, task, task_def): worker = task["worker"] task_def["tags"]["worker-implementation"] = "scriptworker" @@ -117,3 +117,43 @@ def build_push_apk_payload(config, task, task_def): task_def['payload'] = { 'release_name': worker['release-name'] } + + +@payload_builder( + "scriptworker-github", + schema={ + Required("upstream-artifacts"): [ + { + Required("taskId"): taskref_or_string, + Required("taskType"): text_type, + Required("paths"): [text_type], + } + ], + Required("artifact-map"): [object], + Required("action"): text_type, + Required("git-tag"): text_type, + Required("git-revision"): text_type, + Required("github-project"): text_type, + Required("is-prerelease"): bool, + Required("release-name"): text_type, + }, +) +def build_github_release_payload(config, task, task_def): + worker = task["worker"] + + task_def["tags"]["worker-implementation"] = "scriptworker" + + task_def["payload"] = { + "artifactMap": worker["artifact-map"], + "gitTag": worker["git-tag"], + "gitRevision": worker["git-revision"], + "isPrerelease": worker["is-prerelease"], + "releaseName": worker["release-name"], + "upstreamArtifacts": worker["upstream-artifacts"], + } + + scope_prefix = config.graph_config["scriptworker"]["scope-prefix"] + task_def["scopes"].extend([ + "{}:github:project:{}".format(scope_prefix, worker["github-project"]), + "{}:github:action:{}".format(scope_prefix, worker["action"]), + ])