From 7d3b23ceeb12c0e3edcb36836e9041b9ca7a1800 Mon Sep 17 00:00:00 2001 From: Mihai Tabara Date: Mon, 16 Nov 2020 14:21:58 +0000 Subject: [PATCH] Bug 1635488 - add Fenix version-bump task. (#16361) Co-authored-by: Johan Lorenzo --- .taskcluster.yml | 7 +-- taskcluster/ci/config.yml | 5 ++ taskcluster/ci/version-bump/kind.yml | 39 ++++++++++++++++ taskcluster/fenix_taskgraph/parameters.py | 46 +++++++++---------- .../fenix_taskgraph/release_promotion.py | 34 +++++++++++--- .../transforms/version_bump.py | 40 ++++++++++++++++ taskcluster/fenix_taskgraph/worker_types.py | 41 ++++++++++++++++- version.txt | 2 +- 8 files changed, 180 insertions(+), 34 deletions(-) create mode 100644 taskcluster/ci/version-bump/kind.yml create mode 100644 taskcluster/fenix_taskgraph/transforms/version_bump.py diff --git a/.taskcluster.yml b/.taskcluster.yml index aaf457d5e..ce21a56a1 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -8,7 +8,7 @@ tasks: - $let: taskgraph: branch: taskgraph - revision: 12992b0f984884ec2b0a7bdedc3b3ba467363eb4 + revision: 2b2622598df02bde211d8cedb334b7b22fb883a4 trustDomain: mobile in: $let: @@ -243,7 +243,7 @@ tasks: # Note: This task is built server side without the context or tooling that # exist in tree so we must hard code the hash image: - mozillareleases/taskgraph:decision-mobile-6607973bc60e32323a541861cc5856cd6a0f51ea9fd664ef7d43bca8df53db47@sha256:8c471aacc469ea8e7bb4846c16efe086f7350a5cc1df570cc6c86b22895a2456 + mozillareleases/taskgraph:decision-mobile-682fbaa1ef17e70ddfe3457da3eaf8e776c4a20fe5bfbdbeba0641fd5bceae2a@sha256:bbb2613aaab79d17e590fbd78c072d0643be40fd1237195703f84280ecc3b302 maxRunTime: 1800 @@ -261,12 +261,13 @@ tasks: $if: 'tasks_for == "action"' then: > PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph && + PIP_IGNORE_INSTALLED=0 pip install --user mozilla-version && taskcluster/scripts/decision-install-sdk.sh && ln -s /builds/worker/artifacts artifacts && ~/.local/bin/taskgraph action-callback else: > PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph && - PIP_IGNORE_INSTALLED=0 pip install --user arrow taskcluster pyyaml && + PIP_IGNORE_INSTALLED=0 pip install --user mozilla-version && taskcluster/scripts/decision-install-sdk.sh && ln -s /builds/worker/artifacts artifacts && ~/.local/bin/taskgraph decision diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index f31059ac2..2f2caf9d8 100644 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -78,6 +78,11 @@ workers: by-level: "3": mobile-3-signing default: mobile-t-signing + tree: + provisioner: scriptworker-k8s + implementation: scriptworker-tree + os: scriptworker + worker-type: 'mobile-{level}-tree' t-bitbar.*: provisioner: proj-autophone implementation: generic-worker diff --git a/taskcluster/ci/version-bump/kind.yml b/taskcluster/ci/version-bump/kind.yml new file mode 100644 index 000000000..c7c7a3511 --- /dev/null +++ b/taskcluster/ci/version-bump/kind.yml @@ -0,0 +1,39 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: fenix_taskgraph.loader.multi_dep:loader + +transforms: + - fenix_taskgraph.transforms.multi_dep:transforms + - fenix_taskgraph.transforms.version_bump:transforms + - taskgraph.transforms.task:transforms + +kind-dependencies: + - push-apk + +primary-dependency: push-apk + +group-by: build-type + +only-for-build-types: + - beta + - release + +job-template: + description: Fenix version bump/tag + worker-type: tree + worker: + bump: true + bump-files: ["version.txt"] + push: + by-level: + "3": true + # If you set the following line to true, you need to grant write access + # to https://github.com/mozilla-release-automation-bot-staging on your + # fork. + default: false + + treeherder: + job-symbol: vb + kind: build diff --git a/taskcluster/fenix_taskgraph/parameters.py b/taskcluster/fenix_taskgraph/parameters.py index 1a1bbb9b3..670ed6e00 100644 --- a/taskcluster/fenix_taskgraph/parameters.py +++ b/taskcluster/fenix_taskgraph/parameters.py @@ -7,49 +7,49 @@ from __future__ import absolute_import, print_function, unicode_literals import os import re +from mozilla_version.fenix import FenixVersion from six import text_type from taskgraph.parameters import extend_parameters_schema from voluptuous import All, Any, Optional, Range, Required - -BETA_SEMVER = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$') -RELEASE_SEMVER = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$') - +from .release_promotion import read_version_file extend_parameters_schema({ Required("pull_request_number"): Any(All(int, Range(min=1)), None), Required("release_type"): text_type, Optional("shipping_phase"): Any('build', 'ship', None), Required("version"): text_type, + Required("next_version"): Any(None, text_type), }) def get_decision_parameters(graph_config, parameters): + parameters.setdefault("release_type", "") head_tag = parameters["head_tag"].decode("utf-8") - parameters["release_type"] = resolve_release_type(head_tag) parameters["version"] = head_tag[1:] if head_tag else "" pr_number = os.environ.get("MOBILE_PULL_REQUEST_NUMBER", None) parameters["pull_request_number"] = None if pr_number is None else int(pr_number) + parameters.setdefault("next_version", None) + # TODO: Remove this block once github-releases are not supported anymore if parameters["tasks_for"] == "github-release": - for param_name in ("release_type", "version"): - if not parameters[param_name]: - raise ValueError( - 'Cannot run github-release if "{}" is not defined. Got: {}'.format( - param_name, parameters[param_name] - ) - ) parameters["target_tasks_method"] = "release" - -def resolve_release_type(head_tag): - if not head_tag: - return "" - elif BETA_SEMVER.match(head_tag): - return "beta" - elif RELEASE_SEMVER.match(head_tag): - return "release" - else: - raise ValueError('Github tag must be in semver format and prefixed with a "v", ' - 'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (release) or "v1.0.0" (release)') + version_string = parameters["version"] + version_in_file = read_version_file() + if version_string != version_in_file: + raise ValueError("Version given in tag ({}) does not match the one in version.txt ({})".format(version_string, version_in_file)) + + version = FenixVersion.parse(version_string) + if version.is_beta: + next_version = version.bump("beta_number") + release_type = "beta" + elif version.is_release: + next_version = version.bump("patch_number") + release_type = "release" + else: + raise ValueError("Unsupported version type: {}".format(version.version_type)) + + parameters["next_version"] = str(next_version).decode("utf-8") + parameters["release_type"] = release_type diff --git a/taskcluster/fenix_taskgraph/release_promotion.py b/taskcluster/fenix_taskgraph/release_promotion.py index 9a34417a7..ce668718b 100644 --- a/taskcluster/fenix_taskgraph/release_promotion.py +++ b/taskcluster/fenix_taskgraph/release_promotion.py @@ -4,6 +4,9 @@ from __future__ import absolute_import, print_function, unicode_literals +import os + +from mozilla_version.fenix import FenixVersion from taskgraph.actions.registry import register_callback_action from taskgraph.util.taskcluster import get_artifact @@ -12,10 +15,8 @@ from taskgraph.decision import taskgraph_decision from taskgraph.parameters import Parameters from taskgraph.util.taskgraph import find_decision_task, find_existing_tasks_from_previous_kinds -from .parameters import resolve_release_type - RELEASE_PROMOTION_PROJECTS = ( - "https://github.com/JohanLorenzo/fenix", + "https://github.com/mozilla-mobile/fenix", ) @@ -89,8 +90,15 @@ def is_release_promotion_available(parameters): 'relying on the in-tree version will break things.'), 'default': '', }, + "next_version": { + "type": "string", + "description": ( + "Next version.", + ), + "default": "", + }, }, - "required": ['release_promotion_flavor', 'version', 'build_number'], + "required": ["release_promotion_flavor", "version", "build_number", "next_version"], } ) def release_promotion_action(parameters, graph_config, input, task_group_id, task_id): @@ -133,9 +141,23 @@ def release_promotion_action(parameters, graph_config, input, task_group_id, tas parameters['optimize_target_tasks'] = True parameters['shipping_phase'] = input['release_promotion_flavor'] + version_in_file = read_version_file() parameters['version'] = input['version'] if input.get('version') else read_version_file() - parameters['head_tag'] = 'v{}'.format(parameters['version']) - parameters['release_type'] = resolve_release_type(parameters['head_tag']) + version_string = parameters['version'] + if version_string != version_in_file: + raise ValueError("Version given in tag ({}) does not match the one in version.txt ({})".format(version_string, version_in_file)) + parameters['head_tag'] = 'v{}'.format(version_string) + + parameters['next_version'] = input['next_version'] + + version = FenixVersion.parse(version_string) + if version.is_beta: + release_type = "beta" + elif version.is_release: + release_type = "release" + else: + raise ValueError("Unsupported version type: {}".format(version.version_type)) + parameters['release_type'] = release_type parameters['pull_request_number'] = None diff --git a/taskcluster/fenix_taskgraph/transforms/version_bump.py b/taskcluster/fenix_taskgraph/transforms/version_bump.py new file mode 100644 index 000000000..9e380e2fe --- /dev/null +++ b/taskcluster/fenix_taskgraph/transforms/version_bump.py @@ -0,0 +1,40 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the jobs defined in the version bump kind +kind. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + for key in ("worker.push",): + resolve_keyed_by( + task, + key, + item_name=task["name"], + **{ + 'build-type': task["attributes"]["build-type"], + 'level': config.params["level"], + } + ) + yield task + + +@transforms.add +def build_worker_definition(config, tasks): + for task in tasks: + worker_definition = {} + task["worker"].update(worker_definition) + + yield task diff --git a/taskcluster/fenix_taskgraph/worker_types.py b/taskcluster/fenix_taskgraph/worker_types.py index 1a8073e71..920037728 100644 --- a/taskcluster/fenix_taskgraph/worker_types.py +++ b/taskcluster/fenix_taskgraph/worker_types.py @@ -6,7 +6,7 @@ from __future__ import absolute_import, print_function, unicode_literals from six import text_type -from voluptuous import Required, Optional +from voluptuous import Any, Required, Optional from taskgraph.util.schema import taskref_or_string from taskgraph.transforms.task import payload_builder @@ -157,3 +157,42 @@ def build_github_release_payload(config, task, task_def): "{}:github:project:{}".format(scope_prefix, worker["github-project"]), "{}:github:action:{}".format(scope_prefix, worker["action"]), ]) + + +@payload_builder( + "scriptworker-tree", + schema={ + Optional("upstream-artifacts"): [ + { + Optional("taskId"): taskref_or_string, + Optional("taskType"): text_type, + Optional("paths"): [text_type], + } + ], + Required("bump"): bool, + Optional("bump-files"): [text_type], + Optional("push"): bool, + }, +) +def build_version_bump_payload(config, task, task_def): + worker = task["worker"] + task_def["tags"]["worker-implementation"] = "scriptworker" + + task_def['payload'] = {'actions': []} + actions = task_def['payload']['actions'] + + if worker['bump']: + if not worker['bump-files']: + raise Exception("Version Bump requested without bump-files") + + bump_info = {} + bump_info["next_version"] = config.params["next_version"] + bump_info['files'] = worker['bump-files'] + task_def['payload']['version_bump_info'] = bump_info + actions.append('version_bump') + + if worker["push"]: + task_def['payload']['push'] = True + + if worker.get('force-dry-run'): + task_def['payload']['dry_run'] = True diff --git a/version.txt b/version.txt index 2f9037bc3..af2d9c332 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -4.0.0-beta.2 +84.0.0-beta.1