diff --git a/.taskcluster.yml b/.taskcluster.yml index 3e53b491c..ec40ffd89 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -104,7 +104,21 @@ tasks: TASKS_PRIORITY: ${tasks_priority} TRUST_LEVEL: ${trust_level} features: + chainOfTrust: true taskclusterProxy: true + artifacts: + public/task-graph.json: + type: file + path: /opt/fenix/task-graph.json + expires: ${expires_in} + public/actions.json: + type: file + path: /opt/fenix/actions.json + expires: ${expires_in} + public/parameters.yml: + type: file + path: /opt/fenix/parameters.yml + expires: ${expires_in} extra: tasks_for: ${tasks_for} treeherder: @@ -116,50 +130,51 @@ tasks: in: $if: 'tasks_for in ["github-pull-request", "github-push"]' then: - $let: - pr_or_push_parameters: - payload: - command: - - >- - git fetch ${repository} ${head_branch} - && git config advice.detachedHead false - && git checkout ${head_rev} - && python automation/taskcluster/decision_task.py pr-or-push - in: - - $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]' - then: - $let: - pull_request_title: ${event.pull_request.title} - pull_request_number: ${event.pull_request.number} - pull_request_url: ${event.pull_request.html_url} - in: - $mergeDeep: - - {$eval: 'default_task_definition'} - - {$eval: 'pr_or_push_parameters'} - - scopes: - - ${assume_scope_prefix}:pull-request - payload: - env: - GITHUB_PULL_TITLE: ${pull_request_title} - extra: - treeherder: - symbol: D-PR - metadata: - name: 'Fenix - Decision task (Pull Request #${pull_request_number})' - description: 'Building and testing the Fenix - triggered by [#${pull_request_number}](${pull_request_url})' - - $if: 'tasks_for == "github-push" && head_branch[:10] != "refs/tags/"' - then: + - $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]' + then: + $let: + pull_request_title: ${event.pull_request.title} + pull_request_number: ${event.pull_request.number} + pull_request_url: ${event.pull_request.html_url} + in: $mergeDeep: - {$eval: 'default_task_definition'} - - {$eval: 'pr_or_push_parameters'} - scopes: - - ${assume_scope_prefix}:branch:${short_head_branch} + - ${assume_scope_prefix}:pull-request + payload: + command: + - >- + git fetch ${repository} ${head_branch} + && git config advice.detachedHead false + && git checkout FETCH_HEAD + && python automation/taskcluster/decision_task.py pull-request + env: + GITHUB_PULL_TITLE: ${pull_request_title} extra: treeherder: - symbol: D + symbol: D-PR metadata: - name: Fenix - Decision task - description: Schedules the build and test tasks for Fenix. + name: 'Fenix - Decision task (Pull Request #${pull_request_number})' + description: 'Building and testing the Fenix - triggered by [#${pull_request_number}](${pull_request_url})' + - $if: 'tasks_for == "github-push" && head_branch[:10] != "refs/tags/"' + then: + $mergeDeep: + - {$eval: 'default_task_definition'} + - scopes: + - ${assume_scope_prefix}:branch:${short_head_branch} + payload: + command: + - >- + git fetch ${repository} ${head_branch} + && git config advice.detachedHead false + && git checkout FETCH_HEAD + && python automation/taskcluster/decision_task.py push + extra: + treeherder: + symbol: D + metadata: + name: Fenix - Decision task + description: Schedules the build and test tasks for Fenix. else: - $if: 'tasks_for == "cron"' then: @@ -172,31 +187,15 @@ tasks: routes: - notify.email.fenix-eng-notifications@mozilla.com.on-failed payload: - features: - taskclusterProxy: true - chainOfTrust: true command: - >- git fetch ${repository} ${head_branch} && git config advice.detachedHead false - && git checkout ${head_rev} + && git checkout FETCH_HEAD && python automation/taskcluster/decision_task.py \ release \ --nightly \ --track ${track} - artifacts: - public/task-graph.json: - type: file - path: /opt/fenix/task-graph.json - expires: ${expires_in} - public/actions.json: - type: file - path: /opt/fenix/actions.json - expires: ${expires_in} - public/parameters.yml: - type: file - path: /opt/fenix/parameters.yml - expires: ${expires_in} extra: cron: {$json: {$eval: 'cron'}} treeherder: diff --git a/automation/taskcluster/decision_task.py b/automation/taskcluster/decision_task.py index 798471c6f..118d567e8 100644 --- a/automation/taskcluster/decision_task.py +++ b/automation/taskcluster/decision_task.py @@ -13,7 +13,7 @@ import os import taskcluster from lib import build_variants -from lib.tasks import TaskBuilder, schedule_task_graph +from lib.tasks import TaskBuilder, schedule_task_graph, _get_architecture_and_build_type_and_product_from_variant from lib.chain_of_trust import ( populate_chain_of_trust_task_graph, populate_chain_of_trust_required_but_unused_files @@ -42,27 +42,31 @@ BUILDER = TaskBuilder( ) -def pr_or_push(): - if SKIP_TASKS_TRIGGER in PR_TITLE: +def pr_or_push(is_master_push): + if not is_master_push and SKIP_TASKS_TRIGGER in PR_TITLE: print("Pull request title contains", SKIP_TASKS_TRIGGER) print("Exit") return {} - print("Fetching build variants from gradle") - variants = build_variants.from_gradle() - - if len(variants) == 0: - raise ValueError("Could not get build variants from gradle") - - print("Got variants: {}".format(' '.join(variants))) - build_tasks = {} + signing_tasks = {} other_tasks = {} - for variant in variants: - build_tasks[taskcluster.slugId()] = BUILDER.craft_assemble_task(variant) + for variant in build_variants.from_gradle(): + assemble_task_id = taskcluster.slugId() + build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant) build_tasks[taskcluster.slugId()] = BUILDER.craft_test_task(variant) + arch, build_type, _ = _get_architecture_and_build_type_and_product_from_variant(variant) + # autophone only supports arm and aarch64, so only sign/perftest those builds + if ( + build_type == 'releaseRaptor' and + arch in ('arm', 'aarch64') and + is_master_push + ): + signing_tasks[taskcluster.slugId()] = BUILDER.craft_master_commit_signing_task(assemble_task_id, variant) + # raptor task will be added in follow-up + for craft_function in ( BUILDER.craft_detekt_task, BUILDER.craft_ktlint_task, @@ -71,12 +75,13 @@ def pr_or_push(): ): other_tasks[taskcluster.slugId()] = craft_function() - return (build_tasks, other_tasks) + return (build_tasks, signing_tasks, other_tasks) def nightly(track): is_staging = track == 'staging-nightly' architectures = ['x86', 'arm', 'aarch64'] + apk_paths = ["public/target.{}.apk".format(arch) for arch in architectures] build_tasks = {} signing_tasks = {} @@ -85,19 +90,18 @@ def nightly(track): build_task_id = taskcluster.slugId() build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(architectures, is_staging) - artifacts = ["public/target.{}.apk".format(arch) for arch in architectures] signing_task_id = taskcluster.slugId() - signing_tasks[signing_task_id] = BUILDER.craft_signing_task( + signing_tasks[signing_task_id] = BUILDER.craft_nightly_signing_task( build_task_id, - apks=artifacts, + apk_paths=apk_paths, is_staging=is_staging, ) push_task_id = taskcluster.slugId() push_tasks[push_task_id] = BUILDER.craft_push_task( signing_task_id, - apks=artifacts, - is_staging=is_staging + apks=apk_paths, + is_staging=is_staging, ) return (build_tasks, signing_tasks, push_tasks) @@ -110,7 +114,8 @@ if __name__ == "__main__": subparsers = parser.add_subparsers(dest='command') - subparsers.add_parser('pr-or-push') + subparsers.add_parser('pull-request') + subparsers.add_parser('push') release_parser = subparsers.add_parser('release') release_parser.add_argument('--nightly', action="store_true", default=False) @@ -121,9 +126,12 @@ if __name__ == "__main__": result = parser.parse_args() command = result.command + taskcluster_queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'}) - if command == 'pr-or-push': - ordered_groups_of_tasks = pr_or_push() + if command == 'pull-request': + ordered_groups_of_tasks = pr_or_push(False) + elif command == 'push': + ordered_groups_of_tasks = pr_or_push(True) elif command == 'release': ordered_groups_of_tasks = nightly(result.track) else: diff --git a/automation/taskcluster/lib/build_variants.py b/automation/taskcluster/lib/build_variants.py index e4c7ad95b..f0d3e7c34 100644 --- a/automation/taskcluster/lib/build_variants.py +++ b/automation/taskcluster/lib/build_variants.py @@ -8,6 +8,7 @@ import subprocess def from_gradle(): + print('Fetching build variants from gradle') process = subprocess.Popen([ "./gradlew", "--no-daemon", "--quiet", "printBuildVariants" ], stdout=subprocess.PIPE) @@ -21,4 +22,8 @@ def from_gradle(): variants_json = variants_line.split(' ', 1)[1] variants = json.loads(variants_json) + if len(variants) == 0: + raise RuntimeError('Expected at least one build variant from gradle') + + print("Got variants: " + ' '.join(variants)) return variants diff --git a/automation/taskcluster/lib/chain_of_trust.py b/automation/taskcluster/lib/chain_of_trust.py index 7ef50c02f..c30756fe0 100644 --- a/automation/taskcluster/lib/chain_of_trust.py +++ b/automation/taskcluster/lib/chain_of_trust.py @@ -12,7 +12,7 @@ def populate_chain_of_trust_required_but_unused_files(): def populate_chain_of_trust_task_graph(full_task_graph): - # taskgraph must follow the format: + # full_task_graph must follow the format: # { # task_id: full_task_definition # } diff --git a/automation/taskcluster/lib/tasks.py b/automation/taskcluster/lib/tasks.py index 3f7c0d5b5..d24c9478c 100644 --- a/automation/taskcluster/lib/tasks.py +++ b/automation/taskcluster/lib/tasks.py @@ -10,7 +10,7 @@ import json import os import taskcluster -from lib.util import convert_camel_case_into_kebab_case +from lib.util import convert_camel_case_into_kebab_case, lower_case_first_letter DEFAULT_EXPIRES_IN = '1 year' _OFFICIAL_REPO_URL = 'https://github.com/mozilla-mobile/fenix' @@ -22,12 +22,7 @@ class TaskBuilder(object): task_id, repo_url, git_ref, - short_head_branch, - commit, - owner, - source, - scheduler_id, - date_string, + short_head_branch, commit, owner, source, scheduler_id, date_string, tasks_priority='lowest', trust_level=1 ): @@ -39,6 +34,7 @@ class TaskBuilder(object): self.owner = owner self.source = source self.scheduler_id = scheduler_id + self.trust_level = trust_level self.tasks_priority = tasks_priority self.date = arrow.get(date_string) self.trust_level = trust_level @@ -111,7 +107,6 @@ class TaskBuilder(object): description='Building and testing variant {}'.format(variant), gradle_task='assemble{}'.format(variant.capitalize()), artifacts=_craft_artifacts_from_variant(variant), - routes=self._craft_branch_routes(variant), treeherder={ 'groupSymbol': _craft_treeherder_group_symbol_from_variant(variant), 'jobKind': 'build', @@ -139,33 +134,6 @@ class TaskBuilder(object): }, ) - def _craft_branch_routes(self, variant): - routes = [] - - if self.repo_url == _OFFICIAL_REPO_URL and self.short_head_branch == 'master': - architecture, build_type, product = \ - _get_architecture_and_build_type_and_product_from_variant(variant) - product = convert_camel_case_into_kebab_case(product) - postfix = convert_camel_case_into_kebab_case('{}-{}'.format(architecture, build_type)) - - routes = [ - 'index.project.mobile.fenix.branch.{}.revision.{}.{}.{}'.format( - self.short_head_branch, self.commit, product, postfix - ), - 'index.project.mobile.fenix.branch.{}.latest.{}.{}'.format( - self.short_head_branch, product, postfix - ), - 'index.project.mobile.fenix.branch.{}.pushdate.{}.{}.{}.revision.{}.{}.{}'.format( - self.short_head_branch, self.date.year, self.date.month, self.date.day, - self.commit, product, postfix - ), - 'index.project.mobile.fenix.branch.{}.pushdate.{}.{}.{}.latest.{}.{}'.format( - self.short_head_branch, self.date.year, self.date.month, self.date.day, - product, postfix - ), - ] - return routes - def craft_detekt_task(self): return self._craft_clean_gradle_task( name='detekt', @@ -251,14 +219,12 @@ class TaskBuilder(object): scopes = [] if scopes is None else scopes routes = [] if routes is None else routes - checkout_command = ( - "export TERM=dumb && " - "git fetch {} {} --tags && " - "git config advice.detachedHead false && " - "git checkout {}".format( - self.repo_url, self.git_ref, self.commit - ) - ) + checkout_command = ' && '.join([ + "export TERM=dumb", + "git fetch {} {}".format(self.repo_url, self.git_ref), + "git config advice.detachedHead false", + "git checkout FETCH_HEAD", + ]) command = '{} && {}'.format(checkout_command, command) @@ -293,6 +259,32 @@ class TaskBuilder(object): treeherder=treeherder, ) + def _craft_signing_task(self, name, description, signing_type, assemble_task_id, apk_paths, routes, treeherder): + signing_format = "autograph_apk" + payload = { + 'upstreamArtifacts': [{ + 'paths': apk_paths, + 'formats': [signing_format], + 'taskId': assemble_task_id, + 'taskType': 'build' + }] + } + + return self._craft_default_task_definition( + worker_type='mobile-signing-dep-v1' if signing_format == 'dep' else 'mobile-signing-v1', + provisioner_id='scriptworker-prov-v1', + dependencies=[assemble_task_id], + routes=routes, + scopes=[ + "project:mobile:fenix:releng:signing:format:{}".format(signing_format), + "project:mobile:fenix:releng:signing:cert:{}".format(signing_type), + ], + name=name, + description=description, + payload=payload, + treeherder=treeherder, + ) + def _craft_default_task_definition( self, worker_type, provisioner_id, dependencies, routes, scopes, name, description, payload, treeherder=None @@ -332,19 +324,49 @@ class TaskBuilder(object): }, } - def craft_signing_task( - self, build_task_id, apks, is_staging=True, + def craft_master_commit_signing_task( + self, assemble_task_id, variant ): - signing_format = 'autograph_apk' - payload = { - "upstreamArtifacts": [{ - "paths": apks, - "formats": [signing_format], - "taskId": build_task_id, - "taskType": "build", - }], - } + architecture, build_type, product = _get_architecture_and_build_type_and_product_from_variant(variant) + product = convert_camel_case_into_kebab_case(product) + postfix = convert_camel_case_into_kebab_case('{}-{}'.format(architecture, build_type)) + routes = [ + 'index.project.mobile.fenix.branch.master.revision.{}.{}.{}'.format( + self.commit, product, postfix + ), + 'index.project.mobile.fenix.branch.master.latest.{}.{}'.format( + product, postfix + ), + 'index.project.mobile.fenix.branch.master.pushdate.{}.{}.{}.revision.{}.{}.{}'.format( + self.date.year, self.date.month, self.date.day, self.commit, + product, postfix + ), + 'index.project.mobile.fenix.branch.master.pushdate.{}.{}.{}.latest.{}.{}'.format( + self.date.year, self.date.month, self.date.day, product, postfix + ), + ] + + return self._craft_signing_task( + name='sign: {}'.format(variant), + description='Dep-signing variant {}'.format(variant), + signing_type="dep-signing", + assemble_task_id=assemble_task_id, + apk_paths=["public/target.apk"], + routes=routes, + treeherder={ + 'groupSymbol': _craft_treeherder_group_symbol_from_variant(variant), + 'jobKind': 'other', + 'machine': { + 'platform': _craft_treeherder_platform_from_variant(variant), + }, + 'symbol': 'As', + 'tier': 1, + }, + ) + def craft_nightly_signing_task( + self, build_task_id, apk_paths, is_staging=True, + ): index_release = 'staging-signed-nightly' if is_staging else 'signed-nightly' routes = [ "index.project.mobile.fenix.{}.nightly.{}.{}.{}.latest".format( @@ -356,20 +378,13 @@ class TaskBuilder(object): "index.project.mobile.fenix.{}.nightly.latest".format(index_release), ] - return self._craft_default_task_definition( - worker_type='mobile-signing-dep-v1' if is_staging else 'mobile-signing-v1', - provisioner_id='scriptworker-prov-v1', - dependencies=[build_task_id], - routes=routes, - scopes=[ - "project:mobile:fenix:releng:signing:format:{}".format(signing_format), - "project:mobile:fenix:releng:signing:cert:{}".format( - 'dep-signing' if is_staging else 'release-signing' - ) - ], + return self._craft_signing_task( name="Signing task", description="Sign release builds of Fenix", - payload=payload, + signing_type="dep-signing" if is_staging else "release-signing", + assemble_task_id=build_task_id, + apk_paths=apk_paths, + routes=routes, treeherder={ 'jobKind': 'other', 'machine': { @@ -447,8 +462,8 @@ def _craft_apk_full_path_from_variant(variant): ) short_variant = variant[:-len(build_type)] - postfix = '-unsigned' if build_type == 'release' else '' - product = '{}{}'.format(product[0].lower(), product[1:]) + postfix = '-unsigned' if build_type.startswith('release') else '' + product = lower_case_first_letter(product) return '/opt/fenix/app/build/outputs/apk/{short_variant}/{build_type}/app-{architecture}-{product}-{build_type}{postfix}.apk'.format( # noqa: E501 architecture=architecture, @@ -479,7 +494,7 @@ def _get_architecture_and_build_type_and_product_from_variant(variant): for supported_build_type in _SUPPORTED_BUILD_TYPES: if variant.endswith(supported_build_type): - build_type = supported_build_type.lower() + build_type = lower_case_first_letter(supported_build_type) break else: raise ValueError( @@ -528,4 +543,5 @@ def schedule_task_graph(ordered_groups_of_tasks): # allows to have the full definition. This is needed to make Chain of Trust happy 'task': queue.task(task_id), } + return full_task_graph diff --git a/automation/taskcluster/lib/util.py b/automation/taskcluster/lib/util.py index 8851b1918..e887af63c 100644 --- a/automation/taskcluster/lib/util.py +++ b/automation/taskcluster/lib/util.py @@ -5,3 +5,7 @@ def convert_camel_case_into_kebab_case(string): # Inspired from https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case # noqa: E501 first_pass = re.sub('(.)([A-Z][a-z]+)', r'\1-\2', string) return re.sub('([a-z0-9])([A-Z])', r'\1-\2', first_pass).lower() + + +def lower_case_first_letter(string): + return '{}{}'.format(string[0].lower(), string[1:])