Update taskcluster settings to use short names instead of indexes

taskcluster/ci/browsertime/kind.yml

@@ -83,7 +83,6 @@ job-defaults:
             - '--cfg=mozharness/configs/raptor/android_hw_config.py'
             - '--app=fenix'
             - '--browsertime'
-            - '--cold'
             - '--no-conditioned-profile'
             - '--binary=org.mozilla.fenix'
             - '--activity=org.mozilla.fenix.IntentReceiverActivity'
@@ -101,168 +100,54 @@ job-defaults:
             - linux64-node
 
 jobs:
-    tp6m-1-cold:
+    tp6m:
-        test-name: amazon
+        page-load-tests:
-        treeherder:
+            - amazon
-            symbol: 'Btime(tp6m-1-c)'
+            - instagram
-
+            - [bing-search-restaurants, bing-s-r]
-    tp6m-2-cold:
+            - [ebay-kleinanzeigen-search, ebay-k-s]
-        test-name: google
+            - wikipedia
-        web-render-only: False
+            - booking
-        treeherder:
+            - [cnn-ampstories, cnn-amp]
-            symbol: 'Btime(tp6m-2-c)'
+            - bbc
-
+            - imdb
-    tp6m-3-cold:
+            - [facebook-cristiano, fb-cris]
-        test-name: instagram
+            - youtube
-        treeherder:
+            - bing
-            symbol: 'Btime(tp6m-3-c)'
+            - [ebay-kleinanzeigen, ebay-k]
-
+            - [google-maps, gmaps]
-    tp6m-4-cold:
+            - reddit
-        test-name: bing-search-restaurants
+            - [stackoverflow, stacko]
-        treeherder:
+            - jianshu
-            symbol: 'Btime(tp6m-4-c)'
+            - web-de
-
+            - cnn
-    tp6m-5-cold:
+            - [google-search-restaurants, gsearch-r]
-        test-name: ebay-kleinanzeigen-search
+
-        treeherder:
+    tp6m-hv:
-            symbol: 'Btime(tp6m-5-c)'
-
-    tp6m-6-cold:
-        test-name: amazon-search
-        web-render-only: False
-        treeherder:
-            symbol: 'Btime(tp6m-6-c)'
-
-    tp6m-7-cold:
-        test-name: wikipedia
-        treeherder:
-            symbol: 'Btime(tp6m-7-c)'
-
-    tp6m-8-cold:
-        test-name: booking
-        treeherder:
-            symbol: 'Btime(tp6m-8-c)'
-
-    tp6m-9-cold:
-        test-name: cnn-ampstories
-        treeherder:
-            symbol: 'Btime(tp6m-9-c)'
-
-    tp6m-10-cold:
-        test-name: bbc
-        treeherder:
-            symbol: 'Btime(tp6m-10-c)'
-
-    tp6m-11-cold:
-        test-name: microsoft-support
-        web-render-only: False
-        treeherder:
-            symbol: 'Btime(tp6m-11-c)'
-
-    tp6m-12-cold:
-        test-name: imdb
-        treeherder:
-            symbol: 'Btime(tp6m-12-c)'
-
-    tp6m-13-cold:
-        test-name: espn
-        web-render-only: False
-        treeherder:
-            symbol: 'Btime(tp6m-13-c)'
-
-    tp6m-14-cold:
-        test-name: facebook-cristiano
-        treeherder:
-            symbol: 'Btime(tp6m-14-c)'
-
-    tp6m-15-cold:
-        test-name: facebook
-        web-render-only: False
-        treeherder:
-            symbol: 'Btime(tp6m-15-c)'
-
-    tp6m-16-cold:
-        test-name: youtube
-        treeherder:
-            symbol: 'Btime(tp6m-16-c)'
-
-    tp6m-17-cold:
-        test-name: bing
-        treeherder:
-            symbol: 'Btime(tp6m-17-c)'
-
-    tp6m-18-cold:
-        test-name: ebay-kleinanzeigen
-        treeherder:
-            symbol: 'Btime(tp6m-18-c)'
-
-    tp6m-19-cold:
-        test-name: google-maps
-        treeherder:
-            symbol: 'Btime(tp6m-19-c)'
-
-    tp6m-20-cold:
-        test-name: youtube-watch
         web-render-only: False
-        treeherder:
+        page-load-tests:
-            symbol: 'Btime(tp6m-20-c)'
+            - google
-
+            - [amazon-search, amazon-s]
-    tp6m-21-cold:
+            - [microsoft-support, micros-sup]
-        test-name: reddit
+            - espn
-        treeherder:
+            - facebook
-            symbol: 'Btime(tp6m-21-c)'
+            - [youtube-watch, youtube-w]
-
+            - allrecipes
-    tp6m-22-cold:
-        test-name: stackoverflow
-        treeherder:
-            symbol: 'Btime(tp6m-22-c)'
-
-    tp6m-23-cold:
-        test-name: jianshu
-        treeherder:
-            symbol: 'Btime(tp6m-23-c)'
-
-    tp6m-24-cold:
-        test-name: allrecipes
-        web-render-only: False
-        treeherder:
-            symbol: 'Btime(tp6m-24-c)'
-
-    tp6m-25-cold:
-        test-name: web-de
-        treeherder:
-            symbol: 'Btime(tp6m-25-c)'
-
-    tp6m-27-cold:
-        test-name: cnn
-        treeherder:
-            symbol: 'Btime(tp6m-27-c)'
-
-    tp6m-28-cold:
-        test-name: google-search-restaurants
-        treeherder:
-            symbol: 'Btime(tp6m-28-c)'
 
     youtube-playback-av1-sfr:
         description: "Raptor YouTube Playback AV1 SFR on Fenix"
         test-name: youtube-playback-av1-sfr
         run-visual-metrics: False
         chimera: False
-        treeherder:
-            symbol: 'Btime(ytp-av1-sfr)'
 
     youtube-playback-h264-sfr:
         description: "Raptor YouTube Playback H264 SFR on Fenix"
         test-name: youtube-playback-h264-sfr
         run-visual-metrics: False
         chimera: False
-        treeherder:
-            symbol: Btime(ytp-h264-sfr)
 
     youtube-playback-vp9-sfr:
         description: "Raptor YouTube Playback VP9 SFR on Fenix"
         run-visual-metrics: False
         chimera: False
         test-name: youtube-playback-vp9-sfr
-        treeherder:
-            symbol: Btime(ytp-vp9-sfr)

taskcluster/fenix_taskgraph/transforms/browsertime.py

@@ -11,6 +11,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 import copy
 import json
 
+from copy import deepcopy
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.treeherder import inherit_treeherder_from_dep
 from taskgraph.util.schema import resolve_keyed_by
@@ -18,6 +19,28 @@ from taskgraph.util.schema import resolve_keyed_by
 transforms = TransformSequence()
 
 
+@transforms.add
+def split_raptor_subtests(config, tests):
+    for test in tests:
+        # For tests that have 'page-load-tests' listed, we want to create a separate
+        # test job for every subtest (i.e. split out each page-load URL into its own job)
+        subtests = test.pop("page-load-tests", None)
+        if not subtests:
+            yield test
+            continue
+
+        for subtest in subtests:
+            pageload_test = deepcopy(test)
+
+            if isinstance(subtest, list):
+                pageload_test["test-name"] = subtest[0]
+                pageload_test["subtest-symbol"] = subtest[1]
+            else:
+                pageload_test["test-name"] = subtest
+                pageload_test["subtest-symbol"] = subtest
+            yield pageload_test
+
+
 @transforms.add
 def add_variants(config, tasks):
     only_types = config.config["only-for-build-types"]
@@ -26,7 +49,7 @@ def add_variants(config, tasks):
     tests = list(tasks)
 
     for dep_task in config.kind_dependencies_tasks:
-        build_type = dep_task.attributes.get("build-type", '')
+        build_type = dep_task.attributes.get("build-type", "")
         if build_type not in only_types:
             continue
 
@@ -91,6 +114,22 @@ def build_browsertime_task(config, tasks):
         task["run"]["command"].append("--test={}".format(test_name))
         task["run"]["command"].extend(task.pop("args", []))
 
+        # Setup treherder symbol
+        symbol = task.pop("subtest-symbol", None)
+
+        # taskcluster is merging task attributes with the default ones
+        # resulting the --cold extra option in the ytp warm tasks
+        if "youtube-playback" in task["name"]:
+            symbol = test_name.replace("youtube-playback-", "ytp-")
+
+        # Setup chimera for combined warm+cold testing
+        if task.pop("chimera", False):
+            task["run"]["command"].append("--chimera")
+
+        # Add '-c' to taskcluster symbol when running cold tests
+        elif "--cold" in task["run"]["command"]:
+            symbol += "-c"
+
         # Setup visual metrics
         run_visual_metrics = task.pop("run-visual-metrics", False)
         if run_visual_metrics:
@@ -98,15 +137,11 @@ def build_browsertime_task(config, tasks):
             task["run"]["command"].append("--browsertime-no-ffwindowrecorder")
             task["attributes"]["run-visual-metrics"] = True
 
-        # Setup chimera for combined warm+cold testing
+        # Build taskcluster group and symol
-        if task.pop("chimera", False):
+        task["treeherder"]["symbol"] = "Btime(%s)" % symbol
-            task["run"]["command"].append("--chimera")
+        task["name"] = (
-
+            task["name"].replace("tp6m-", "tp6m-{}-".format(symbol)).replace("-hv", "")
-        # taskcluster is merging task attributes with the default ones
+        )
-        # resulting the --cold extra option in the ytp warm tasks
-        if 'youtube-playback' in task["name"]:
-            task["run"]["command"].remove("--cold")
-
         yield task
 
 
@@ -129,7 +164,7 @@ def enable_webrender(config, tasks):
 
 @transforms.add
 def fill_email_data(config, tasks):
-    product_name = config.graph_config['taskgraph']['repositories']['mobile']['name']
+    product_name = config.graph_config["taskgraph"]["repositories"]["mobile"]["name"]
     format_kwargs = {
         "product_name": product_name.lower(),
         "head_rev": config.params["head_rev"],
@@ -138,7 +173,9 @@ def fill_email_data(config, tasks):
     for task in tasks:
         format_kwargs["task_name"] = task["name"]
 
-        resolve_keyed_by(task, 'notify', item_name=task["name"], level=config.params["level"])
+        resolve_keyed_by(
+            task, "notify", item_name=task["name"], level=config.params["level"]
+        )
         email = task["notify"].get("email")
         if email:
             email["link"]["href"] = email["link"]["href"].format(**format_kwargs)