For #19801 - Run taskgraph generation with Python 3 (#19802)

* For #19801 - Update to latest version of taskgraph

* For #19801 - Update to latest decision-mobile image

We need to explicitly set the Java version as Java 11 now co-exists in
the decision-mobile image.

* For #19801 - Run decision task with Python 3

.taskcluster.yml

@@ -8,7 +8,7 @@ tasks:
     - $let:
           taskgraph:
               branch: taskgraph
-              revision: a458418ef7cdd6778f1283926c6116966255bc24
+              revision: 9daff451cfbe82c5c70237a7b3dbcf4fd3238299
           trustDomain: mobile
       in:
           $let:
@@ -243,7 +243,7 @@ tasks:
                               # Note: This task is built server side without the context or tooling that
                               # exist in tree so we must hard code the hash
                               image:
-                                  mozillareleases/taskgraph:decision-mobile-682fbaa1ef17e70ddfe3457da3eaf8e776c4a20fe5bfbdbeba0641fd5bceae2a@sha256:bbb2613aaab79d17e590fbd78c072d0643be40fd1237195703f84280ecc3b302
+                                  mozillareleases/taskgraph:decision-mobile-44b6b7b4c370220eff56efa8b508aa5157ef9c6e74847c7ecc19d640946ba49e@sha256:4107cbc5e154502529e4d38efa4dc89c05ee54e2cbc6e2e66023e68407502894
 
                               maxRunTime: 1800
 
@@ -260,14 +260,14 @@ tasks:
                                     in:
                                         $if: 'tasks_for == "action"'
                                         then: >
-                                            PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph &&
+                                            PIP_IGNORE_INSTALLED=0 pip3 install --user /builds/worker/checkouts/taskgraph &&
-                                            PIP_IGNORE_INSTALLED=0 pip install --user mozilla-version &&
+                                            PIP_IGNORE_INSTALLED=0 pip3 install --user mozilla-version &&
                                             taskcluster/scripts/decision-install-sdk.sh &&
                                             ln -s /builds/worker/artifacts artifacts &&
                                             ~/.local/bin/taskgraph action-callback
                                         else: >
-                                            PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph &&
+                                            PIP_IGNORE_INSTALLED=0 pip3 install --user /builds/worker/checkouts/taskgraph &&
-                                            PIP_IGNORE_INSTALLED=0 pip install --user mozilla-version &&
+                                            PIP_IGNORE_INSTALLED=0 pip3 install --user mozilla-version &&
                                             taskcluster/scripts/decision-install-sdk.sh &&
                                             ln -s /builds/worker/artifacts artifacts &&
                                             ~/.local/bin/taskgraph decision

taskcluster/fenix_taskgraph/gradle.py

@@ -40,14 +40,14 @@ def _fetch_all_variants():
 def _run_gradle_process(gradle_command, **kwargs):
     gradle_properties = [
         '-P{property_name}={value}'.format(property_name=property_name, value=value)
-        for property_name, value in kwargs.iteritems()
+        for property_name, value in kwargs.items()
     ]
 
-    process = subprocess.Popen(["./gradlew", "--no-daemon", "--quiet", gradle_command] + gradle_properties, stdout=subprocess.PIPE)
+    process = subprocess.Popen(["./gradlew", "--no-daemon", "--quiet", gradle_command] + gradle_properties, stdout=subprocess.PIPE, universal_newlines=True)
     output, err = process.communicate()
     exit_code = process.wait()
 
-    if exit_code is not 0:
+    if exit_code != 0:
         raise RuntimeError("Gradle command returned error: {}".format(exit_code))
 
     return output

taskcluster/fenix_taskgraph/loader/__init__.py

@@ -24,7 +24,7 @@ def group_tasks(config, tasks):
 
     groups = group_by_fn(config, tasks)
 
-    for combinations in groups.itervalues():
+    for combinations in groups.values():
         dependencies = [copy.deepcopy(t) for t in combinations]
         yield dependencies
 

taskcluster/fenix_taskgraph/loader/multi_dep.py

@@ -20,7 +20,7 @@ schema = Schema({
     Required(
         'dependent-tasks',
         'dictionary of dependent tasks, keyed by kind',
-    ): {basestring: Task},
+    ): {str: Task},
 })
 
 
@@ -66,7 +66,7 @@ def get_primary_dep(config, dep_tasks):
     is the primary dependency. If it's undefined, return the first dep.
     """
     primary_dependencies = config.get('primary-dependency')
-    if isinstance(primary_dependencies, basestring):
+    if isinstance(primary_dependencies, str):
         primary_dependencies = [primary_dependencies]
     if not primary_dependencies:
         assert len(dep_tasks) == 1, "Must define a primary-dependency!"

taskcluster/fenix_taskgraph/parameters.py

@@ -25,7 +25,7 @@ extend_parameters_schema({
 
 def get_decision_parameters(graph_config, parameters):
     parameters.setdefault("release_type", "")
-    head_tag = parameters["head_tag"].decode("utf-8")
+    head_tag = parameters["head_tag"]
     parameters["version"] = head_tag[1:] if head_tag else ""
 
     pr_number = os.environ.get("MOBILE_PULL_REQUEST_NUMBER", None)
@@ -54,5 +54,5 @@ def get_decision_parameters(graph_config, parameters):
         else:
             raise ValueError("Unsupported version type: {}".format(version.version_type))
 
-        parameters["next_version"] = str(next_version).decode("utf-8")
+        parameters["next_version"] = str(next_version)
         parameters["release_type"] = release_type

taskcluster/fenix_taskgraph/target_tasks.py

@@ -20,7 +20,7 @@ def target_tasks_default(full_task_graph, parameters, graph_config):
 
         return task.attributes.get("release-type", "") == parameters["release_type"]
 
-    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
+    return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)]
 
 
 @_target_task("nightly")
@@ -32,7 +32,7 @@ def target_tasks_nightly(full_task_graph, parameters, graph_config):
         # See bug 1628413 for more context.
         return task.attributes.get("nightly", False) and task.kind != "push-apk"
 
-    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
+    return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)]
 
 
 @_target_task("nightly-on-google-play")
@@ -48,7 +48,7 @@ def target_tasks_nightly_on_google_play(full_task_graph, parameters, graph_confi
             task.kind not in ("browsertime", "visual-metrics", "raptor")
         )
 
-    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
+    return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)]
 
 
 def _filter_fennec(fennec_type, task, parameters):
@@ -59,7 +59,7 @@ def _filter_fennec(fennec_type, task, parameters):
 def target_tasks_fennec_nightly(full_task_graph, parameters, graph_config):
     """Select the set of tasks required for a production build signed with the fennec key."""
 
-    return [l for l, t in full_task_graph.tasks.iteritems() if _filter_fennec("production", t, parameters)]
+    return [l for l, t in full_task_graph.tasks.items() if _filter_fennec("production", t, parameters)]
 
 
 @_target_task("bump_android_components")
@@ -69,7 +69,7 @@ def target_tasks_bump_android_components(full_task_graph, parameters, graph_conf
     def filter(task, parameters):
         return task.attributes.get("bump-type", "") == "android-components"
 
-    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
+    return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)]
 
 
 @_target_task("screenshots")
@@ -79,4 +79,4 @@ def target_tasks_screnshots(full_task_graph, parameters, graph_config):
     def filter(task, parameters):
         return task.attributes.get("screenshots", False)
 
-    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
+    return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)]

taskcluster/fenix_taskgraph/transforms/github_release.py

@@ -36,8 +36,8 @@ def build_worker_definition(config, tasks):
     for task in tasks:
         worker_definition = {
             "artifact-map": _build_artifact_map(task),
-            "git-tag": config.params["head_tag"].decode("utf-8"),
+            "git-tag": config.params["head_tag"],
-            "git-revision": config.params["head_rev"].decode("utf-8"),
+            "git-revision": config.params["head_rev"],
             "release-name": task["worker"]["release-name"].format(version=config.params["version"]),
         }
 

taskcluster/fenix_taskgraph/transforms/multi_dep.py

@@ -20,7 +20,7 @@ def build_name_and_attributes(config, tasks):
     for task in tasks:
         task["dependencies"] = {
             dep_key: dep.label
-            for dep_key, dep in _get_all_deps(task).iteritems()
+            for dep_key, dep in _get_all_deps(task).items()
         }
         primary_dep = task["primary-dependency"]
         copy_of_attributes = primary_dep.attributes.copy()

taskcluster/fenix_taskgraph/transforms/version_bump.py

@@ -35,7 +35,7 @@ def resolve_keys(config, tasks):
 def build_worker_definition(config, tasks):
     for task in tasks:
         worker_definition = {
-            "branch": config.params["head_ref"].decode("utf-8"),
+            "branch": config.params["head_ref"],
         }
         task["worker"].update(worker_definition)
 

taskcluster/scripts/decision-install-sdk.sh

@@ -12,4 +12,6 @@ $CURL --output "$SDK_ZIP_LOCATION" "https://dl.google.com/android/repository/sdk
 echo "$ANDROID_SDK_SHA256  $SDK_ZIP_LOCATION" | sha256sum --check
 unzip -d "$ANDROID_SDK_ROOT" "$SDK_ZIP_LOCATION"
 rm "$SDK_ZIP_LOCATION"
+
+export JAVA_HOME="/usr/lib/jvm/java-1.8.0-openjdk-amd64"
 yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses