No issue: add script to measure duration to first frame.

I only tested this for nightly but we can fix it for other channels as
we use it.

tools/measure_time_to_first_frame.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+import argparse
+import os
+import subprocess
+import time
+from pprint import pprint
+
+DESC = """Measures the duration from process start until the first frame is drawn
+using the "TotalTime:" field from `adb shell am start -W`. This script is a python
+reimplementation of https://medium.com/androiddevelopers/testing-app-startup-performance-36169c27ee55
+with additional functionality.
+
+IMPORTANT: this method does not provide a complete picture of start up. Using
+./mach perftest (or the deprecated FNPRMS) is the preferred approach because those
+provide more comprehensive views of start up. However, this is useful for lightweight
+testing if you know exactly what you're looking for.
+"""
+
+DEFAULT_ITER_COUNT = 25
+
+CHANNEL_TO_PKG = {
+    'nightly': 'org.mozilla.fenix',
+    'beta': 'org.mozilla.firefox.beta',
+    'release': 'org.mozilla.firefox',
+    'debug': 'org.mozilla.fenix.debug'
+}
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description=DESC, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument(
+        "release_channel", choices=['nightly', 'beta', 'release', 'debug'], help="the release channel to measure"
+    )
+    parser.add_argument(
+        "startup_type", choices=['cold_main', 'cold_view'], help="the type of start up to measure. see https://wiki.mozilla.org/Performance/Fenix#Terminology for descriptions of cold/warm/hot and main/view"
+    )
+    parser.add_argument("path", help="the path to save the measurement results; will be overwritten")
+
+    parser.add_argument("-c", "--iter-count", default=DEFAULT_ITER_COUNT, type=int,
+                        help="the number of iterations to run. defaults to {}".format(DEFAULT_ITER_COUNT))
+    parser.add_argument("-f", "--force", action="store_true", help="overwrite the given path rather than stopping on file existence")
+    return parser.parse_args()
+
+
+def validate_args(args):
+    # This helps prevent us from accidentally overwriting previous measurements.
+    if not args.force:
+        if os.path.exists(args.path):
+            raise Exception("Given `path` unexpectedly exists: pick a new path or use --force to overwrite.")
+
+
+def get_package_id(release_channel):
+    package_id = CHANNEL_TO_PKG.get(release_channel)
+    if not package_id:
+        raise Exception('this should never happen: this should be validated by argparse')
+    return package_id
+
+
+def get_activity_manager_args():
+    return ['adb', 'shell', 'am']
+
+
+def force_stop(pkg_id):
+    args = get_activity_manager_args() + ['force-stop', pkg_id]
+    subprocess.run(args, check=True)
+
+
+def disable_startup_profiling():
+    # Startup profiling sets the app to the "debug-app" which executes extra code to
+    # read a config file off disk that triggers the profiling. Removing the app as a
+    # debug app should address that issue but isn't a perfect clean up.
+    args = get_activity_manager_args() + ['clear-debug-app']
+    subprocess.run(args, check=True)
+
+
+def get_start_cmd(startup_type, pkg_id):
+    args_prefix = get_activity_manager_args() + ['start-activity', '-W', '-n']
+    if startup_type == 'cold_main':
+        cmd = args_prefix + ['{}/.App'.format(pkg_id)]
+    elif startup_type == 'cold_view':
+        pkg_activity = '{}/org.mozilla.fenix.IntentReceiverActivity'.format(pkg_id)
+        cmd = args_prefix + [
+            pkg_activity,
+            '-d', 'https://example.com',
+            '-a', 'android.intent.action.VIEW'
+        ]
+    else:
+        raise Exception('Should never happen (if argparse is set up correctly')
+    return cmd
+
+
+def measure(pkg_id, start_cmd_args, iter_count):
+    # Startup profiling may accidentally be left enabled and throw off the results.
+    # To prevent this, we disable it.
+    disable_startup_profiling()
+
+    # After an (re)installation, we've observed the app starts up more slowly than subsequent runs.
+    # As such, we start it once beforehand to let it settle.
+    force_stop(pkg_id)
+    subprocess.run(start_cmd_args, check=True, capture_output=True)  # capture_output so it doesn't print to the console.
+    time.sleep(5)  # To hopefully reach visual completeness.
+
+    measurements = []
+    for i in range(0, iter_count):
+        force_stop(pkg_id)
+        time.sleep(1)
+        proc = subprocess.run(start_cmd_args, check=True, capture_output=True)  # expected to wait for app to start.
+        measurements.append(get_measurement_from_stdout(proc.stdout))
+    return measurements
+
+
+def get_measurement_from_stdout(stdout):
+    # Sample input:
+    #
+    # Starting: Intent { cmp=org.mozilla.fenix/.App }
+    # Status: ok
+    # Activity: org.mozilla.fenix/.App
+    # ThisTime: 5662
+    # TotalTime: 5662
+    # WaitTime: 5680
+    # Complete
+    total_time_prefix = b'TotalTime: '
+    lines = stdout.split(b'\n')
+    matching_lines = [line for line in lines if line.startswith(total_time_prefix)]
+    if len(matching_lines) != 1:
+        raise Exception('Each run should only have one {} but this unexpectedly had more: '.format(total_time_prefix) +
+                        matching_lines)
+    duration = int(matching_lines[0][len(total_time_prefix):])
+    return duration
+
+
+def save_measurements(path, measurements):
+    with open(path, 'w') as f:
+        for measurement in measurements:
+            f.write(str(measurement) + '\n')
+
+
+def main():
+    args = parse_args()
+    validate_args(args)
+
+    # Exceptions and script piping like these are why we prefer mozperftest. :)
+    print("Clear the onboarding experience manually, if it's desired and you haven't already done so.")
+    print("\nYou can use this script to find the average from the results file: https://github.com/mozilla-mobile/perf-tools/blob/9dd8bf1ea0ea8b2663e21d341a1572c5249c513d/average_times.py")
+
+    pkg_id = get_package_id(args.release_channel)
+    start_cmd = get_start_cmd(args.startup_type, pkg_id)
+    measurements = measure(pkg_id, start_cmd, args.iter_count)
+    save_measurements(args.path, measurements)
+
+
+if __name__ == '__main__':
+    main()