From 7df0006aaaf3781176d4ac17473383091ef4f2de Mon Sep 17 00:00:00 2001 From: Michael Comella Date: Tue, 9 Mar 2021 17:04:46 -0800 Subject: [PATCH] No issue: add script to measure duration to first frame. I only tested this for nightly but we can fix it for other channels as we use it. --- tools/measure_time_to_first_frame.py | 157 +++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100755 tools/measure_time_to_first_frame.py diff --git a/tools/measure_time_to_first_frame.py b/tools/measure_time_to_first_frame.py new file mode 100755 index 000000000..0056becde --- /dev/null +++ b/tools/measure_time_to_first_frame.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import argparse +import os +import subprocess +import time +from pprint import pprint + +DESC = """Measures the duration from process start until the first frame is drawn +using the "TotalTime:" field from `adb shell am start -W`. This script is a python +reimplementation of https://medium.com/androiddevelopers/testing-app-startup-performance-36169c27ee55 +with additional functionality. + +IMPORTANT: this method does not provide a complete picture of start up. Using +./mach perftest (or the deprecated FNPRMS) is the preferred approach because those +provide more comprehensive views of start up. However, this is useful for lightweight +testing if you know exactly what you're looking for. +""" + +DEFAULT_ITER_COUNT = 25 + +CHANNEL_TO_PKG = { + 'nightly': 'org.mozilla.fenix', + 'beta': 'org.mozilla.firefox.beta', + 'release': 'org.mozilla.firefox', + 'debug': 'org.mozilla.fenix.debug' +} + + +def parse_args(): + parser = argparse.ArgumentParser(description=DESC, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + "release_channel", choices=['nightly', 'beta', 'release', 'debug'], help="the release channel to measure" + ) + parser.add_argument( + "startup_type", choices=['cold_main', 'cold_view'], help="the type of start up to measure. see https://wiki.mozilla.org/Performance/Fenix#Terminology for descriptions of cold/warm/hot and main/view" + ) + parser.add_argument("path", help="the path to save the measurement results; will be overwritten") + + parser.add_argument("-c", "--iter-count", default=DEFAULT_ITER_COUNT, type=int, + help="the number of iterations to run. defaults to {}".format(DEFAULT_ITER_COUNT)) + parser.add_argument("-f", "--force", action="store_true", help="overwrite the given path rather than stopping on file existence") + return parser.parse_args() + + +def validate_args(args): + # This helps prevent us from accidentally overwriting previous measurements. + if not args.force: + if os.path.exists(args.path): + raise Exception("Given `path` unexpectedly exists: pick a new path or use --force to overwrite.") + + +def get_package_id(release_channel): + package_id = CHANNEL_TO_PKG.get(release_channel) + if not package_id: + raise Exception('this should never happen: this should be validated by argparse') + return package_id + + +def get_activity_manager_args(): + return ['adb', 'shell', 'am'] + + +def force_stop(pkg_id): + args = get_activity_manager_args() + ['force-stop', pkg_id] + subprocess.run(args, check=True) + + +def disable_startup_profiling(): + # Startup profiling sets the app to the "debug-app" which executes extra code to + # read a config file off disk that triggers the profiling. Removing the app as a + # debug app should address that issue but isn't a perfect clean up. + args = get_activity_manager_args() + ['clear-debug-app'] + subprocess.run(args, check=True) + + +def get_start_cmd(startup_type, pkg_id): + args_prefix = get_activity_manager_args() + ['start-activity', '-W', '-n'] + if startup_type == 'cold_main': + cmd = args_prefix + ['{}/.App'.format(pkg_id)] + elif startup_type == 'cold_view': + pkg_activity = '{}/org.mozilla.fenix.IntentReceiverActivity'.format(pkg_id) + cmd = args_prefix + [ + pkg_activity, + '-d', 'https://example.com', + '-a', 'android.intent.action.VIEW' + ] + else: + raise Exception('Should never happen (if argparse is set up correctly') + return cmd + + +def measure(pkg_id, start_cmd_args, iter_count): + # Startup profiling may accidentally be left enabled and throw off the results. + # To prevent this, we disable it. + disable_startup_profiling() + + # After an (re)installation, we've observed the app starts up more slowly than subsequent runs. + # As such, we start it once beforehand to let it settle. + force_stop(pkg_id) + subprocess.run(start_cmd_args, check=True, capture_output=True) # capture_output so it doesn't print to the console. + time.sleep(5) # To hopefully reach visual completeness. + + measurements = [] + for i in range(0, iter_count): + force_stop(pkg_id) + time.sleep(1) + proc = subprocess.run(start_cmd_args, check=True, capture_output=True) # expected to wait for app to start. + measurements.append(get_measurement_from_stdout(proc.stdout)) + return measurements + + +def get_measurement_from_stdout(stdout): + # Sample input: + # + # Starting: Intent { cmp=org.mozilla.fenix/.App } + # Status: ok + # Activity: org.mozilla.fenix/.App + # ThisTime: 5662 + # TotalTime: 5662 + # WaitTime: 5680 + # Complete + total_time_prefix = b'TotalTime: ' + lines = stdout.split(b'\n') + matching_lines = [line for line in lines if line.startswith(total_time_prefix)] + if len(matching_lines) != 1: + raise Exception('Each run should only have one {} but this unexpectedly had more: '.format(total_time_prefix) + + matching_lines) + duration = int(matching_lines[0][len(total_time_prefix):]) + return duration + + +def save_measurements(path, measurements): + with open(path, 'w') as f: + for measurement in measurements: + f.write(str(measurement) + '\n') + + +def main(): + args = parse_args() + validate_args(args) + + # Exceptions and script piping like these are why we prefer mozperftest. :) + print("Clear the onboarding experience manually, if it's desired and you haven't already done so.") + print("\nYou can use this script to find the average from the results file: https://github.com/mozilla-mobile/perf-tools/blob/9dd8bf1ea0ea8b2663e21d341a1572c5249c513d/average_times.py") + + pkg_id = get_package_id(args.release_channel) + start_cmd = get_start_cmd(args.startup_type, pkg_id) + measurements = measure(pkg_id, start_cmd, args.iter_count) + save_measurements(args.path, measurements) + + +if __name__ == '__main__': + main()