From e0c1c51b6a150f10e17a64e1eb2e1807cd109206 Mon Sep 17 00:00:00 2001 From: Michael Comella Date: Wed, 21 Jul 2021 16:54:50 -0700 Subject: [PATCH] For perf-frontend-issues#211: support focus in measure_start_up.py. --- tools/measure_start_up.py | 58 ++++++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/tools/measure_start_up.py b/tools/measure_start_up.py index 1db90f7e1..24e21eaf1 100755 --- a/tools/measure_start_up.py +++ b/tools/measure_start_up.py @@ -23,7 +23,14 @@ we try to keep this simple and avoid implementing all the things they do. DEFAULT_ITER_COUNT = 25 -CHANNEL_TO_PKG = { +FOCUS_CHANNEL_TO_PKG = { + 'nightly': 'org.mozilla.focus', # it seems problematic that this is the same as release. + 'beta': 'org.mozilla.focus.beta', # only present since post-fenix update. + 'release': 'org.mozilla.focus', + 'debug': 'org.mozilla.focus.debug' +} + +FENIX_CHANNEL_TO_PKG = { 'nightly': 'org.mozilla.fenix', 'beta': 'org.mozilla.firefox.beta', 'release': 'org.mozilla.firefox', @@ -38,11 +45,17 @@ TESTS = [TEST_COLD_MAIN_FF, TEST_COLD_MAIN_RESTORE, TEST_COLD_VIEW_FF, TEST_COLD TEST_URI = 'https://example.com' +PROD_FENIX = 'fenix' +PROD_FOCUS = 'focus' +PRODUCTS = [PROD_FENIX, PROD_FOCUS] + def parse_args(): parser = argparse.ArgumentParser(description=DESC, formatter_class=argparse.RawTextHelpFormatter) + + assert FENIX_CHANNEL_TO_PKG.keys() == FOCUS_CHANNEL_TO_PKG.keys(), 'should be equal to use one for choices= below' parser.add_argument( - "release_channel", choices=CHANNEL_TO_PKG.keys(), help="the release channel to measure" + "release_channel", choices=FENIX_CHANNEL_TO_PKG.keys(), help="the release channel to measure" ) parser.add_argument( "test_name", choices=TESTS, help="""the measurement methodology to use. Options: @@ -60,6 +73,12 @@ See https://wiki.mozilla.org/Performance/Fenix#Terminology for descriptions of c )) parser.add_argument("path", help="the path to save the measurement results; will abort if file exists") + # We ordinarily wouldn't specify a default because it may cause the user to get results + # from a product they didn't intend but this script lives in the fenix repo so having fenix + # as a default would be less confusing for users. + parser.add_argument("-p", "--product", default=PROD_FENIX, choices=PRODUCTS, + help="which product to get measurements from") + parser.add_argument("-c", "--iter-count", default=DEFAULT_ITER_COUNT, type=int, help="the number of iterations to run. defaults to {}".format(DEFAULT_ITER_COUNT)) parser.add_argument("-f", "--force", action="store_true", @@ -75,6 +94,14 @@ def validate_args(args): raise Exception("Given `path` unexpectedly exists: pick a new path or use --force to overwrite.") +def product_channel_to_pkg_id(product, channel): + if product == PROD_FENIX: + pkg_to_channel_map = FENIX_CHANNEL_TO_PKG + elif product == PROD_FOCUS: + pkg_to_channel_map = FOCUS_CHANNEL_TO_PKG + return pkg_to_channel_map[channel] + + def get_adb_shell_args(): return ['adb', 'shell'] @@ -131,7 +158,7 @@ def get_start_cmd(test_name, pkg_id): return cmd -def measure(test_name, pkg_id, start_cmd_args, iter_count): +def measure(test_name, product, pkg_id, start_cmd_args, iter_count): # Startup profiling may accidentally be left enabled and throw off the results. # To prevent this, we disable it. disable_startup_profiling() @@ -152,18 +179,18 @@ def measure(test_name, pkg_id, start_cmd_args, iter_count): subprocess.run(['adb', 'logcat', '-c'], check=True) proc = subprocess.run(start_cmd_args, check=True, capture_output=True) # expected to wait for app to start. - measurements.append(get_measurement(test_name, pkg_id, proc.stdout)) + measurements.append(get_measurement(test_name, product, pkg_id, proc.stdout)) return measurements -def get_measurement(test_name, pkg_id, stdout): +def get_measurement(test_name, product, pkg_id, stdout): if test_name in [TEST_COLD_MAIN_FF, TEST_COLD_VIEW_FF]: measurement = get_measurement_from_am_start_log(stdout) elif test_name in [TEST_COLD_VIEW_NAV_START, TEST_COLD_MAIN_RESTORE]: time.sleep(4) # We must sleep until the navigation start event occurs. proc = subprocess.run(['adb', 'logcat', '-d'], check=True, capture_output=True) - measurement = get_measurement_from_nav_start_logcat(pkg_id, proc.stdout) + measurement = get_measurement_from_nav_start_logcat(product, pkg_id, proc.stdout) return measurement @@ -186,7 +213,7 @@ def get_measurement_from_am_start_log(stdout): return duration -def get_measurement_from_nav_start_logcat(pkg_id, logcat_bytes): +def get_measurement_from_nav_start_logcat(product, pkg_id, logcat_bytes): # Relevant lines: # 05-18 14:32:47.366 1759 6003 I ActivityManager: START u0 {act=android.intent.action.VIEW dat=https://example.com/... typ=text/html flg=0x10000000 cmp=org.mozilla.fenix/.IntentReceiverActivity} from uid 2000 # noqa # 05-18 14:32:47.402 1759 6003 I ActivityManager: Start proc 9007:org.mozilla.fenix/u0a170 for activity org.mozilla.fenix/.IntentReceiverActivity # noqa @@ -209,7 +236,18 @@ def get_measurement_from_nav_start_logcat(pkg_id, logcat_bytes): def get_page_start_datetime(): page_start_re = re.compile('GeckoSession: handleMessage GeckoView:PageStart uri=') page_start_lines = [line for line in lines if page_start_re.search(line)] - assert len(page_start_lines) == 2, 'found len=' + str(len(page_start_lines)) # Lines: about:blank & target URL. + page_start_line_count = len(page_start_lines) + page_start_assert_msg = 'found len=' + str(page_start_line_count) + + # In focus versions <= v8.8.2, it logs 3 PageStart lines and these include actual uris. + # We need to handle our assertion differently due to the different line count. + # + # In focus versions >= v8.8.3, this measurement is broken because the logcat were removed. + is_old_version_of_focus = 'about:blank' in page_start_lines[0] and product == PROD_FOCUS + if is_old_version_of_focus: + assert page_start_line_count == 3, page_start_assert_msg # Lines: about:blank, target URL, target URL. + else: + assert page_start_line_count == 2, page_start_assert_msg # Lines: about:blank, target URL. return line_to_datetime(page_start_lines[1]) # 2nd PageStart is for target URL. logcat = logcat_bytes.decode('UTF-8') # Easier to work with and must for strptime. @@ -249,10 +287,10 @@ def main(): args = parse_args() validate_args(args) - pkg_id = CHANNEL_TO_PKG[args.release_channel] + pkg_id = product_channel_to_pkg_id(args.product, args.release_channel) start_cmd = get_start_cmd(args.test_name, pkg_id) print_preface_text(args.test_name) - measurements = measure(args.test_name, pkg_id, start_cmd, args.iter_count) + measurements = measure(args.test_name, args.product, pkg_id, start_cmd, args.iter_count) save_measurements(args.path, measurements)