From 5913d09770bbed1e66701db6c35709d429c1baec Mon Sep 17 00:00:00 2001
From: rahulsainani <rsainani@mozilla.com>
Date: Thu, 16 Feb 2023 16:09:38 +0530
Subject: [PATCH] Bug 1807324 - Add macrobenchmarks for startup metrics

---
 .buildconfig.yml                              | 11 +++
 app/build.gradle                              | 10 ++-
 app/src/main/AndroidManifest.xml              |  4 ++
 app/src/main/java/org/mozilla/fenix/Config.kt |  2 +-
 benchmark/build.gradle                        | 70 +++++++++++++++++++
 benchmark/src/main/AndroidManifest.xml        |  7 ++
 .../fenix/benchmark/StartupBenchmark.kt       | 49 +++++++++++++
 .../fenix/benchmark/utils/Constants.kt        |  8 +++
 .../benchmark/utils/MacroBenchmarkRule.kt     | 37 ++++++++++
 buildSrc/src/main/java/FenixDependencies.kt   |  5 +-
 settings.gradle                               |  1 +
 11 files changed, 200 insertions(+), 4 deletions(-)
 create mode 100644 benchmark/build.gradle
 create mode 100644 benchmark/src/main/AndroidManifest.xml
 create mode 100644 benchmark/src/main/java/org/mozilla/fenix/benchmark/StartupBenchmark.kt
 create mode 100644 benchmark/src/main/java/org/mozilla/fenix/benchmark/utils/Constants.kt
 create mode 100644 benchmark/src/main/java/org/mozilla/fenix/benchmark/utils/MacroBenchmarkRule.kt

diff --git a/.buildconfig.yml b/.buildconfig.yml
index 4cf84b667..62d820f7a 100644
--- a/.buildconfig.yml
+++ b/.buildconfig.yml
@@ -135,6 +135,17 @@ variants:
     fileName: app-fenix-x86_64-beta-unsigned.apk
   build_type: beta
   name: fenixBeta
+- apks:
+  - abi: arm64-v8a
+    fileName: app-fenix-arm64-v8a-benchmark-unsigned.apk
+  - abi: armeabi-v7a
+    fileName: app-fenix-armeabi-v7a-benchmark-unsigned.apk
+  - abi: x86
+    fileName: app-fenix-x86-benchmark-unsigned.apk
+  - abi: x86_64
+    fileName: app-fenix-x86_64-benchmark-unsigned.apk
+  build_type: benchmark
+  name: fenixBenchmark
 - apks:
   - abi: noarch
     fileName: app-debug-androidTest.apk
diff --git a/app/build.gradle b/app/build.gradle
index 5d3cf8906..6f2ad3b0c 100644
--- a/app/build.gradle
+++ b/app/build.gradle
@@ -11,8 +11,6 @@ apply plugin: 'kotlin-parcelize'
 apply plugin: 'jacoco'
 apply plugin: 'androidx.navigation.safeargs.kotlin'
 apply plugin: 'com.google.android.gms.oss-licenses-plugin'
-apply plugin: 'androidx.benchmark'
-
 
 import com.android.build.OutputFile
 import groovy.json.JsonOutput
@@ -148,6 +146,11 @@ android {
                     "deepLinkScheme": deepLinkSchemeValue
             ]
         }
+        benchmark releaseTemplate >> {
+            initWith buildTypes.nightly
+            applicationIdSuffix ".fenix"
+            debuggable false
+        }
     }
 
     buildFeatures {
@@ -447,6 +450,7 @@ nimbus {
             fenixNightly: "nightly",
             fenixBeta: "beta",
             fenixRelease: "release",
+            fenixBenchmark: "developer",
     ]
     // This is generated by the FML and should be checked into git.
     // It will be fetched by Experimenter (the Nimbus experiment website)
@@ -620,6 +624,8 @@ dependencies {
     implementation FenixDependencies.google_play_review
     implementation FenixDependencies.google_play_review_ktx
 
+    implementation FenixDependencies.androidx_profileinstaller
+
     androidTestImplementation FenixDependencies.uiautomator
     androidTestImplementation "tools.fastlane:screengrab:2.0.0"
     // This Falcon version is added to maven central now required for Screengrab
diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml
index 66928ce86..a1a85adcf 100644
--- a/app/src/main/AndroidManifest.xml
+++ b/app/src/main/AndroidManifest.xml
@@ -48,6 +48,10 @@
         android:usesCleartextTraffic="true"
         tools:ignore="UnusedAttribute">
 
+        <profileable
+            android:shell="true"
+            tools:targetApi="29" />
+
         <!--
           We inherited this entry (${applicationId}.App) from Fennec. We need to keep this as our
           main launcher to avoid launcher icons on the home screen disappearing for all our users.
diff --git a/app/src/main/java/org/mozilla/fenix/Config.kt b/app/src/main/java/org/mozilla/fenix/Config.kt
index bb5688fe2..534cf2d80 100644
--- a/app/src/main/java/org/mozilla/fenix/Config.kt
+++ b/app/src/main/java/org/mozilla/fenix/Config.kt
@@ -53,7 +53,7 @@ enum class ReleaseChannel {
 object Config {
     val channel = when (BuildConfig.BUILD_TYPE) {
         "debug" -> ReleaseChannel.Debug
-        "nightly" -> ReleaseChannel.Nightly
+        "nightly", "benchmark" -> ReleaseChannel.Nightly
         "beta" -> ReleaseChannel.Beta
         "release" -> ReleaseChannel.Release
         else -> {
diff --git a/benchmark/build.gradle b/benchmark/build.gradle
new file mode 100644
index 000000000..dc8bb9a6e
--- /dev/null
+++ b/benchmark/build.gradle
@@ -0,0 +1,70 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+plugins {
+    id 'com.android.test'
+    id 'org.jetbrains.kotlin.android'
+}
+
+android {
+    namespace 'org.mozilla.fenix.benchmark'
+    compileSdk Config.compileSdkVersion
+
+    compileOptions {
+        sourceCompatibility = JavaVersion.VERSION_11
+        targetCompatibility = JavaVersion.VERSION_11
+    }
+
+    kotlinOptions {
+        jvmTarget = "11"
+    }
+
+    defaultConfig {
+        minSdk 23
+        targetSdk Config.targetSdkVersion
+        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+    }
+
+    buildTypes {
+        // This benchmark buildType is used for benchmarking, and should function like your
+        // release build (for example, with minification on). It's signed with a debug key
+        // for easy local testing.
+        benchmark {
+            debuggable = true
+            signingConfig signingConfigs.debug
+            matchingFallbacks = ["release"]
+        }
+    }
+
+    targetProjectPath = ":app"
+    experimentalProperties["android.experimental.self-instrumenting"] = true
+}
+
+/**
+ * This fixes the dependency resolution issue with Glean Native. The glean gradle plugin does this
+ * and that's applied to the app module. Since there are no other uses of the glean plugin in the
+ * benchmark module, we do this manually here.
+ */
+configurations.all {
+    resolutionStrategy.capabilitiesResolution.withCapability("org.mozilla.telemetry:glean-native") {
+        def toBeSelected = candidates.find { it.id instanceof ModuleComponentIdentifier && it.id.module.contains('geckoview') }
+        if (toBeSelected != null) {
+            select(toBeSelected)
+        }
+        because 'use GeckoView Glean instead of standalone Glean'
+    }
+}
+
+dependencies {
+    implementation FenixDependencies.androidx_junit
+    implementation FenixDependencies.espresso_core
+    implementation FenixDependencies.uiautomator
+    implementation FenixDependencies.androidx_benchmark_macro_junit4
+}
+
+androidComponents {
+    beforeVariants(selector().all()) {
+        enabled = buildType == "benchmark"
+    }
+}
diff --git a/benchmark/src/main/AndroidManifest.xml b/benchmark/src/main/AndroidManifest.xml
new file mode 100644
index 000000000..251d1c7b2
--- /dev/null
+++ b/benchmark/src/main/AndroidManifest.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+
+    <queries>
+        <package android:name="org.mozilla.fenix" />
+    </queries>
+</manifest>
diff --git a/benchmark/src/main/java/org/mozilla/fenix/benchmark/StartupBenchmark.kt b/benchmark/src/main/java/org/mozilla/fenix/benchmark/StartupBenchmark.kt
new file mode 100644
index 000000000..014df73e3
--- /dev/null
+++ b/benchmark/src/main/java/org/mozilla/fenix/benchmark/StartupBenchmark.kt
@@ -0,0 +1,49 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.fenix.benchmark
+
+import androidx.benchmark.macro.StartupMode
+import androidx.benchmark.macro.StartupTimingMetric
+import androidx.benchmark.macro.junit4.MacrobenchmarkRule
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import org.junit.Rule
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.mozilla.fenix.benchmark.utils.measureRepeatedDefault
+
+/**
+ * This is a startup benchmark.
+ * It navigates to the device's home screen, and launches the default activity.
+ *
+ * Before running this benchmark,
+ * switch your app's active build variant in the Studio (affects Studio runs only)
+ *
+ * Run this benchmark from Studio to see startup measurements, and captured system traces
+ * for investigating your app's performance.
+ */
+@RunWith(AndroidJUnit4::class)
+class StartupBenchmark {
+    @get:Rule
+    val benchmarkRule = MacrobenchmarkRule()
+
+    @Test
+    fun startupCold() = startupBenchmark(StartupMode.COLD)
+
+    @Test
+    fun startupWarm() = startupBenchmark(StartupMode.WARM)
+
+    @Test
+    fun startupHot() = startupBenchmark(StartupMode.HOT)
+
+    private fun startupBenchmark(startupMode: StartupMode) = benchmarkRule.measureRepeatedDefault(
+        metrics = listOf(StartupTimingMetric()),
+        startupMode = startupMode,
+        setupBlock = {
+            pressHome()
+        }
+    ) {
+        startActivityAndWait()
+    }
+}
diff --git a/benchmark/src/main/java/org/mozilla/fenix/benchmark/utils/Constants.kt b/benchmark/src/main/java/org/mozilla/fenix/benchmark/utils/Constants.kt
new file mode 100644
index 000000000..14fe42611
--- /dev/null
+++ b/benchmark/src/main/java/org/mozilla/fenix/benchmark/utils/Constants.kt
@@ -0,0 +1,8 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.fenix.benchmark.utils
+
+const val TARGET_PACKAGE = "org.mozilla.fenix"
+const val DEFAULT_ITERATIONS = 5
diff --git a/benchmark/src/main/java/org/mozilla/fenix/benchmark/utils/MacroBenchmarkRule.kt b/benchmark/src/main/java/org/mozilla/fenix/benchmark/utils/MacroBenchmarkRule.kt
new file mode 100644
index 000000000..d3fbd499c
--- /dev/null
+++ b/benchmark/src/main/java/org/mozilla/fenix/benchmark/utils/MacroBenchmarkRule.kt
@@ -0,0 +1,37 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.fenix.benchmark.utils
+
+import androidx.annotation.IntRange
+import androidx.benchmark.macro.CompilationMode
+import androidx.benchmark.macro.MacrobenchmarkScope
+import androidx.benchmark.macro.Metric
+import androidx.benchmark.macro.StartupMode
+import androidx.benchmark.macro.junit4.MacrobenchmarkRule
+
+/**
+ * Extension function that calls [MacrobenchmarkRule.measureRepeated] with
+ * defaults parameters set for [packageName] and [iterations].
+ */
+fun MacrobenchmarkRule.measureRepeatedDefault(
+    packageName: String = TARGET_PACKAGE,
+    metrics: List<Metric>,
+    compilationMode: CompilationMode = CompilationMode.DEFAULT,
+    startupMode: StartupMode? = null,
+    @IntRange(from = 1)
+    iterations: Int = DEFAULT_ITERATIONS,
+    setupBlock: MacrobenchmarkScope.() -> Unit = {},
+    measureBlock: MacrobenchmarkScope.() -> Unit,
+) {
+    measureRepeated(
+        packageName = packageName,
+        metrics = metrics,
+        compilationMode = compilationMode,
+        startupMode = startupMode,
+        iterations = iterations,
+        setupBlock = setupBlock,
+        measureBlock = measureBlock,
+    )
+}
diff --git a/buildSrc/src/main/java/FenixDependencies.kt b/buildSrc/src/main/java/FenixDependencies.kt
index ebd152739..32f8d2655 100644
--- a/buildSrc/src/main/java/FenixDependencies.kt
+++ b/buildSrc/src/main/java/FenixDependencies.kt
@@ -23,11 +23,12 @@ object FenixVersions {
     const val androidx_compose = "1.3.1"
     const val androidx_compose_compiler = "1.4.3"
     const val androidx_appcompat = "1.3.0"
-    const val androidx_benchmark = "1.0.0"
+    const val androidx_benchmark = "1.1.1"
     const val androidx_biometric = "1.1.0"
     const val androidx_coordinator_layout = "1.1.0"
     const val androidx_constraint_layout = "2.0.4"
     const val androidx_preference = "1.1.1"
+    const val androidx_profileinstaller = "1.2.2"
     const val androidx_legacy = "1.0.0"
     const val androidx_annotation = "1.5.0"
     const val androidx_lifecycle = "2.5.1"
@@ -83,6 +84,8 @@ object FenixDependencies {
     const val androidx_compose_material = "androidx.compose.material:material:${FenixVersions.androidx_compose}"
     const val androidx_annotation = "androidx.annotation:annotation:${FenixVersions.androidx_annotation}"
     const val androidx_benchmark_junit4 = "androidx.benchmark:benchmark-junit4:${FenixVersions.androidx_benchmark}"
+    const val androidx_benchmark_macro_junit4 = "androidx.benchmark:benchmark-macro-junit4:${FenixVersions.androidx_benchmark}"
+    const val androidx_profileinstaller = "androidx.profileinstaller:profileinstaller:${FenixVersions.androidx_profileinstaller}"
     const val androidx_biometric = "androidx.biometric:biometric:${FenixVersions.androidx_biometric}"
     const val androidx_fragment = "androidx.fragment:fragment-ktx:${FenixVersions.androidx_fragment}"
     const val androidx_appcompat = "androidx.appcompat:appcompat:${FenixVersions.androidx_appcompat}"
diff --git a/settings.gradle b/settings.gradle
index b64c32be8..bc230d68e 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -27,6 +27,7 @@ plugins {
 include ':app'
 include ':mozilla-detekt-rules'
 include ':mozilla-lint-rules'
+include ':benchmark'
 
 // Synchronized library configuration for all modules
 // This "componentsVersion" number is defined in "version.txt" and should follow