Bug 1807350 - Baseline profiles for nightly

app/build.gradle

@@ -206,7 +206,14 @@ android {
 
             reset()
 
-            include "x86", "armeabi-v7a", "arm64-v8a", "x86_64"
+            // As gradle is unable to pick the right apk to install when multiple apks are generated
+            // while running benchmark tests or generating baseline profiles. To circumvent this,
+            // this flag is passed to make sure only one apk is generated so gradle can pick that one.
+            if (project.hasProperty("benchmarkTest")) {
+                include "arm64-v8a"
+            } else {
+                include "x86", "armeabi-v7a", "arm64-v8a", "x86_64"
+            }
         }
     }
 

benchmark/build.gradle

@@ -2,6 +2,8 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+import com.android.build.api.dsl.ManagedVirtualDevice
+
 plugins {
     id 'com.android.test'
     id 'org.jetbrains.kotlin.android'
@@ -39,6 +41,18 @@ android {
 
     targetProjectPath = ":app"
     experimentalProperties["android.experimental.self-instrumenting"] = true
+
+    testOptions {
+        managedDevices {
+            devices {
+                pixel6Api34(ManagedVirtualDevice) {
+                    device = "Pixel 6"
+                    apiLevel = 34
+                    systemImageSource = "google"
+                }
+            }
+        }
+    }
 }
 
 /**

benchmark/src/main/java/org/mozilla/fenix/benchmark/BaselineProfilesStartupBenchmark.kt

@@ -0,0 +1,70 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.fenix.benchmark
+
+import android.os.Build
+import androidx.annotation.RequiresApi
+import androidx.benchmark.macro.BaselineProfileMode
+import androidx.benchmark.macro.CompilationMode
+import androidx.benchmark.macro.StartupMode
+import androidx.benchmark.macro.StartupTimingMetric
+import androidx.benchmark.macro.junit4.MacrobenchmarkRule
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import org.junit.Rule
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.mozilla.fenix.benchmark.utils.measureRepeatedDefault
+
+/**
+ * This test class benchmarks the speed of app startup. Run this benchmark to verify how effective
+ * a Baseline Profile is. It does this by comparing [CompilationMode.None], which represents the
+ * app with no Baseline Profiles optimizations, and [CompilationMode.Partial], which uses Baseline Profiles.
+ *
+ * Before running make sure `autosignReleaseWithDebugKey=true` is present in local.properties.
+ *
+ * Run this benchmark to see startup measurements and captured system traces for verifying
+ * the effectiveness of your Baseline Profiles. You can run it directly from Android
+ * Studio as an instrumentation test that logs the benchmark metrics with links to the Perfetto traces,
+ *
+ * or using the gradle command:
+ *
+ * ```
+ * ./gradlew :benchmark:connectedBenchmarkAndroidTest -P android.testInstrumentationRunnerArguments.class=org.mozilla.fenix.benchmark.BaselineProfilesStartupBenchmark -P benchmarkTest
+ * ```
+ *
+ * The metric results will be in `benchmark/build/outputs/connected_android_test_additional_output` folder.
+ *
+ * Run the benchmarks on a physical device, not an emulator because the emulator doesn't represent
+ * real world performance and shares system resources with its host.
+ *
+ * For more information, see the [Macrobenchmark documentation](https://d.android.com/macrobenchmark#create-macrobenchmark)
+ * and the [instrumentation arguments documentation](https://d.android.com/topic/performance/benchmarking/macrobenchmark-instrumentation-args).
+ **/
+@RunWith(AndroidJUnit4::class)
+@RequiresApi(Build.VERSION_CODES.N)
+class BaselineProfilesStartupBenchmark {
+
+    @get:Rule
+    val benchmarkRule = MacrobenchmarkRule()
+
+    @Test
+    fun startupNone() = startupBenchmark(CompilationMode.None())
+
+    @Test
+    fun startupPartialWithBaselineProfiles() =
+        startupBenchmark(CompilationMode.Partial(baselineProfileMode = BaselineProfileMode.Require))
+
+    private fun startupBenchmark(compilationMode: CompilationMode) =
+        benchmarkRule.measureRepeatedDefault(
+            metrics = listOf(StartupTimingMetric()),
+            startupMode = StartupMode.COLD,
+            compilationMode = compilationMode,
+            setupBlock = {
+                pressHome()
+            },
+        ) {
+            startActivityAndWait()
+        }
+}

benchmark/src/main/java/org/mozilla/fenix/benchmark/baselineprofile/StartupOnlyBaselineProfileGenerator.kt

@@ -0,0 +1,60 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.fenix.benchmark.baselineprofile
+
+import android.os.Build
+import androidx.annotation.RequiresApi
+import androidx.benchmark.macro.junit4.BaselineProfileRule
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import org.junit.Rule
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.mozilla.fenix.benchmark.utils.TARGET_PACKAGE
+
+/**
+ * This test class generates a basic startup baseline profile for the target package.
+ *
+ * Refer to the [baseline profile documentation](https://d.android.com/topic/performance/baselineprofiles)
+ * for more information.
+ *
+ * Make sure `autosignReleaseWithDebugKey=true` is present in local.properties.
+ *
+ * Generate the baseline profile using this gradle task:
+ * ```
+ *  ./gradlew :benchmark:pixel6Api34BenchmarkAndroidTest -P android.testInstrumentationRunnerArguments.class=org.mozilla.fenix.benchmark.baselineprofile.StartupOnlyBaselineProfileGenerator -P benchmarkTest -P disableOptimization
+ * ```
+ *
+ * Check [documentation](https://d.android.com/topic/performance/benchmarking/macrobenchmark-instrumentation-args)
+ * for more information about available instrumentation arguments.
+ *
+ * As the experiment is only for nightly, run
+ * ```
+ * ./gradlew copyBaselineProfile
+ * ```
+ * This will copy the baseline profiles to the nightly folder. These are the profiles that will be compiled with the nightly build.
+ *
+ * Then, copy the profiles to app/src/benchmark/baselineProfiles to verify the improvements by running
+ * the [org.mozilla.fenix.benchmark.BaselineProfilesStartupBenchmark] benchmark.
+ * Notice that when we run the benchmark, we run the benchmark variant and not the nightly so copying the profiles here is important.
+ * These shouldn't be pushed to version control.
+ *
+ * When using this class to generate a baseline profile, only API 33+ or rooted API 28+ are supported.
+ **/
+@RequiresApi(Build.VERSION_CODES.P)
+@RunWith(AndroidJUnit4::class)
+class StartupOnlyBaselineProfileGenerator {
+
+    @get:Rule
+    val rule = BaselineProfileRule()
+
+    @Test
+    fun generateBaselineProfile() {
+        rule.collect(
+            packageName = TARGET_PACKAGE,
+        ) {
+            startActivityAndWait()
+        }
+    }
+}

build.gradle

@@ -1,8 +1,11 @@
+// Top-level build file where you can add configuration options common to all sub-projects/modules.
+
 import io.gitlab.arturbosch.detekt.Detekt
 import io.gitlab.arturbosch.detekt.DetektCreateBaselineTask
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
-
-// Top-level build file where you can add configuration options common to all sub-projects/modules.
+import java.nio.file.Files
+import java.nio.file.Paths
+import java.nio.file.StandardCopyOption
 
 buildscript {
     // This logic is duplicated in the allprojects block: I don't know how to fix that.
@@ -283,3 +286,18 @@ tasks.register("githubLintDetektDetails", GithubDetailsTask) {
 tasks.register("githubLintAndroidDetails", GithubDetailsTask) {
     text = "### [Android Lint Results Fenix]({reportsUrl}/lint-results-debug.html)"
 }
+
+// Task to copy generated baseline profile to the app module nightly variant.
+tasks.register("copyBaselineProfile", DefaultTask) {
+    doLast {
+        File profileFile = fileTree('benchmark/build/outputs') {
+            include '**/*baseline-prof.txt'
+        }.getSingleFile()
+        def destinationPath = Paths.get("app", "src", "nightly", "baselineProfiles", "baseline-prof.txt")
+        File destinationDir = destinationPath.toFile().parentFile
+        if (!destinationDir.exists()) {
+            destinationDir.mkdirs()
+        }
+        Files.copy(profileFile.toPath(), destinationPath, StandardCopyOption.REPLACE_EXISTING)
+    }
+}