diff --git a/app/benchmark.gradle b/app/benchmark.gradle new file mode 100644 index 0000000000..e055128dd2 --- /dev/null +++ b/app/benchmark.gradle @@ -0,0 +1,66 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +// This comment contains the central documentation for how we configured Jetpack Benchmark. Currently: +// - microbenchmark: configured differently than recommended (see inline notes below) +// - macrobenchmark: not configured +// +// To run our benchmarks, you need to set the "benchmark" gradle property. You can: +// - (preferred) Run via the command line (change the class you run on): +// ./gradlew -Pbenchmark app :connectedCheck -Pandroid.testInstrumentationRunnerArguments.class=org.mozilla.fenix.perf.SampleBenchmark +// - Use the IDE. Temporarily set the "benchmark" property in app/build.gradle with "ext.benchmark=true" +// near the top of the file. DO NOT COMMIT THIS. +// - (note: I was unable to get IDE run configurations working) +// +// To get the results, look at this file (we recommend using the median; results are in nanoseconds): +// app/build/outputs/connected_android_test_additional_output/nightlyAndroidTest/connected//org.mozilla.fenix-benchmarkData.json +// +// I was unable to get the results to print directly in Android Studio (perhaps it's my device). +// +// The official documentation suggests configuring microbenchmark in a separate module. This would +// require any benchmarked code to be in a library module, not the :app module (see below). To avoid +// this requirement, we created the "benchmark" gradle property. +// +// For the most accurate results, the documentation recommends running tests on rooted devices with +// the CPU clock locked. +// +// See https://developer.android.com/studio/profile/benchmark#what-to-benchmark for when writing a +// jetpack microbenchmark is a good fit. + +// I think `android` represents this object: +// https://google.github.io/android-gradle-dsl/3.3/com.android.build.gradle.AppExtension.html +ext.maybeConfigForJetpackBenchmark = { android -> + if (!project.hasProperty("benchmark")) { + return + } + + // The official documentation https://developer.android.com/studio/profile/benchmark#full-setup + // recommends setting up the Microbenchmark library in a separate module from your app: AFAICT, + // the reason for this is to prevent the benchmarks from being configured against debug + // builds. We chose not to do this because it's a lot of work to pull code out into a + // separate module just to benchmark it. We were able to replicate the outcome by setting + // this testBuildType property. + android.testBuildType "nightly" + + // WARNING: our proguard configuration for androidTest is not set up correctly so the tests + // fail if we don't disable minification. DISABLING MINIFICATION PRODUCES BENCHMARKS THAT ARE + // LESS REPRESENTATIVE TO THE USER EXPERIENCE, however, so we made this tradeoff to reduce + // implementation time. + project.ext.disableOptimization = true + + android.defaultConfig { + // WARNING: the benchmark framework warns you if you're running the test in a configuration + // that will compromise the accuracy of the results. Unfortunately, I couldn't get everything + // working so I had to suppress some things. + // + // - ACTIVITY-MISSING: we're supposed to use the test instrumentation runner, + // "androidx.benchmark.junit4.AndroidBenchmarkRunner". However, when I do so, I get an error + // that we're unable to launch the activity. My understanding is that this runner will use an + // "IsolationActivity" to reduce the impact of other work on the device from affecting the benchmark + // and to opt into a lower-max CPU frequency on unrooted devices that support it + // - UNLOCKED: ./gradlew lockClocks, which locks the CPU frequency, fails on my device. See + // https://issuetracker.google.com/issues/176836267 for potential workarounds. + testInstrumentationRunnerArgument 'androidx.benchmark.suppressErrors', 'ACTIVITY-MISSING,UNLOCKED' + } +} diff --git a/app/build.gradle b/app/build.gradle index ce7d6fcbd4..a4e3b7648b 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -8,6 +8,7 @@ apply plugin: 'kotlin-android-extensions' apply plugin: 'jacoco' apply plugin: 'androidx.navigation.safeargs.kotlin' apply plugin: 'com.google.android.gms.oss-licenses-plugin' +apply plugin: 'androidx.benchmark' import com.android.build.OutputFile @@ -17,9 +18,12 @@ import org.gradle.internal.logging.text.StyledTextOutputFactory import static org.gradle.api.tasks.testing.TestResult.ResultType +apply from: 'benchmark.gradle' + android { compileSdkVersion Config.compileSdkVersion + project.maybeConfigForJetpackBenchmark(it) if (project.hasProperty("testBuildType")) { // Allowing to configure the test build type via command line flag (./gradlew -PtestBuildType=beta ..) // in order to run UI tests against other build variants than debug in automation. @@ -553,6 +557,7 @@ dependencies { androidTestImplementation Deps.androidx_junit androidTestImplementation Deps.androidx_work_testing + androidTestImplementation Deps.androidx_benchmark_junit4 androidTestImplementation Deps.mockwebserver testImplementation Deps.mozilla_support_test testImplementation Deps.mozilla_support_test_libstate diff --git a/app/src/androidTest/java/org/mozilla/fenix/perf/SampleBenchmark.kt b/app/src/androidTest/java/org/mozilla/fenix/perf/SampleBenchmark.kt new file mode 100644 index 0000000000..f4da3d187f --- /dev/null +++ b/app/src/androidTest/java/org/mozilla/fenix/perf/SampleBenchmark.kt @@ -0,0 +1,38 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +package org.mozilla.fenix.perf + +import androidx.benchmark.junit4.BenchmarkRule +import androidx.benchmark.junit4.measureRepeated +import androidx.test.ext.junit.runners.AndroidJUnit4 +import org.junit.Ignore +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith + +/** + * To run this benchmark: + * - Comment out @Ignore: DO NOT COMMIT THIS! + * - See run instructions in app/benchmark.gradle + * + * See https://developer.android.com/studio/profile/benchmark#write-benchmark for how to write a + * real benchmark, including testing UI code. See + * https://developer.android.com/studio/profile/benchmark#what-to-benchmark for when jetpack + * microbenchmark is a good fit. + */ +@Ignore("This is a sample: we don't want it to run when we run all the tests") +@RunWith(AndroidJUnit4::class) +class SampleBenchmark { + @get:Rule + val benchmarkRule = BenchmarkRule() + + @Test + fun additionBenchmark() = benchmarkRule.measureRepeated { + var i = 0 + while (i < 10_000_000) { + i++ + } + } +} diff --git a/build.gradle b/build.gradle index 8b5644fe6a..3b1cb9781a 100644 --- a/build.gradle +++ b/build.gradle @@ -63,6 +63,7 @@ buildscript { dependencies { classpath Deps.tools_androidgradle classpath Deps.tools_kotlingradle + classpath Deps.tools_benchmarkgradle classpath Deps.androidx_safeargs classpath Deps.allopen classpath Deps.osslicenses_plugin diff --git a/buildSrc/src/main/java/Dependencies.kt b/buildSrc/src/main/java/Dependencies.kt index b9f682b4e0..e6cbb58a46 100644 --- a/buildSrc/src/main/java/Dependencies.kt +++ b/buildSrc/src/main/java/Dependencies.kt @@ -18,6 +18,7 @@ object Versions { const val jna = "5.6.0" const val androidx_appcompat = "1.3.0" + const val androidx_benchmark = "1.0.0" const val androidx_biometric = "1.1.0" const val androidx_coordinator_layout = "1.1.0" const val androidx_constraint_layout = "2.0.4" @@ -55,6 +56,7 @@ object Versions { object Deps { const val tools_androidgradle = "com.android.tools.build:gradle:${Versions.android_gradle_plugin}" const val tools_kotlingradle = "org.jetbrains.kotlin:kotlin-gradle-plugin:${Versions.kotlin}" + const val tools_benchmarkgradle = "androidx.benchmark:benchmark-gradle-plugin:${Versions.androidx_benchmark}" const val kotlin_stdlib = "org.jetbrains.kotlin:kotlin-stdlib-jdk7:${Versions.kotlin}" const val kotlin_stdlib_jdk8 = "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${Versions.kotlin}" const val kotlin_reflect = "org.jetbrains.kotlin:kotlin-reflect:${Versions.kotlin}" @@ -163,6 +165,7 @@ object Deps { const val leakcanary = "com.squareup.leakcanary:leakcanary-android:${Versions.leakcanary}" const val androidx_annotation = "androidx.annotation:annotation:${Versions.androidx_annotation}" + const val androidx_benchmark_junit4 = "androidx.benchmark:benchmark-junit4:${Versions.androidx_benchmark}" const val androidx_biometric = "androidx.biometric:biometric:${Versions.androidx_biometric}" const val androidx_fragment = "androidx.fragment:fragment-ktx:${Versions.androidx_fragment}" const val androidx_appcompat = "androidx.appcompat:appcompat:${Versions.androidx_appcompat}" diff --git a/gradle.properties b/gradle.properties index 79226b3c47..1930aca433 100644 --- a/gradle.properties +++ b/gradle.properties @@ -19,3 +19,6 @@ android.enableR8=true android.enableR8.fullMode=true android.enableUnitTestBinaryResources=false org.gradle.parallel=false + +# Enables copying of Jetpack Benchmark results from the device to the build directory. +android.enableAdditionalTestOutput=true