From e3dc851221478fed009fdb8814403930a3196f13 Mon Sep 17 00:00:00 2001 From: isabelrios Date: Thu, 14 Jan 2021 15:47:36 +0100 Subject: [PATCH] Changes to run tests on beta builds (#16786) * Changes to run tests on beta builds * remove dontoptime and fix bookmarks tests * fix ktlint and smokeTest * add variable for package name * rebase and fix conflicts * add yml files and modify script to run tests * fix yml file indentation * changes in kind file and add try_task * fix error kind file * fix error kind file 2 * fix error kind file 3 * fix reviewer comments * remove mockwebserver implementation as per reviewer comment * run beta tests only on releases branches * reg exp for master branch * reg exp for master branch2 * modify try_task with release branch * remove try file * let's start by running only one test * address latest reviewers comments * double check correct tasks for release branch * remove try file after confirming beta build run Co-authored-by: isabel rios --- app/build.gradle | 1 - app/proguard-rules.pro | 2 +- .../org/mozilla/fenix/helpers/TestHelper.kt | 3 + .../mozilla/fenix/ui/robots/BookmarksRobot.kt | 3 +- .../mozilla/fenix/ui/robots/BrowserRobot.kt | 15 +++-- .../mozilla/fenix/ui/robots/DownloadRobot.kt | 5 +- .../robots/EnhancedTrackingProtectionRobot.kt | 3 +- .../fenix/ui/robots/HomeScreenRobot.kt | 3 +- ...rySubMenusMultipleSelectionToolbarRobot.kt | 5 +- .../fenix/ui/robots/NavigationToolbarRobot.kt | 24 +++----- .../mozilla/fenix/ui/robots/SearchRobot.kt | 3 +- .../SettingsSubMenuPrivateBrowsingRobot.kt | 5 +- .../androidTest/flank-x86-beta.yml | 57 +++++++++++++++++++ automation/taskcluster/androidTest/ui-test.sh | 2 + taskcluster/ci/ui-test/kind.yml | 16 +++++- 15 files changed, 108 insertions(+), 39 deletions(-) create mode 100644 automation/taskcluster/androidTest/flank-x86-beta.yml diff --git a/app/build.gradle b/app/build.gradle index aff69a0e5..55590b698 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -550,7 +550,6 @@ dependencies { androidTestImplementation Deps.androidx_junit androidTestImplementation Deps.androidx_work_testing androidTestImplementation Deps.mockwebserver - testImplementation Deps.mozilla_support_test testImplementation Deps.mozilla_support_test_libstate testImplementation Deps.androidx_junit diff --git a/app/proguard-rules.pro b/app/proguard-rules.pro index aa2ae32ae..3c44e42bf 100644 --- a/app/proguard-rules.pro +++ b/app/proguard-rules.pro @@ -119,4 +119,4 @@ # Keep Android Lifecycle methods # https://bugzilla.mozilla.org/show_bug.cgi?id=1596302 --keep class androidx.lifecycle.** { *; } \ No newline at end of file +-keep class androidx.lifecycle.** { *; } diff --git a/app/src/androidTest/java/org/mozilla/fenix/helpers/TestHelper.kt b/app/src/androidTest/java/org/mozilla/fenix/helpers/TestHelper.kt index 2ae01aa50..dc7521062 100644 --- a/app/src/androidTest/java/org/mozilla/fenix/helpers/TestHelper.kt +++ b/app/src/androidTest/java/org/mozilla/fenix/helpers/TestHelper.kt @@ -32,6 +32,9 @@ import org.mozilla.fenix.helpers.ext.waitNotNull import org.mozilla.fenix.ui.robots.mDevice object TestHelper { + + val packageName = InstrumentationRegistry.getInstrumentation().targetContext.packageName + fun scrollToElementByText(text: String): UiScrollable { val appView = UiScrollable(UiSelector().scrollable(true)) appView.scrollTextIntoView(text) diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/BookmarksRobot.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/BookmarksRobot.kt index 36b014071..2d2c8fda7 100644 --- a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/BookmarksRobot.kt +++ b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/BookmarksRobot.kt @@ -35,6 +35,7 @@ import org.junit.Assert.assertEquals import org.mozilla.fenix.R import org.mozilla.fenix.helpers.TestAssetHelper import org.mozilla.fenix.helpers.TestAssetHelper.waitingTime +import org.mozilla.fenix.helpers.TestHelper.packageName import org.mozilla.fenix.helpers.click import org.mozilla.fenix.helpers.ext.waitNotNull @@ -216,7 +217,7 @@ class BookmarksRobot { } fun openThreeDotMenu(bookmarkTitle: String, interact: ThreeDotMenuBookmarksRobot.() -> Unit): ThreeDotMenuBookmarksRobot.Transition { - mDevice.waitNotNull(Until.findObject(res("org.mozilla.fenix.debug:id/overflow_menu"))) + mDevice.waitNotNull(Until.findObject(res("$packageName:id/overflow_menu"))) threeDotMenu(bookmarkTitle).click() ThreeDotMenuBookmarksRobot().interact() diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/BrowserRobot.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/BrowserRobot.kt index 890274b3b..99be574f0 100644 --- a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/BrowserRobot.kt +++ b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/BrowserRobot.kt @@ -41,6 +41,7 @@ import org.mozilla.fenix.ext.components import org.mozilla.fenix.helpers.Constants.LONG_CLICK_DURATION import org.mozilla.fenix.helpers.SessionLoadedIdlingResource import org.mozilla.fenix.helpers.TestAssetHelper.waitingTime +import org.mozilla.fenix.helpers.TestHelper.packageName import org.mozilla.fenix.helpers.click import org.mozilla.fenix.helpers.ext.waitNotNull @@ -57,9 +58,10 @@ class BrowserRobot { sessionLoadedIdlingResource = SessionLoadedIdlingResource() mDevice.waitNotNull( - Until.findObject(By.res("org.mozilla.fenix.debug:id/mozac_browser_toolbar_url_view")), - waitingTime + Until.findObject(By.res("$packageName:id/mozac_browser_toolbar_url_view")), + waitingTime ) + runWithIdleRes(sessionLoadedIdlingResource) { onView(withId(R.id.mozac_browser_toolbar_url_view)) .check(matches(withText(containsString(url.replace("http://", ""))))) @@ -87,7 +89,7 @@ class BrowserRobot { sessionLoadedIdlingResource = SessionLoadedIdlingResource() mDevice.waitNotNull( - Until.findObject(By.res("org.mozilla.fenix.debug:id/engineView")), + Until.findObject(By.res("$packageName:id/engineView")), waitingTime ) @@ -419,11 +421,8 @@ class BrowserRobot { fun openTabDrawer(interact: TabDrawerRobot.() -> Unit): TabDrawerRobot.Transition { mDevice.waitForIdle(waitingTime) tabsCounter().click() - - mDevice.waitNotNull( - Until.findObject(By.res("org.mozilla.fenix.debug:id/tab_layout")), - waitingTime - ) + mDevice.waitNotNull(Until.findObject(By.res("$packageName:id/tab_layout")), + waitingTime) TabDrawerRobot().interact() return TabDrawerRobot.Transition() diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/DownloadRobot.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/DownloadRobot.kt index 48ccc4c2e..c771602a5 100644 --- a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/DownloadRobot.kt +++ b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/DownloadRobot.kt @@ -26,6 +26,8 @@ import org.mozilla.fenix.helpers.TestHelper import org.mozilla.fenix.helpers.click import org.mozilla.fenix.helpers.ext.waitNotNull import org.mozilla.fenix.helpers.Constants.PackageName.GOOGLE_APPS_PHOTOS +import org.mozilla.fenix.helpers.TestAssetHelper.waitingTime +import org.mozilla.fenix.helpers.TestHelper.packageName /** * Implementation of Robot Pattern for download UI handling. @@ -40,7 +42,6 @@ class DownloadRobot { fun verifyPhotosAppOpens() = assertPhotosOpens() class Transition { - fun clickDownload(interact: DownloadRobot.() -> Unit): Transition { clickDownloadButton().click() @@ -93,7 +94,7 @@ fun downloadRobot(interact: DownloadRobot.() -> Unit): DownloadRobot.Transition } private fun assertDownloadPrompt() { - mDevice.waitNotNull(Until.findObjects(By.res("org.mozilla.fenix.debug:id/download_button"))) + mDevice.waitNotNull(Until.findObjects(By.res("$packageName:id/download_button"))) } private fun assertDownloadNotificationPopup() { diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/EnhancedTrackingProtectionRobot.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/EnhancedTrackingProtectionRobot.kt index 3af407b80..aa6adc18a 100644 --- a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/EnhancedTrackingProtectionRobot.kt +++ b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/EnhancedTrackingProtectionRobot.kt @@ -15,6 +15,7 @@ import androidx.test.uiautomator.UiDevice import androidx.test.uiautomator.Until import org.mozilla.fenix.R import org.mozilla.fenix.helpers.TestAssetHelper +import org.mozilla.fenix.helpers.TestHelper.packageName import org.mozilla.fenix.helpers.click import org.mozilla.fenix.helpers.ext.waitNotNull import org.mozilla.fenix.helpers.isChecked @@ -89,7 +90,7 @@ fun enhancedTrackingProtection(interact: EnhancedTrackingProtectionRobot.() -> U private fun assertEnhancedTrackingProtectionNotice() { mDevice.waitNotNull( - Until.findObject(By.res("org.mozilla.fenix.debug:id/onboarding_message")), + Until.findObject(By.res("$packageName:id/onboarding_message")), TestAssetHelper.waitingTime ) } diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/HomeScreenRobot.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/HomeScreenRobot.kt index 044838a23..428f0bfc6 100644 --- a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/HomeScreenRobot.kt +++ b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/HomeScreenRobot.kt @@ -50,6 +50,7 @@ import org.mozilla.fenix.R import org.mozilla.fenix.ext.components import org.mozilla.fenix.helpers.TestAssetHelper import org.mozilla.fenix.helpers.TestAssetHelper.waitingTime +import org.mozilla.fenix.helpers.TestHelper.packageName import org.mozilla.fenix.helpers.TestHelper.scrollToElementByText import org.mozilla.fenix.helpers.click import org.mozilla.fenix.helpers.ext.waitNotNull @@ -369,7 +370,7 @@ class HomeScreenRobot { tabsCounter().click() mDevice.waitNotNull( - Until.findObject(By.res("org.mozilla.fenix.debug:id/tab_layout")), + Until.findObject(By.res("$packageName:id/tab_layout")), waitingTime ) diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/LibrarySubMenusMultipleSelectionToolbarRobot.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/LibrarySubMenusMultipleSelectionToolbarRobot.kt index 483ad546f..10d39cca5 100644 --- a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/LibrarySubMenusMultipleSelectionToolbarRobot.kt +++ b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/LibrarySubMenusMultipleSelectionToolbarRobot.kt @@ -23,6 +23,7 @@ import org.mozilla.fenix.helpers.click import org.mozilla.fenix.helpers.ext.waitNotNull import org.hamcrest.Matchers.allOf import org.mozilla.fenix.helpers.TestAssetHelper +import org.mozilla.fenix.helpers.TestHelper.packageName /* * Implementation of Robot Pattern for the multiple selection toolbar of History and Bookmarks menus. @@ -99,7 +100,7 @@ class LibrarySubMenusMultipleSelectionToolbarRobot { fun clickOpenNewTab(interact: TabDrawerRobot.() -> Unit): TabDrawerRobot.Transition { openInNewTabButton().click() mDevice.waitNotNull( - Until.findObject(By.res("org.mozilla.fenix.debug:id/tab_layout")), + Until.findObject(By.res("$packageName:id/tab_layout")), waitingTime ) @@ -110,7 +111,7 @@ class LibrarySubMenusMultipleSelectionToolbarRobot { fun clickOpenPrivateTab(interact: TabDrawerRobot.() -> Unit): TabDrawerRobot.Transition { openInPrivateTabButton().click() mDevice.waitNotNull( - Until.findObject(By.res("org.mozilla.fenix.debug:id/tab_layout")), + Until.findObject(By.res("$packageName:id/tab_layout")), waitingTime ) diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/NavigationToolbarRobot.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/NavigationToolbarRobot.kt index 250aa2546..1a66130a9 100644 --- a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/NavigationToolbarRobot.kt +++ b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/NavigationToolbarRobot.kt @@ -26,13 +26,13 @@ import androidx.test.platform.app.InstrumentationRegistry import androidx.test.uiautomator.By import androidx.test.uiautomator.UiDevice import androidx.test.uiautomator.Until -import kotlinx.android.synthetic.main.fragment_search_dialog.view.* import org.hamcrest.CoreMatchers.anyOf import org.hamcrest.CoreMatchers.containsString import org.hamcrest.CoreMatchers.not import org.mozilla.fenix.R import org.mozilla.fenix.helpers.SessionLoadedIdlingResource import org.mozilla.fenix.helpers.TestAssetHelper.waitingTime +import org.mozilla.fenix.helpers.TestHelper.packageName import org.mozilla.fenix.helpers.assertions.AwesomeBarAssertion.Companion.suggestionsAreEqualTo import org.mozilla.fenix.helpers.assertions.AwesomeBarAssertion.Companion.suggestionsAreGreaterThan import org.mozilla.fenix.helpers.click @@ -62,12 +62,12 @@ class NavigationToolbarRobot { fun goBackToWebsite(interact: BrowserRobot.() -> Unit): BrowserRobot.Transition { mDevice.waitNotNull( - Until.findObject(By.res("org.mozilla.fenix.debug:id/toolbar")), + Until.findObject(By.res("$packageName:id/toolbar")), waitingTime ) urlBar().click() mDevice.waitNotNull( - Until.findObject(By.res("org.mozilla.fenix.debug:id/mozac_browser_toolbar_edit_url_view")), + Until.findObject(By.res("$packageName:id/mozac_browser_toolbar_edit_url_view")), waitingTime ) clearAddressBar().click() @@ -84,13 +84,12 @@ class NavigationToolbarRobot { ): BrowserRobot.Transition { sessionLoadedIdlingResource = SessionLoadedIdlingResource() - mDevice.waitNotNull( - Until.findObject(By.res("org.mozilla.fenix.debug:id/toolbar")), - waitingTime + mDevice.waitNotNull(Until.findObject(By.res("$packageName:id/toolbar")), + waitingTime ) urlBar().click() mDevice.waitNotNull( - Until.findObject(By.res("org.mozilla.fenix.debug:id/mozac_browser_toolbar_edit_url_view")), + Until.findObject(By.res("$packageName:id/mozac_browser_toolbar_edit_url_view")), waitingTime ) @@ -111,10 +110,7 @@ class NavigationToolbarRobot { } fun openThreeDotMenu(interact: ThreeDotMenuMainRobot.() -> Unit): ThreeDotMenuMainRobot.Transition { - mDevice.waitNotNull( - Until.findObject(By.res("org.mozilla.fenix.debug:id/mozac_browser_toolbar_menu")), - waitingTime - ) + mDevice.waitNotNull(Until.findObject(By.res("$packageName:id/mozac_browser_toolbar_menu")), waitingTime) threeDotButton().click() ThreeDotMenuMainRobot().interact() @@ -136,11 +132,7 @@ class NavigationToolbarRobot { interact: BrowserRobot.() -> Unit ): BrowserRobot.Transition { sessionLoadedIdlingResource = SessionLoadedIdlingResource() - mDevice.waitNotNull( - Until.findObject(By.res("org.mozilla.fenix.debug:id/toolbar")), - waitingTime - ) - + mDevice.waitNotNull(Until.findObject(By.res("$packageName:id/toolbar")), waitingTime) urlBar().click() awesomeBar().perform(replaceText(url.toString()), pressImeActionButton()) diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/SearchRobot.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/SearchRobot.kt index 31efbbb8d..5f565b257 100644 --- a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/SearchRobot.kt +++ b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/SearchRobot.kt @@ -38,6 +38,7 @@ import org.mozilla.fenix.R import org.mozilla.fenix.helpers.SessionLoadedIdlingResource import org.mozilla.fenix.helpers.TestAssetHelper import org.mozilla.fenix.helpers.TestAssetHelper.waitingTime +import org.mozilla.fenix.helpers.TestHelper.packageName import org.mozilla.fenix.helpers.click import org.mozilla.fenix.helpers.ext.waitNotNull @@ -70,7 +71,7 @@ class SearchRobot { fun clickSearchEngineShortcutButton() { val searchEnginesShortcutButton = mDevice.findObject(UiSelector() - .resourceId("org.mozilla.fenix.debug:id/search_engines_shortcut_button")) + .resourceId("$packageName:id/search_engines_shortcut_button")) searchEnginesShortcutButton.waitForExists(waitingTime) searchEnginesShortcutButton.click() } diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/SettingsSubMenuPrivateBrowsingRobot.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/SettingsSubMenuPrivateBrowsingRobot.kt index 624f6b453..5ce0944af 100644 --- a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/SettingsSubMenuPrivateBrowsingRobot.kt +++ b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/SettingsSubMenuPrivateBrowsingRobot.kt @@ -125,9 +125,6 @@ private fun assertOpenLinksInPrivateTabOff() { } private fun assertPrivateBrowsingShortcutIcon() { - mDevice.wait( - Until.findObject(text("Private Firefox Preview")), - waitingTime - ) + mDevice.wait(Until.findObject(text("Private Firefox Preview")), waitingTime) assertTrue(mDevice.hasObject(text("Private Firefox Preview"))) } diff --git a/automation/taskcluster/androidTest/flank-x86-beta.yml b/automation/taskcluster/androidTest/flank-x86-beta.yml new file mode 100644 index 000000000..7ad86bbd8 --- /dev/null +++ b/automation/taskcluster/androidTest/flank-x86-beta.yml @@ -0,0 +1,57 @@ +# gcloud args match the official gcloud cli +# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +gcloud: + results-bucket: fenix_test_artifacts + record-video: true + + # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. + timeout: 20m + # will start test then close socket. no reports will be generated. + # to retrieve results later, use the "refresh" command + # reports will be generated from /results/matrix_ids.json + #async: true + # will start test then leave socket open. reports will be published + # to /results + # see: https://github.com/TestArmada/flank/issues/339 + async: false + + # results-history-name + # by default, set to app name + # declare results-history-name to create a separate dropdown menu in Firebase + # see: https://github.com/TestArmada/flank/issues/341 + #results-history-name: tmp_parallel + + # The number of times a test execution should be re-attempted if one or more failures occur. + # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. + num-flaky-test-attempts: 1 + + # test and app are the only required args + app: /app/path + test: /test/path + + auto-google-login: true + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + test-targets: + # Let's start by running a single test, then if stable change to smoketest + - class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest + # - class org.mozilla.fenix.ui.SmokeTest + + device: + - model: Pixel2 + version: 28 + +flank: + project: GOOGLE_PROJECT + # test shards - the amount of groups to split the test suite into + # set to -1 to use one shard per test. + max-test-shards: 1 + # num-test-runs: the amount of times to run the tests. + # 1 runs the tests once. 10 runs all the tests 10x + num-test-runs: 1 + diff --git a/automation/taskcluster/androidTest/ui-test.sh b/automation/taskcluster/androidTest/ui-test.sh index e45e9b8cc..8e02e4be0 100755 --- a/automation/taskcluster/androidTest/ui-test.sh +++ b/automation/taskcluster/androidTest/ui-test.sh @@ -82,6 +82,8 @@ elif [[ "${device_type}" == "arm-start-test" ]]; then flank_template="${PATH_TEST}/flank-armeabi-v7a-start-test.yml" elif [[ "${device_type}" == "x86-screenshots-tests" ]]; then flank_template="${PATH_TEST}/flank-x86-screenshots-tests.yml" +elif [[ "${device_type}" == "x86-beta-tests" ]]; then + flank_template="${PATH_TEST}/flank-x86-beta.yml" else echo "FAILURE: flank config file not found!" exitcode=1 diff --git a/taskcluster/ci/ui-test/kind.yml b/taskcluster/ci/ui-test/kind.yml index 2febaa7bc..f4c7ed80b 100644 --- a/taskcluster/ci/ui-test/kind.yml +++ b/taskcluster/ci/ui-test/kind.yml @@ -49,7 +49,7 @@ jobs: x86-debug: description: Test Fenix run-on-tasks-for: [github-pull-request, github-push] - run-on-git-branches: [master] + run-on-git-branches: ["^((?!releases[_/]).+)$"] run: commands: - [automation/taskcluster/androidTest/ui-test.sh, x86, app.apk, android-test.apk, '50'] @@ -66,3 +66,17 @@ jobs: - [automation/taskcluster/androidTest/ui-test.sh, x86-screenshots-tests, app.apk, android-test.apk, '-1'] treeherder: symbol: debug(screenshots-x86) + x86-beta: + attributes: + build-type: beta-firebase + description: Test Fenix + run-on-tasks-for: [github-push] + run-on-git-branches: ["^releases[/_].+$"] + dependencies: + signing: signing-beta-firebase + signing-android-test: signing-android-test-beta + run: + commands: + - [automation/taskcluster/androidTest/ui-test.sh, x86-beta-tests, app.apk, android-test.apk, '50'] + treeherder: + symbol: beta(ui-test-x86-beta)