Add taskcluster job for UI tests (#4088)
parent
48aeb19db1
commit
b39afe1548
@ -0,0 +1,56 @@
|
||||
# gcloud args match the official gcloud cli
|
||||
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
|
||||
gcloud:
|
||||
results-bucket: fenix_test_artifacts
|
||||
record-video: true
|
||||
|
||||
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
|
||||
timeout: 30m
|
||||
# will start test then close socket. no reports will be generated.
|
||||
# to retrieve results later, use the "refresh" command
|
||||
# reports will be generated from /results/matrix_ids.json
|
||||
#async: true
|
||||
# will start test then leave socket open. reports will be published
|
||||
# to /results
|
||||
# see: https://github.com/TestArmada/flank/issues/339
|
||||
async: false
|
||||
|
||||
# results-history-name
|
||||
# by default, set to app name
|
||||
# declare results-history-name to create a separate dropdown menu in Firebase
|
||||
# see: https://github.com/TestArmada/flank/issues/341
|
||||
#results-history-name: tmp_parallel
|
||||
|
||||
# test and app are the only required args
|
||||
app: /APP/PATH
|
||||
test: /TEST/PATH
|
||||
|
||||
auto-google-login: true
|
||||
use-orchestrator: true
|
||||
environment-variables:
|
||||
clearPackageData: true
|
||||
directories-to-pull:
|
||||
- /sdcard/screenshots
|
||||
performance-metrics: true
|
||||
|
||||
device:
|
||||
- model: shamu
|
||||
version: 21
|
||||
- model: sailfish
|
||||
version: 25
|
||||
- model: sailfish
|
||||
version: 28
|
||||
|
||||
flank:
|
||||
project: GOOGLE_PROJECT
|
||||
# test shards - the amount of groups to split the test suite into
|
||||
# set to -1 to use one shard per test.
|
||||
max-test-shards: -1
|
||||
# repeat tests - the amount of times to run the tests.
|
||||
# 1 runs the tests once. 10 runs all the tests 10x
|
||||
repeat-tests: 1
|
||||
# always run - these tests are inserted at the beginning of every shard
|
||||
# useful if you need to grant permissions or login before other tests run
|
||||
#test-targets-always-run:
|
||||
#- class com.example.app.ExampleUiTest#testPasses
|
||||
# - class org.mozilla.focus.activty.SwitchContextTest#testPasses
|
@ -0,0 +1,54 @@
|
||||
# gcloud args match the official gcloud cli
|
||||
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
|
||||
gcloud:
|
||||
results-bucket: fenix_test_artifacts
|
||||
record-video: true
|
||||
|
||||
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
|
||||
timeout: 30m
|
||||
# will start test then close socket. no reports will be generated.
|
||||
# to retrieve results later, use the "refresh" command
|
||||
# reports will be generated from /results/matrix_ids.json
|
||||
#async: true
|
||||
# will start test then leave socket open. reports will be published
|
||||
# to /results
|
||||
# see: https://github.com/TestArmada/flank/issues/339
|
||||
async: false
|
||||
|
||||
# results-history-name
|
||||
# by default, set to app name
|
||||
# declare results-history-name to create a separate dropdown menu in Firebase
|
||||
# see: https://github.com/TestArmada/flank/issues/341
|
||||
#results-history-name: tmp_parallel
|
||||
|
||||
# test and app are the only required args
|
||||
app: /APP/PATH
|
||||
test: /TEST/PATH
|
||||
|
||||
auto-google-login: true
|
||||
use-orchestrator: true
|
||||
environment-variables:
|
||||
clearPackageData: true
|
||||
directories-to-pull:
|
||||
- /sdcard/screenshots
|
||||
performance-metrics: true
|
||||
|
||||
device:
|
||||
- model: Nexus9
|
||||
version: 21
|
||||
- model: Nexus9
|
||||
version: 22
|
||||
|
||||
flank:
|
||||
project: GOOGLE_PROJECT
|
||||
# test shards - the amount of groups to split the test suite into
|
||||
# set to -1 to use one shard per test.
|
||||
max-test-shards: -1
|
||||
# repeat tests - the amount of times to run the tests.
|
||||
# 1 runs the tests once. 10 runs all the tests 10x
|
||||
repeat-tests: 1
|
||||
# always run - these tests are inserted at the beginning of every shard
|
||||
# useful if you need to grant permissions or login before other tests run
|
||||
#test-targets-always-run:
|
||||
#- class com.example.app.ExampleUiTest#testPasses
|
||||
# - class org.mozilla.focus.activty.SwitchContextTest#testPasses
|
@ -0,0 +1,52 @@
|
||||
# gcloud args match the official gcloud cli
|
||||
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
|
||||
gcloud:
|
||||
results-bucket: fenix_test_artifacts
|
||||
record-video: true
|
||||
|
||||
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
|
||||
timeout: 30m
|
||||
# will start test then close socket. no reports will be generated.
|
||||
# to retrieve results later, use the "refresh" command
|
||||
# reports will be generated from /results/matrix_ids.json
|
||||
#async: true
|
||||
# will start test then leave socket open. reports will be published
|
||||
# to /results
|
||||
# see: https://github.com/TestArmada/flank/issues/339
|
||||
async: false
|
||||
|
||||
# results-history-name
|
||||
# by default, set to app name
|
||||
# declare results-history-name to create a separate dropdown menu in Firebase
|
||||
# see: https://github.com/TestArmada/flank/issues/341
|
||||
#results-history-name: tmp_parallel
|
||||
|
||||
# test and app are the only required args
|
||||
app: /app/path
|
||||
test: /test/path
|
||||
|
||||
auto-google-login: true
|
||||
use-orchestrator: true
|
||||
environment-variables:
|
||||
clearPackageData: true
|
||||
directories-to-pull:
|
||||
- /sdcard/screenshots
|
||||
performance-metrics: true
|
||||
|
||||
device:
|
||||
- model: Nexus7
|
||||
version: 21
|
||||
|
||||
flank:
|
||||
project: GOOGLE_PROJECT
|
||||
# test shards - the amount of groups to split the test suite into
|
||||
# set to -1 to use one shard per test.
|
||||
max-test-shards: -1
|
||||
# repeat tests - the amount of times to run the tests.
|
||||
# 1 runs the tests once. 10 runs all the tests 10x
|
||||
repeat-tests: 1
|
||||
# always run - these tests are inserted at the beginning of every shard
|
||||
# useful if you need to grant permissions or login before other tests run
|
||||
#test-targets-always-run:
|
||||
#- class com.example.app.ExampleUiTest#testPasses
|
||||
# - class org.mozilla.focus.activty.SwitchContextTest#testPasses
|
@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env bash
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
# This script does the following:
|
||||
# 1. Retrieves gcloud service account token
|
||||
# 2. Activates gcloud service account
|
||||
# 3. Connects to google Firebase (using TestArmada's Flank tool)
|
||||
# 4. Executes UI tests
|
||||
# 5. Puts test artifacts into the test_artifacts folder
|
||||
|
||||
# NOTE:
|
||||
# Flank supports sharding across multiple devices at a time, but gcloud API
|
||||
# only supports 1 defined APK per test run.
|
||||
|
||||
|
||||
# If a command fails then do not proceed and fail this script too.
|
||||
set -e
|
||||
|
||||
#########################
|
||||
# The command line help #
|
||||
#########################
|
||||
display_help() {
|
||||
echo "Usage: $0 Build_Variant [Number_Shards...]"
|
||||
echo
|
||||
echo "Examples:"
|
||||
echo "To run UI tests on ARM device shard (1 test / shard)"
|
||||
echo "$ ui-test.sh arm -1"
|
||||
echo
|
||||
echo "To run UI tests on X86 device (on 3 shards)"
|
||||
echo "$ ui-test.sh feature x86 3"
|
||||
echo
|
||||
}
|
||||
|
||||
# Basic parameter check
|
||||
if [[ $# -lt 1 ]]; then
|
||||
echo "Error: please provide at least one build variant (arm|x86)"
|
||||
display_help
|
||||
exit 1
|
||||
fi
|
||||
|
||||
device_type="$1" # arm | x86
|
||||
if [[ ! -z "$2" ]]; then
|
||||
num_shards=$2
|
||||
fi
|
||||
|
||||
JAVA_BIN="/usr/bin/java"
|
||||
PATH_TEST="./automation/taskcluster/androidTest"
|
||||
FLANK_BIN="/build/test-tools/flank.jar"
|
||||
FLANK_CONF_ARM="${PATH_TEST}/flank-arm.yml"
|
||||
FLANK_CONF_X86="${PATH_TEST}/flank-x86.yml"
|
||||
|
||||
echo
|
||||
echo "RETRIEVE SERVICE ACCT TOKEN"
|
||||
echo
|
||||
python automation/taskcluster/helper/get-secret.py --json -s project/mobile/fenix/firebase -k firebaseToken -f $GOOGLE_APPLICATION_CREDENTIALS
|
||||
echo
|
||||
echo
|
||||
|
||||
echo
|
||||
echo "ACTIVATE SERVICE ACCT"
|
||||
echo
|
||||
# this is where the Google Testcloud project ID is set
|
||||
gcloud config set project "$GOOGLE_PROJECT"
|
||||
echo
|
||||
|
||||
gcloud auth activate-service-account --key-file "$GOOGLE_APPLICATION_CREDENTIALS"
|
||||
echo
|
||||
echo
|
||||
|
||||
# From now on disable exiting on error. If the tests fail we want to continue
|
||||
# and try to download the artifacts. We will exit with the actual error code later.
|
||||
set +e
|
||||
|
||||
if [[ "${device_type,,}" == "x86" ]]
|
||||
then
|
||||
deviceType="X86"
|
||||
flank_template="$FLANK_CONF_X86"
|
||||
else
|
||||
deviceType="Arm"
|
||||
flank_template="$FLANK_CONF_ARM"
|
||||
fi
|
||||
|
||||
APK_APP="./app/build/outputs/apk/${deviceType,,}/debug/app-${deviceType,,}-debug.apk"
|
||||
APK_TEST="./app/build/outputs/apk/androidTest/${deviceType,,}/debug/app-${deviceType,,}-debug-androidTest.apk"
|
||||
|
||||
|
||||
# function to exit script with exit code from test run.
|
||||
# (Only 0 if all test executions passed)
|
||||
function failure_check() {
|
||||
if [[ $exitcode -ne 0 ]]; then
|
||||
echo
|
||||
echo
|
||||
echo "ERROR: UI test run failed, please check above URL"
|
||||
fi
|
||||
exit $exitcode
|
||||
}
|
||||
|
||||
echo
|
||||
echo "EXECUTE TEST(S)"
|
||||
echo
|
||||
$JAVA_BIN -jar $FLANK_BIN android run --config=$flank_template --max-test-shards=$num_shards --app=$APK_APP --test=$APK_TEST --project=$GOOGLE_PROJECT
|
||||
exitcode=$?
|
||||
failure_check
|
||||
echo
|
||||
echo
|
||||
|
||||
echo
|
||||
echo "COPY ARTIFACTS"
|
||||
echo
|
||||
cp -r ./results ./test_artifacts
|
||||
exitcode=$?
|
||||
failure_check
|
||||
echo
|
||||
echo
|
||||
|
||||
echo
|
||||
echo "RESULTS"
|
||||
echo
|
||||
ls -la ./results
|
||||
echo
|
||||
echo
|
||||
|
||||
echo "All UI test(s) have passed!"
|
||||
echo
|
||||
echo
|
||||
|
||||
|
||||
|
Loading…
Reference in New Issue