From fa6b6038046ab057c38b15399cff2a5438723d47 Mon Sep 17 00:00:00 2001 From: Jackie Johnson <107960801+jjSDET@users.noreply.github.com> Date: Fri, 2 Feb 2024 02:02:59 -0600 Subject: [PATCH] Bug 1873897 - Update Testrail Milestone Release Trigger and Slack Message --- .../androidTest/lib/testrail_conn.py | 104 ---------- .../taskcluster/androidTest/testrail.py | 180 ------------------ automation/taskcluster/androidTest/ui-test.sh | 9 - 3 files changed, 293 deletions(-) delete mode 100644 automation/taskcluster/androidTest/lib/testrail_conn.py delete mode 100644 automation/taskcluster/androidTest/testrail.py diff --git a/automation/taskcluster/androidTest/lib/testrail_conn.py b/automation/taskcluster/androidTest/lib/testrail_conn.py deleted file mode 100644 index e2e18dd9f..000000000 --- a/automation/taskcluster/androidTest/lib/testrail_conn.py +++ /dev/null @@ -1,104 +0,0 @@ -# flake8: noqa -"""TestRail API binding for Python 3.x. - -(API v2, available since TestRail 3.0) - -Compatible with TestRail 3.0 and later. - -Learn more: - -http://docs.gurock.com/testrail-api2/start -http://docs.gurock.com/testrail-api2/accessing - -Copyright Gurock Software GmbH. See license.md for details. -""" - -import base64 -import json - -import requests - - -class APIClient: - def __init__(self, base_url): - self.user = "" - self.password = "" - if not base_url.endswith("/"): - base_url += "/" - self.__url = base_url + "index.php?/api/v2/" - - def send_get(self, uri, filepath=None): - """Issue a GET request (read) against the API. - - Args: - uri: The API method to call including parameters, e.g. get_case/1. - filepath: The path and file name for attachment download; used only - for 'get_attachment/:attachment_id'. - - Returns: - A dict containing the result of the request. - """ - return self.__send_request("GET", uri, filepath) - - def send_post(self, uri, data): - """Issue a POST request (write) against the API. - - Args: - uri: The API method to call, including parameters, e.g. add_case/1. - data: The data to submit as part of the request as a dict; strings - must be UTF-8 encoded. If adding an attachment, must be the - path to the file. - - Returns: - A dict containing the result of the request. - """ - return self.__send_request("POST", uri, data) - - def __send_request(self, method, uri, data): - url = self.__url + uri - - auth = str( - base64.b64encode(bytes("%s:%s" % (self.user, self.password), "utf-8")), - "ascii", - ).strip() - headers = {"Authorization": "Basic " + auth} - - if method == "POST": - if uri[:14] == "add_attachment": # add_attachment API method - files = {"attachment": (open(data, "rb"))} - response = requests.post(url, headers=headers, files=files) - files["attachment"].close() - else: - headers["Content-Type"] = "application/json" - payload = bytes(json.dumps(data), "utf-8") - response = requests.post(url, headers=headers, data=payload) - else: - headers["Content-Type"] = "application/json" - response = requests.get(url, headers=headers) - - if response.status_code > 201: - try: - error = response.json() - except ( - requests.exceptions.HTTPError - ): # response.content not formatted as JSON - error = str(response.content) - raise APIError( - "TestRail API returned HTTP %s (%s)" % (response.status_code, error) - ) - else: - if uri[:15] == "get_attachment/": # Expecting file, not JSON - try: - open(data, "wb").write(response.content) - return data - except FileNotFoundError: - return "Error saving attachment." - else: - try: - return response.json() - except requests.exceptions.HTTPError: - return {} - - -class APIError(Exception): - pass diff --git a/automation/taskcluster/androidTest/testrail.py b/automation/taskcluster/androidTest/testrail.py deleted file mode 100644 index e735e417a..000000000 --- a/automation/taskcluster/androidTest/testrail.py +++ /dev/null @@ -1,180 +0,0 @@ -""" -This Python script is designed to automate the process of creating milestones -and test runs in TestRail, and updating test cases based on the results of -automated smoke tests for different product releases. - -Below is a summary of its functionality in order of execution: - -1. Environment and Credentials Setup: - - Imports necessary libraries and modules. - - Loads environmental variables from "execution_metadata.env". - - Reads and processes TestRail credentials from '.testrail_credentials.json'. - -2. Environment Variables Validation: - - Retrieves and validates several environment variables like `PRODUCT_TYPE`, - `RELEASE_TYPE`, `VERSION_NUMBER`, and `TEST_STATUS`. - - Ensures `TEST_STATUS` is either 'PASS' or 'FAIL'. - -3. Utility Functions: - - `parse_release_number()`: Parses the version number to extract a specific part. - - `build_milestone_name()`: Constructs a milestone name based on product type, - release type, and version number. - - `build_milestone_description()`: Creates a detailed description for the milestone - including the current date and placeholders for various testing statuses. - -4. TestRail Integration: - - Defines a `TestRail` class that handles interactions with the TestRail API. - - Includes methods to create milestones, create test runs, and update test cases. - -5. Main Execution: - - Checks if `TEST_STATUS` is 'PASS'. If not, it raises an error to trigger a Slack notification. - - Sets parameters for a demo TestRail project. - - Instantiates the `TestRail` class. - - Creates a milestone in TestRail and retrieves its ID. - - Creates test runs for each device/API combination (currently hardcoded for phase 1 testing) - and updates test cases to 'passed' status. - -6. Phase 1 and Phase 2 Notes: - - The script is currently in Phase 1, where certain values are hardcoded for testing. - - In Phase 2, these hardcoded values will be parameterized for broader usage. -""" - - -import json -import os -import textwrap -from lib.testrail_conn import APIClient -from dotenv import load_dotenv -from datetime import datetime - -try: - load_dotenv("execution_metadata.env") # Attempt to load .env file -except FileNotFoundError: - raise FileNotFoundError("The .env file was not found.") -except Exception as e: - raise Exception(f"An error occurred while loading the .env file: {e}") - -try: - with open(".testrail_credentials.json", "r") as file: - secret = json.load(file) - TESTRAIL_HOST = secret["host"] - TESTRAIL_USERNAME = secret["username"] - TESTRAIL_PASSWORD = secret["password"] -except json.JSONDecodeError as e: - raise ValueError(f"Failed to load testrail credentials : {e}") - -try: - PRODUCT_TYPE = os.environ["PRODUCT_TYPE"] - RELEASE_TYPE = os.environ["RELEASE_TYPE"] - VERSION_NUMBER = os.environ["MOBILE_HEAD_REF"] - TEST_STATUS = os.environ["TEST_STATUS"] - - if TEST_STATUS not in ("PASS", "FAIL"): - raise ValueError(f"ERROR: Invalid TEST_STATUS value: {TEST_STATUS}") -except KeyError as e: - raise ValueError(f"ERROR: Missing Environment Variable: {e}") - - -def parse_release_number(VERSION_NUMBER): - parts = VERSION_NUMBER.split("_") - return parts[1] - - -def build_milestone_name(product_type, release_type, version_number): - return f"Automated smoke testing sign-off - {product_type} {release_type} {version_number}" - - -def build_milestone_description(milestone_name): - current_date = datetime.now() - formatted_date = current_date = current_date.strftime("%B %d, %Y") - return textwrap.dedent( - f""" - RELEASE: {milestone_name}\n\n\ - RELEASE_TAG_URL: https://github.com/mozilla-mobile/firefox-android/releases\n\n\ - RELEASE_DATE: {formatted_date}\n\n\ - TESTING_STATUS: [ TBD ]\n\n\ - QA_RECOMMENDATION:[ TBD ]\n\n\ - QA_RECOMENTATION_VERBOSE: \n\n\ - TESTING_SUMMARY\n\n\ - Known issues: n/a\n\ - New issue: n/a\n\ - Verified issue: - """ - ) - - -class TestRail: - def __init__(self): - try: - self.client = APIClient(TESTRAIL_HOST) - self.client.user = TESTRAIL_USERNAME - self.client.password = TESTRAIL_PASSWORD - except KeyError as e: - raise ValueError(f"ERROR: Missing Testrail Env Var: {e}") - - # Public Methods - - def create_milestone(self, testrail_project_id, title, description): - data = {"name": title, "description": description} - return self.client.send_post(f"add_milestone/{testrail_project_id}", data) - - def create_test_run( - self, testrail_project_id, testrail_milestone_id, name_run, testrail_suite_id - ): - data = { - "name": name_run, - "milestone_id": testrail_milestone_id, - "suite_id": testrail_suite_id, - } - return self.client.send_post(f"add_run/{testrail_project_id}", data) - - def update_test_cases_to_passed( - self, testrail_project_id, testrail_run_id, testrail_suite_id - ): - test_cases = self._get_test_cases(testrail_project_id, testrail_suite_id) - data = { - "results": [ - {"case_id": test_case["id"], "status_id": 1} for test_case in test_cases - ] - } - return testrail._update_test_run_results(testrail_run_id, data) - - # Private Methods - - def _get_test_cases(self, testrail_project_id, testrail_test_suite_id): - return self.client.send_get( - f"get_cases/{testrail_project_id}&suite_id={testrail_test_suite_id}" - ) - - def _update_test_run_results(self, testrail_run_id, data): - return self.client.send_post(f"add_results_for_cases/{testrail_run_id}", data) - - -if __name__ == "__main__": - if TEST_STATUS != "PASS": - raise ValueError("Tests failed. Sending Slack Notification....") - - # There are for a dummy Testrail project used for Phase 1 testing of this script - # They will be parameterized during Phase 2 of script hardening - PROJECT_ID = 53 # Firefox for FireTV - TEST_SUITE_ID = 45442 # Demo Test Suite - - testrail = TestRail() - milestone_name = build_milestone_name( - PRODUCT_TYPE, RELEASE_TYPE, parse_release_number(VERSION_NUMBER) - ) - milestone_description = build_milestone_description(milestone_name) - - # Create milestone for 'Firefox for FireTV' and store the ID - milestone_id = testrail.create_milestone( - PROJECT_ID, milestone_name, milestone_description - )["id"] - - # Create test run for each Device/API and update test cases to 'passed' - # The Firebase Test devices are temporarily hard-coded during testing - # and will be parameterized in Phase 2 of hardening - for test_run_name in ["Google Pixel 32(Android11)", "Google Pixel2(Android9)"]: - test_run_id = testrail.create_test_run( - PROJECT_ID, milestone_id, test_run_name, TEST_SUITE_ID - )["id"] - testrail.update_test_cases_to_passed(PROJECT_ID, test_run_id, TEST_SUITE_ID) diff --git a/automation/taskcluster/androidTest/ui-test.sh b/automation/taskcluster/androidTest/ui-test.sh index a4df3971b..3c5b29c17 100755 --- a/automation/taskcluster/androidTest/ui-test.sh +++ b/automation/taskcluster/androidTest/ui-test.sh @@ -102,18 +102,9 @@ function failure_check() { echo if [[ $exitcode -ne 0 ]]; then echo "FAILURE: UI test run failed, please check above URL" - TEST_STATUS="FAIL" else echo "All UI test(s) have passed!" - TEST_STATUS="PASS" fi - - { - echo "TEST_STATUS=${TEST_STATUS}" - echo "PRODUCT_TYPE=${PRODUCT_TYPE}" - echo "RELEASE_TYPE=${RELEASE_TYPE}" - } >> execution_metadata.env - echo echo "RESULTS" echo