Skip to content

Commit 932adb8

Browse files
malmsteinmalmstein
committed
update test worflow
1 parent 78a056a commit 932adb8

File tree

2 files changed

+219
-47
lines changed

2 files changed

+219
-47
lines changed

‎.github/workflows/release_tests.yml‎

Lines changed: 186 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,198 @@
1-
name: Release - Full Release Process
1+
name: Release - Run E2E Maestro Tests
22

33
on:
44
workflow_dispatch:
55
inputs:
66
app-version:
7-
description: 'App Version for Release'
7+
description: 'App Version for Testing'
88
required: true
99
default: 'PLACEHOLDER'
10+
test-tag:
11+
description: 'Maestro Tests tag to include'
12+
required: true
13+
default: 'releaseTest'
1014

1115
env:
12-
emoji_start: ":flight_departure:" # 🛫
16+
ASANA_PAT: ${{ secrets.ASANA_ACCESS_TOKEN }}
1317
emoji_info: ":information_source:" # ℹ️
14-
15-
concurrency:
16-
group: ${{ github.workflow }}-${{ github.event.inputs.app-version }}
17-
cancel-in-progress: true
18+
emoji_success: ":white_check_mark:" # ✅
19+
emoji_failure: ":x:" # ❌
1820

1921
jobs:
20-
notify_mattermost_start:
21-
name: 'Step 1: Notify Mattermost of Release start'
22-
uses: ./.github/workflows/notify_mattermost.yml
23-
with:
24-
mattermost-message: ${{env.emoji_start}} Starting release process for version ${{ github.event.inputs.app-version }}
25-
mattermost-channel-name: ${{ vars.MM_RELEASE_NOTIFY_CHANNEL }}
26-
secrets: inherit
27-
28-
run_release_tests:
29-
name: 'Step 3: Run Release Tests'
30-
needs: notify_mattermost_start
31-
uses: ./.github/workflows/release_tests.yml
32-
with:
33-
app-version: ${{ github.event.inputs.app-version }}
34-
test-tag: 'omnibarTest'
35-
secrets: inherit
36-
37-
notify_mattermost_finish:
38-
name: 'Step 1: Notify Mattermost of Release completed'
39-
uses: ./.github/workflows/notify_mattermost.yml
40-
needs: run_release_tests
41-
with:
42-
mattermost-message: ${{env.emoji_end}} Release ${{ github.event.inputs.app-version }} completed successfully and is now in review.
43-
mattermost-channel-name: ${{ vars.MM_RELEASE_NOTIFY_CHANNEL }}
44-
secrets: inherit
22+
run-release-tests:
23+
name: Create release APK and run E2E Maestro tests
24+
runs-on: ubuntu-latest
25+
steps:
26+
- name: Checkout repository
27+
uses: actions/checkout@v4
28+
with:
29+
submodules: recursive
30+
ref: ${{ github.event.inputs.app-version }}
31+
32+
- name: Set up JDK version
33+
uses: actions/setup-java@v4
34+
with:
35+
java-version-file: .github/.java-version
36+
distribution: 'adopt'
37+
38+
- name: Create folder
39+
if: always()
40+
run: mkdir apk
41+
42+
- name: Decode keys
43+
uses: davidSchuppa/base64Secret-toFile-action@v2
44+
with:
45+
secret: ${{ secrets.FAKE_RELEASE_PROPERTIES }}
46+
fileName: ddg_android_build.properties
47+
destination-path: $HOME/jenkins_static/com.duckduckgo.mobile.android/
48+
49+
- name: Decode key file
50+
uses: davidSchuppa/base64Secret-toFile-action@v2
51+
with:
52+
secret: ${{ secrets.FAKE_RELEASE_KEY }}
53+
fileName: android
54+
destination-path: $HOME/jenkins_static/com.duckduckgo.mobile.android/
55+
56+
- name: Setup Gradle
57+
uses: gradle/actions/setup-gradle@v3
58+
59+
- name: Assemble release APK
60+
run: ./gradlew assemblePlayRelease -Pforce-default-variant -Pskip-onboarding
61+
62+
- name: Move APK to new folder
63+
if: always()
64+
run: find . -name "*.apk" -exec mv '{}' apk/release.apk \;
65+
66+
- name: Notify Mattermost of Maestro tests
67+
id: send-mm-tests-started
68+
uses: duckduckgo/[email protected]
69+
with:
70+
mattermost-token: ${{ secrets.MM_AUTH_TOKEN }}
71+
mattermost-team-id: ${{ secrets.MM_TEAM_ID }}
72+
mattermost-channel-name: ${{ vars.MM_RELEASE_NOTIFY_CHANNEL }}
73+
mattermost-message: ${{env.emoji_info}} Release ${{ github.event.inputs.app-version }}. Running Maestro tests for tag ${{ github.event.inputs.test-tag }} https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }}
74+
action: 'send-mattermost-message'
75+
76+
- name: Maestro tests flows
77+
id: release-tests
78+
uses: mobile-dev-inc/[email protected]
79+
timeout-minutes: 120
80+
with:
81+
api-key: ${{ secrets.ROBIN_API_KEY }}
82+
project-id: ${{ vars.ROBIN_ANDROID_PROJECT_ID }}
83+
timeout: ${{ vars.ROBIN_TIMEOUT_MINUTES }}
84+
app-file: apk/release.apk
85+
android-api-level: 30
86+
workspace: .maestro
87+
include-tags: ${{ github.event.inputs.test-tag }}
88+
89+
- name: Analyze Maestro Flow Results
90+
id: analyze-flow-results
91+
if: always()
92+
run: |
93+
echo "Console URL: ${{ steps.release-tests.outputs.MAESTRO_CLOUD_CONSOLE_URL }}"
94+
echo "Upload Status: ${{ steps.release-tests.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }}"
95+
echo "App Binary ID: ${{ steps.release-tests.outputs.MAESTRO_CLOUD_APP_BINARY_ID }}"
96+
97+
flow_results_json='${{ steps.release-tests.outputs.MAESTRO_CLOUD_FLOW_RESULTS }}'
98+
echo "Raw Flow Results JSON: $flow_results_json"
99+
100+
# Default to success, change if issues are found
101+
final_status="success"
102+
103+
# Check for empty or invalid JSON (though Maestro action should provide valid JSON)
104+
if ! echo "$flow_results_json" | jq -e . > /dev/null 2>&1; then
105+
echo "::warning::MAESTRO_CLOUD_FLOW_RESULTS is not valid JSON or is empty."
106+
final_status="unknown_format"
107+
else
108+
# Check for any flow with status "ERROR"
109+
if echo "$flow_results_json" | jq -e '.[] | select(.status=="ERROR")' > /dev/null; then
110+
echo "::error::At least one Maestro flow has status: ERROR."
111+
final_status="failure"
112+
fi
113+
114+
# Check for any flow with status "CANCELED"
115+
# You might decide if CANCELED flows also mean the overall status is a failure for your release
116+
if echo "$flow_results_json" | jq -e '.[] | select(.status=="CANCELED")' > /dev/null; then
117+
echo "::warning::At least one Maestro flow has status: CANCELED."
118+
# If any canceled flow makes the whole thing a failure:
119+
if [ "$final_status" != "failure" ]; then # Don't override if already a critical failure
120+
final_status="canceled_present" # Or treat as "failure" if preferred
121+
fi
122+
fi
123+
124+
# If after all checks, final_status is still "success", it means no "ERROR" or "CANCELED"
125+
if [ "$final_status" == "success" ]; then
126+
# Additional check: ensure there's at least one flow and it's not empty array if that's a concern
127+
if echo "$flow_results_json" | jq -e '. | length > 0' > /dev/null; then
128+
echo "All flows appear to be successful (no ERROR or CANCELED statuses found that are treated as errors)."
129+
else
130+
echo "::warning::MAESTRO_CLOUD_FLOW_RESULTS is an empty array. No flows reported."
131+
final_status="empty_results" # Or "success" if empty results are acceptable
132+
fi
133+
fi
134+
fi
135+
136+
echo "Final determined status: $final_status"
137+
echo "flow_summary_status=$final_status" >> $GITHUB_OUTPUT
138+
139+
- name: Access Outputs (for debugging)
140+
if: always()
141+
run: |
142+
echo "Console URL: ${{ steps.release-tests.outputs.MAESTRO_CLOUD_CONSOLE_URL }}"
143+
echo 'Flow Results (JSON): ${{ steps.release-tests.outputs.MAESTRO_CLOUD_FLOW_RESULTS }}'
144+
echo "Release Tests Step Conclusion: ${{ steps.release-tests.conclusion }}" # From Maestro action itself
145+
echo "Analyzed Flow Summary Status: ${{ steps.analyze-flow-results.outputs.flow_summary_status }}" # From our script
146+
147+
- name: Notify Mattermost - Maestro Tests ALL SUCCEEDED
148+
# Condition 1: Our script says success
149+
# Condition 2: The Maestro action itself also reported overall success
150+
if: always() && steps.analyze-flow-results.outputs.flow_summary_status == 'success' && steps.release-tests.conclusion == 'success'
151+
uses: duckduckgo/[email protected]
152+
with:
153+
action: 'send-mattermost-message'
154+
mattermost-token: ${{ secrets.MM_AUTH_TOKEN }}
155+
mattermost-team-id: ${{ secrets.MM_TEAM_ID }}
156+
mattermost-channel-name: ${{ vars.MM_RELEASE_NOTIFY_CHANNEL }}
157+
mattermost-message: |
158+
${{env.emoji_success}} Release ${{ github.event.inputs.app-version }}: Tests for tag ${{ github.event.inputs.test-tag }} PASSED successfully.
159+
Console: ${{ steps.release-tests.outputs.MAESTRO_CLOUD_CONSOLE_URL }}
160+
161+
- name: Notify Mattermost - Maestro Tests FAILURES or ISSUES DETECTED
162+
# Condition: Our script detected 'failure' OR the Maestro action itself reported failure
163+
if: always() && (steps.analyze-flow-results.outputs.flow_summary_status == 'failure' || steps.release-tests.conclusion == 'failure')
164+
uses: duckduckgo/[email protected]
165+
with:
166+
action: 'send-mattermost-message'
167+
mattermost-token: ${{ secrets.MM_AUTH_TOKEN }}
168+
mattermost-team-id: ${{ secrets.MM_TEAM_ID }}
169+
mattermost-channel-name: ${{ vars.MM_RELEASE_NOTIFY_CHANNEL }}
170+
mattermost-message: |
171+
${{env.emoji_failure}} Release ${{ github.event.inputs.app-version }}: Tests for tag ${{ github.event.inputs.test-tag }} FAILED or encountered issues.
172+
Console: ${{ steps.release-tests.outputs.MAESTRO_CLOUD_CONSOLE_URL }}
173+
174+
- name: Notify Mattermost - Maestro Tests CANCELED Flows Present (Informational or Warning)
175+
# Condition: Our script detected 'canceled_present' AND no critical 'failure' was found
176+
# AND Maestro action itself didn't mark the whole run as a 'failure'
177+
if: always() && steps.analyze-flow-results.outputs.flow_summary_status == 'canceled_present' && steps.release-tests.conclusion != 'failure'
178+
uses: duckduckgo/[email protected]
179+
with:
180+
action: 'send-mattermost-message'
181+
mattermost-token: ${{ secrets.MM_AUTH_TOKEN }}
182+
mattermost-team-id: ${{ secrets.MM_TEAM_ID }}
183+
mattermost-channel-name: ${{ vars.MM_RELEASE_NOTIFY_CHANNEL }}
184+
mattermost-message: |
185+
:warning: Release ${{ github.event.inputs.app-version }}: Some tests for tag ${{ github.event.inputs.test-tag }} were CANCELED. Please review.
186+
Console: ${{ steps.release-tests.outputs.MAESTRO_CLOUD_CONSOLE_URL }}
187+
188+
- name: Create Asana task when workflow failed
189+
if: ${{ failure() }}
190+
id: create-failure-task
191+
uses: duckduckgo/[email protected]
192+
with:
193+
asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }}
194+
asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }}
195+
asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }}
196+
asana-task-name: GH Workflow Failure - Tag Android Release (Robin)
197+
asana-task-description: Run Release Tests in Maestro has failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }}
198+
action: 'create-asana-task'

‎.github/workflows/test_workflow.yml‎

Lines changed: 33 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,25 +2,43 @@ name: Test - Workflow for Testing
22

33
on:
44
workflow_dispatch:
5-
65
inputs:
7-
ref:
8-
description: 'This is an example of an input'
6+
app-version:
7+
description: 'App Version for Release'
98
required: true
9+
default: 'PLACEHOLDER'
1010

1111
env:
12-
ASANA_PAT: ${{ secrets.ASANA_ACCESS_TOKEN }}
13-
GH_TOKEN: ${{ secrets.GT_DAXMOBILE }}
12+
emoji_start: ":flight_departure:" # 🛫
13+
emoji_info: ":information_source:" # ℹ️
14+
15+
concurrency:
16+
group: ${{ github.workflow }}-${{ github.event.inputs.app-version }}
17+
cancel-in-progress: true
1418

1519
jobs:
16-
test-workflow:
17-
runs-on: ubuntu-latest
18-
name: Add here whatever steps you want to test
20+
notify_mattermost_start:
21+
name: 'Step 1: Notify Mattermost of Release start'
22+
uses: ./.github/workflows/notify_mattermost.yml
23+
with:
24+
mattermost-message: ${{env.emoji_start}} Starting release process for version ${{ github.event.inputs.app-version }}
25+
mattermost-channel-name: ${{ vars.MM_TEST_NOTIFY_CHANNEL }}
26+
secrets: inherit
27+
28+
run_release_tests:
29+
name: 'Step 3: Run Release Tests'
30+
needs: notify_mattermost_start
31+
uses: ./.github/workflows/release_tests.yml
32+
with:
33+
app-version: ${{ github.event.inputs.app-version }}
34+
test-tag: 'omnibarTest'
35+
secrets: inherit
1936

20-
steps:
21-
- name: Checkout repository
22-
uses: actions/checkout@v4
23-
with:
24-
submodules: recursive
25-
token: ${{ secrets.GT_DAXMOBILE }}
26-
ref: ${{ github.event.inputs.ref }}
37+
notify_mattermost_finish:
38+
name: 'Step 1: Notify Mattermost of Release completed'
39+
uses: ./.github/workflows/notify_mattermost.yml
40+
needs: run_release_tests
41+
with:
42+
mattermost-message: ${{env.emoji_end}} Release ${{ github.event.inputs.app-version }} completed successfully and is now in review.
43+
mattermost-channel-name: ${{ vars.MM_TEST_NOTIFY_CHANNEL }}
44+
secrets: inherit

0 commit comments

Comments
 (0)