-
Notifications
You must be signed in to change notification settings - Fork 22
183 lines (159 loc) · 7.26 KB
/
Copy pathNightlyDispatcher.yml
File metadata and controls
183 lines (159 loc) · 7.26 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
name: Nightly Dispatcher
permissions:
actions: write
contents: read
on:
schedule:
# Run main branch at 10:00 PM (22:00 UTC)
- cron: '0 22 * * *'
# Run version/mx/10 branch at 4:00 AM UTC
- cron: '0 4 * * *'
jobs:
# This job determines which branch to target based on the schedule
determine-target:
runs-on: ubuntu-latest
outputs:
target-branch: ${{ steps.set-branch.outputs.branch }}
run-name: ${{ steps.set-branch.outputs.run-name }}
steps:
- id: set-branch
run: |
if [[ "${{ github.event.schedule }}" == "0 22 * * *" ]]; then
echo "branch=main" >> $GITHUB_OUTPUT
echo "run-name=Nightly Main Branch Pipeline" >> $GITHUB_OUTPUT
else
echo "branch=version/mx/10" >> $GITHUB_OUTPUT
echo "run-name=Nightly version/mx/10 Branch Pipeline" >> $GITHUB_OUTPUT
fi
# This single job handles dispatching to the correct branch
dispatch-pipeline:
needs: determine-target
runs-on: ubuntu-latest
outputs:
run-id: ${{ steps.trigger.outputs.run_id }}
target-branch: ${{ needs.determine-target.outputs.target-branch }}
steps:
- name: "Trigger Native Pipeline on ${{ needs.determine-target.outputs.target-branch }}"
id: trigger
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
with:
script: |
const targetBranch = '${{ needs.determine-target.outputs.target-branch }}';
const runName = '${{ needs.determine-target.outputs.run-name }}';
console.log(`Triggering workflow on branch: ${targetBranch}`);
const response = await github.rest.actions.createWorkflowDispatch({
owner: context.repo.owner,
repo: context.repo.repo,
workflow_id: 'NativePipeline.yml',
ref: targetBranch,
inputs: {
run_name: runName,
workspace: '*-native'
}
});
console.log('Workflow dispatch triggered successfully');
// Wait longer and find the correct run
let runId = null;
let attempts = 0;
const maxAttempts = 18; // 3 minutes
while (!runId && attempts < maxAttempts) {
await new Promise(resolve => setTimeout(resolve, 10000)); // Wait 10 seconds
attempts++;
console.log(`Attempt ${attempts}/${maxAttempts} to find triggered run...`);
try {
const runs = await github.rest.actions.listWorkflowRuns({
owner: context.repo.owner,
repo: context.repo.repo,
workflow_id: 'NativePipeline.yml',
branch: targetBranch,
per_page: 10
});
// Find run created in the last 10 minutes
const tenMinutesAgo = new Date();
tenMinutesAgo.setMinutes(tenMinutesAgo.getMinutes() - 10);
const recentRun = runs.data.workflow_runs.find(run => {
const runDate = new Date(run.created_at);
return runDate > tenMinutesAgo && run.event === 'workflow_dispatch';
});
if (recentRun) {
runId = recentRun.id;
console.log(`Found triggered run ID: ${runId}`);
break;
}
} catch (error) {
console.log(`Error finding run: ${error.message}`);
}
}
if (!runId) {
console.log('Could not find the triggered workflow run - will continue without monitoring');
core.setOutput('dispatch_success', 'false');
core.setOutput('run_id', '');
return;
}
core.setOutput('dispatch_success', 'true');
core.setOutput('run_id', runId);
return runId;
auto-retry:
needs: [dispatch-pipeline, determine-target]
if: always() && needs.dispatch-pipeline.outputs.run-id != ''
runs-on: ubuntu-latest
steps:
- name: "Monitor and retry failed jobs"
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
with:
script: |
const runId = '${{ needs.dispatch-pipeline.outputs.run-id }}';
const targetBranch = '${{ needs.dispatch-pipeline.outputs.target-branch }}';
if (!runId || runId === 'null' || runId === '') {
console.log('No run ID available from dispatch job - skipping monitoring');
return;
}
console.log(`Starting monitoring for run ID: ${runId} on branch ${targetBranch}`);
// Poll for completion with timeout
let run;
let pollAttempts = 0;
const maxPollAttempts = 240; // 4 hours max
do {
if (pollAttempts >= maxPollAttempts) {
console.log('Monitoring timeout reached (4 hours). Stopping.');
return;
}
await new Promise(resolve => setTimeout(resolve, 60000)); // Wait 1 minute
pollAttempts++;
try {
run = await github.rest.actions.getWorkflowRun({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: runId
});
console.log(`Poll #${pollAttempts}: Run status: ${run.data.status}, conclusion: ${run.data.conclusion || 'N/A'}, attempt: ${run.data.run_attempt}`);
} catch (error) {
console.log(`Error getting run status: ${error.message}`);
continue;
}
} while (run.data.status === 'in_progress' || run.data.status === 'queued');
// Check if we should retry
if (run.data.conclusion === 'failure') {
console.log(`Pipeline failed on attempt ${run.data.run_attempt}. Checking if retry needed...`);
// Only retry if this is the first attempt
if (run.data.run_attempt === 1) {
console.log(`Triggering retry for failed jobs on branch ${targetBranch}...`);
try {
await github.rest.actions.reRunWorkflowFailedJobs({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: runId
});
console.log('Retry triggered successfully for failed jobs only.');
} catch (error) {
console.error(`Failed to trigger retry: ${error.message}`);
console.log('Detailed error:', JSON.stringify(error, null, 2));
}
} else {
console.log('This was already a retry attempt. Not retrying again.');
}
} else if (run.data.conclusion === 'success') {
console.log('Pipeline completed successfully!');
} else {
console.log(`Pipeline completed with conclusion: ${run.data.conclusion}`);
}