Skip to content

Commit 3c0bfe9

Browse files
bircnisilverwindlunnywxiaoguang
authored
fix(actions): run every due schedule exactly once per occurrence (#39078)
Signed-off-by: Lunny Xiao <xiaolunwen@gmail.com> Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com> Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
1 parent fedf219 commit 3c0bfe9

6 files changed

Lines changed: 126 additions & 176 deletions

File tree

models/actions/schedule.go

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -40,17 +40,8 @@ func init() {
4040
db.RegisterModel(new(ActionSchedule))
4141
}
4242

43-
// GetSchedulesMapByIDs returns the schedules by given id slice.
44-
func GetSchedulesMapByIDs(ctx context.Context, ids []int64) (map[int64]*ActionSchedule, error) {
45-
schedules := make(map[int64]*ActionSchedule, len(ids))
46-
if len(ids) == 0 {
47-
return schedules, nil
48-
}
49-
return schedules, db.GetEngine(ctx).In("id", ids).Find(&schedules)
50-
}
51-
52-
// CreateScheduleTask creates new schedule task.
53-
func CreateScheduleTask(ctx context.Context, rows []*ActionSchedule) error {
43+
// CreateScheduleTaskBySchedules creates new schedule task.
44+
func CreateScheduleTaskBySchedules(ctx context.Context, rows []*ActionSchedule) error {
5445
// Return early if there are no rows to insert
5546
if len(rows) == 0 {
5647
return nil
@@ -74,13 +65,16 @@ func CreateScheduleTask(ctx context.Context, rows []*ActionSchedule) error {
7465
ScheduleID: row.ID,
7566
Spec: spec,
7667
}
77-
// Parse the spec and check for errors
7868
schedule, err := specRow.Parse()
7969
if err != nil {
80-
continue // skip to the next spec if there's an error
70+
continue
8171
}
8272

83-
specRow.Next = timeutil.TimeStamp(schedule.Next(now).Unix())
73+
next := schedule.Next(now)
74+
if next.IsZero() {
75+
continue // the spec parses but can never occur, like "0 0 30 2 *"
76+
}
77+
specRow.Next = timeutil.TimeStamp(next.Unix())
8478

8579
// Insert the new schedule spec row
8680
if err = db.Insert(ctx, specRow); err != nil {

models/actions/schedule_spec.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ type ActionScheduleSpec struct {
2626
// Next time the job will run, or the zero time if Cron has not been
2727
// started or this entry's schedule is unsatisfiable
2828
Next timeutil.TimeStamp `xorm:"index"`
29-
// Prev is the last time this job was run, or the zero time if never.
29+
// Prev is the occurrence this spec was last processed for, or the zero time if never.
3030
Prev timeutil.TimeStamp
3131
Spec string
3232

models/actions/schedule_spec_list.go

Lines changed: 0 additions & 97 deletions
This file was deleted.

services/actions/notifier_helper.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -595,7 +595,7 @@ func handleSchedules(
595595
crons = append(crons, run)
596596
}
597597

598-
return actions_model.CreateScheduleTask(ctx, crons)
598+
return actions_model.CreateScheduleTaskBySchedules(ctx, crons)
599599
}
600600

601601
// DetectAndHandleSchedules detects the schedule workflows on the default branch and create schedule tasks

services/actions/schedule_tasks.go

Lines changed: 66 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -22,91 +22,94 @@ import (
2222
"gitea.dev/modules/timeutil"
2323
webhook_module "gitea.dev/modules/webhook"
2424
"gitea.dev/services/convert"
25+
26+
"xorm.io/builder"
2527
)
2628

2729
// StartScheduleTasks start the task
2830
func StartScheduleTasks(ctx context.Context) error {
2931
return startTasks(ctx)
3032
}
3133

32-
// startTasks retrieves specifications in pages, creates a schedule task for each specification,
33-
// and updates the specification's next run time and previous run time.
34-
// The function returns an error if there's an issue with finding or updating the specifications.
34+
// startTasks starts every due spec and returns an error if any of them failed.
3535
func startTasks(ctx context.Context) error {
36-
// Set the page size
37-
pageSize := 50
38-
39-
// Retrieve specs in pages until all specs have been retrieved
36+
var failed int
4037
now := time.Now()
41-
for page := 1; ; page++ {
42-
// Retrieve the specs for the current page
43-
specs, _, err := actions_model.FindSpecs(ctx, actions_model.FindSpecOptions{
44-
ListOptions: db.ListOptions{
45-
Page: page,
46-
PageSize: pageSize,
47-
},
48-
Next: now.Unix(),
49-
})
50-
if err != nil {
51-
return fmt.Errorf("find specs: %w", err)
52-
}
53-
54-
if err := specs.LoadRepos(ctx); err != nil {
55-
return fmt.Errorf("LoadRepos: %w", err)
56-
}
57-
58-
// Loop through each spec and create a schedule task for it
59-
for _, row := range specs {
60-
if row.Repo.IsArchived {
61-
// Skip if the repo is archived
62-
continue
38+
err := db.Iterate(ctx,
39+
builder.And(builder.Gt{"next": 0}, builder.Lte{"next": now.Unix()}),
40+
func(ctx context.Context, row *actions_model.ActionScheduleSpec) error {
41+
// one failing spec must not abort the pass, or a single broken workflow stops every other schedule
42+
if err := startTask(ctx, row, now); err != nil {
43+
failed++
44+
log.Error("start schedule spec %d (repo %d, schedule %d): %v", row.ID, row.RepoID, row.ScheduleID, err)
6345
}
46+
return nil
47+
})
48+
if err != nil {
49+
return fmt.Errorf("iterate specs: %w", err)
50+
}
6451

65-
cfg, err := row.Repo.GetUnit(ctx, unit.TypeActions)
66-
if err != nil {
67-
if repo_model.IsErrUnitTypeNotExist(err) {
68-
// Skip the actions unit of this repo is disabled.
69-
continue
70-
}
71-
return fmt.Errorf("GetUnit: %w", err)
72-
}
73-
if cfg.ActionsConfig().IsWorkflowDisabled(row.Schedule.WorkflowID) {
74-
continue
75-
}
52+
// surfaces as an admin notice through the cron task, once per occurrence rather than once per pass
53+
if failed > 0 {
54+
return fmt.Errorf("%d schedule(s) could not be started", failed)
55+
}
56+
return nil
57+
}
7658

77-
if err := CreateScheduleTask(ctx, row); err != nil {
78-
log.Error("CreateScheduleTask: %v", err)
79-
return err
80-
}
59+
// startTask advances the spec to its next occurrence before creating the run, so a failing workflow
60+
// retries on its own schedule instead of on every pass, and a failed update cannot duplicate the run.
61+
func startTask(ctx context.Context, row *actions_model.ActionScheduleSpec, now time.Time) error {
62+
cronSchedule, err := row.Parse()
63+
if err != nil {
64+
return fmt.Errorf("parse %q: %w", row.Spec, err)
65+
}
66+
row.Prev = row.Next
67+
row.Next = timeutil.TimeStamp(cronSchedule.Next(now.Add(time.Minute)).Unix())
68+
if err := actions_model.UpdateScheduleSpec(ctx, row, "prev", "next"); err != nil {
69+
return fmt.Errorf("update spec: %w", err)
70+
}
8171

82-
// Parse the spec
83-
schedule, err := row.Parse()
84-
if err != nil {
85-
log.Error("Parse: %v", err)
86-
return err
87-
}
72+
// a spec whose schedule or repo row is gone is skipped, not reported on every occurrence
73+
schedule, exist, err := db.GetByID[actions_model.ActionSchedule](ctx, row.ScheduleID)
74+
if err != nil {
75+
return fmt.Errorf("get schedule %d: %w", row.ScheduleID, err)
76+
} else if !exist {
77+
return nil
78+
}
79+
repo, exist, err := db.GetByID[repo_model.Repository](ctx, row.RepoID)
80+
if err != nil {
81+
return fmt.Errorf("get repo %d: %w", row.RepoID, err)
82+
} else if !exist {
83+
return nil
84+
}
85+
row.Schedule, row.Repo = schedule, repo
8886

89-
// Update the spec's next run time and previous run time
90-
row.Prev = row.Next
91-
row.Next = timeutil.TimeStamp(schedule.Next(now.Add(1 * time.Minute)).Unix())
92-
if err := actions_model.UpdateScheduleSpec(ctx, row, "prev", "next"); err != nil {
93-
log.Error("UpdateScheduleSpec: %v", err)
94-
return err
95-
}
96-
}
87+
// only archived repos are skipped; mirrors keep their schedules because a mirror is a normal repo
88+
// for Actions, and nightly builds or scans of the mirrored code are a common reason to run one
89+
if row.Repo.IsArchived {
90+
return nil
91+
}
9792

98-
// Stop if all specs have been retrieved
99-
if len(specs) < pageSize {
100-
break
93+
cfg, err := row.Repo.GetUnit(ctx, unit.TypeActions)
94+
if err != nil {
95+
if repo_model.IsErrUnitTypeNotExist(err) {
96+
return nil
10197
}
98+
return fmt.Errorf("GetUnit: %w", err)
99+
}
100+
if cfg.ActionsConfig().IsWorkflowDisabled(row.Schedule.WorkflowID) {
101+
return nil
102102
}
103103

104+
if err := CreateScheduleTaskBySpec(ctx, row); err != nil {
105+
return fmt.Errorf("create run for %s workflow %q: %w", row.Repo.FullName(), row.Schedule.WorkflowID, err)
106+
}
104107
return nil
105108
}
106109

107-
// CreateScheduleTask creates a scheduled task from a cron action schedule spec.
110+
// CreateScheduleTaskBySpec creates a scheduled task from a cron action schedule spec.
108111
// It creates an action run based on the schedule, inserts it into the database, and creates commit statuses for each job.
109-
func CreateScheduleTask(ctx context.Context, spec *actions_model.ActionScheduleSpec) error {
112+
func CreateScheduleTaskBySpec(ctx context.Context, spec *actions_model.ActionScheduleSpec) error {
110113
cron := spec.Schedule
111114

112115
// Scheduled runs carry no webhook payload; synthesize what github.event.* expects.

services/actions/schedule_tasks_test.go

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,15 @@ package actions
55

66
import (
77
"testing"
8+
"time"
89

10+
actions_model "gitea.dev/models/actions"
11+
"gitea.dev/models/db"
12+
"gitea.dev/models/unittest"
913
"gitea.dev/modules/json"
1014
api "gitea.dev/modules/structs"
15+
"gitea.dev/modules/timeutil"
16+
webhook_module "gitea.dev/modules/webhook"
1117

1218
"github.com/stretchr/testify/assert"
1319
"github.com/stretchr/testify/require"
@@ -73,3 +79,47 @@ func TestWithScheduleInEventPayload(t *testing.T) {
7379
assert.Equal(t, payload, updated)
7480
})
7581
}
82+
83+
func TestStartTasks(t *testing.T) {
84+
require.NoError(t, unittest.PrepareTestDatabase())
85+
86+
insertSchedule := func(repoID, ownerID int64, workflowID, cronSpec, content string, next timeutil.TimeStamp) *actions_model.ActionScheduleSpec {
87+
schedule := &actions_model.ActionSchedule{
88+
Title: workflowID,
89+
RepoID: repoID,
90+
OwnerID: ownerID,
91+
WorkflowID: workflowID,
92+
TriggerUserID: 1,
93+
Ref: "refs/heads/master",
94+
CommitSHA: "c2d72f548424103f01ee1dc02889c1e2bff816b0",
95+
Event: webhook_module.HookEventSchedule,
96+
EventPayload: "{}",
97+
Content: []byte(content),
98+
}
99+
require.NoError(t, db.Insert(t.Context(), schedule))
100+
spec := &actions_model.ActionScheduleSpec{RepoID: repoID, ScheduleID: schedule.ID, Spec: cronSpec, Next: next}
101+
require.NoError(t, db.Insert(t.Context(), spec))
102+
return spec
103+
}
104+
105+
due := timeutil.TimeStamp(time.Now().Add(-time.Minute).Unix())
106+
validWorkflow := "jobs:\n job:\n runs-on: ubuntu-latest\n steps:\n - run: true\n"
107+
108+
// specs are processed by ascending id, so the broken one runs first and used to abort the whole pass
109+
broken := insertSchedule(1, 2, "broken.yml", "@every 1m", "this: [is: not: a: workflow", due)
110+
valid := insertSchedule(4, 5, "valid.yml", "@every 1m", validWorkflow, due)
111+
never := insertSchedule(4, 5, "never.yml", "0 0 30 2 *", validWorkflow, timeutil.TimeStamp(time.Time{}.Unix()))
112+
113+
require.ErrorContains(t, startTasks(t.Context()), "1 schedule(s) could not be started")
114+
115+
assert.Equal(t, 1, unittest.GetCount(t, &actions_model.ActionRun{RepoID: 4, WorkflowID: "valid.yml"}))
116+
assert.Equal(t, 0, unittest.GetCount(t, &actions_model.ActionRun{RepoID: 1, WorkflowID: "broken.yml"}))
117+
assert.Equal(t, 0, unittest.GetCount(t, &actions_model.ActionRun{RepoID: 4, WorkflowID: "never.yml"}))
118+
119+
// the broken spec moves on too, so it does not fail again on every pass
120+
for _, spec := range []*actions_model.ActionScheduleSpec{broken, valid} {
121+
updated := unittest.AssertExistsAndLoadBean(t, &actions_model.ActionScheduleSpec{ID: spec.ID})
122+
assert.Greater(t, updated.Next, spec.Next)
123+
}
124+
assert.Equal(t, never.Next, unittest.AssertExistsAndLoadBean(t, &actions_model.ActionScheduleSpec{ID: never.ID}).Next)
125+
}

0 commit comments

Comments
 (0)