Skip to content

Store workflow/step dependencies in database and update API #6130

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lafriks wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Store workflow/step dependencies in database and update API #6130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c8c51ed
Store workflow/step dependencies in database and update API needed fo…
lafriks Feb 17, 2026
dfd1b8e
Merge branch 'main' into feat/pipeline_deps
lafriks Feb 17, 2026
c52f4da
Merge branch 'main' into feat/pipeline_deps
lafriks Feb 18, 2026
6728b0e
Fix dependencies on matrix workflows
lafriks Feb 18, 2026
3306c46
Merge branch 'main' into feat/pipeline_deps
lafriks Feb 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions cmd/server/openapi/docs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5574,6 +5574,12 @@ const docTemplate = `{
"Step": {
"type": "object",
"properties": {
"depends_on": {
"type": "array",
"items": {
"type": "integer"
}
},
"error": {
"type": "string"
},
Expand Down Expand Up @@ -6168,6 +6174,12 @@ const docTemplate = `{
"$ref": "#/definitions/Step"
}
},
"depends_on": {
"type": "array",
"items": {
"type": "integer"
}
},
"environ": {
"type": "object",
"additionalProperties": {
Expand Down
1 change: 1 addition & 0 deletions pipeline/backend/types/step.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ type Step struct {
Networks []Conn `json:"networks,omitempty"`
DNS []string `json:"dns,omitempty"`
DNSSearch []string `json:"dns_search,omitempty"`
DependsOn []string `json:"depends_on,omitempty"`
Copy link
Member

@6543 6543 Feb 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why do we need it in backend?

Copy link
Contributor Author

@lafriks lafriks Feb 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

because it goes through compiler to backend types and then from backend types it's assigned to model type

Copy link
Member

@6543 6543 Feb 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what?!

OnFailure bool `json:"on_failure,omitempty"`
OnSuccess bool `json:"on_success,omitempty"`
Failure string `json:"failure,omitempty"`
Expand Down
1 change: 1 addition & 0 deletions pipeline/frontend/yaml/compiler/compiler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,7 @@ func TestCompilerCompile(t *testing.T) {
Type: backend_types.StepTypeCommands,
Image: "bash",
Commands: []string{"echo 1"},
DependsOn: []string{"echo env", "echo 2"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolume + ":/test"},
Expand Down
1 change: 1 addition & 0 deletions pipeline/frontend/yaml/compiler/convert.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,7 @@ func (c *Compiler) createProcess(container *yaml_types.Container, workflow *yaml
Networks: networks,
DNS: container.DNS,
DNSSearch: container.DNSSearch,
DependsOn: container.DependsOn,
AuthConfig: authConfig,
OnSuccess: onSuccess,
OnFailure: onFailure,
Expand Down
28 changes: 15 additions & 13 deletions server/model/step.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,19 +26,21 @@ const (

// Step represents a process in the pipeline.
type Step struct {
ID int64 `json:"id" xorm:"pk autoincr 'id'"`
UUID string `json:"uuid" xorm:"INDEX 'uuid'"`
PipelineID int64 `json:"pipeline_id" xorm:"UNIQUE(s) INDEX 'pipeline_id'"`
PID int `json:"pid" xorm:"UNIQUE(s) 'pid'"`
PPID int `json:"ppid" xorm:"ppid"`
Name string `json:"name" xorm:"name"`
State StatusValue `json:"state" xorm:"state"`
Error string `json:"error,omitempty" xorm:"TEXT 'error'"`
Failure string `json:"-" xorm:"failure"`
ExitCode int `json:"exit_code" xorm:"exit_code"`
Started int64 `json:"started,omitempty" xorm:"started"`
Finished int64 `json:"finished,omitempty" xorm:"finished"`
Type StepType `json:"type,omitempty" xorm:"type"`
ID int64 `json:"id" xorm:"pk autoincr 'id'"`
UUID string `json:"uuid" xorm:"INDEX 'uuid'"`
PipelineID int64 `json:"pipeline_id" xorm:"UNIQUE(s) INDEX 'pipeline_id'"`
PID int `json:"pid" xorm:"UNIQUE(s) 'pid'"`
PPID int `json:"ppid" xorm:"ppid"`
Name string `json:"name" xorm:"name"`
State StatusValue `json:"state" xorm:"state"`
Error string `json:"error,omitempty" xorm:"TEXT 'error'"`
Failure string `json:"-" xorm:"failure"`
ExitCode int `json:"exit_code" xorm:"exit_code"`
Started int64 `json:"started,omitempty" xorm:"started"`
Finished int64 `json:"finished,omitempty" xorm:"finished"`
Type StepType `json:"type,omitempty" xorm:"type"`
DependsOn []int64 `json:"depends_on,omitempty" xorm:"json 'depends_on'"`
DependsOnNames []string `json:"-" xorm:"-"`
} // @name Step

// TableName return database table name for xorm.
Expand Down
28 changes: 15 additions & 13 deletions server/model/workflow.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,21 @@ package model

// Workflow represents a workflow in the pipeline.
type Workflow struct {
ID int64 `json:"id" xorm:"pk autoincr 'id'"`
PipelineID int64 `json:"pipeline_id" xorm:"UNIQUE(s) INDEX 'pipeline_id'"`
PID int `json:"pid" xorm:"UNIQUE(s) 'pid'"`
Name string `json:"name" xorm:"name"`
State StatusValue `json:"state" xorm:"state"`
Error string `json:"error,omitempty" xorm:"TEXT 'error'"`
Started int64 `json:"started,omitempty" xorm:"started"`
Finished int64 `json:"finished,omitempty" xorm:"finished"`
AgentID int64 `json:"agent_id,omitempty" xorm:"agent_id"`
Platform string `json:"platform,omitempty" xorm:"platform"`
Environ map[string]string `json:"environ,omitempty" xorm:"json 'environ'"`
AxisID int `json:"-" xorm:"axis_id"`
Children []*Step `json:"children,omitempty" xorm:"-"`
ID int64 `json:"id" xorm:"pk autoincr 'id'"`
PipelineID int64 `json:"pipeline_id" xorm:"UNIQUE(s) INDEX 'pipeline_id'"`
PID int `json:"pid" xorm:"UNIQUE(s) 'pid'"`
Name string `json:"name" xorm:"name"`
State StatusValue `json:"state" xorm:"state"`
Error string `json:"error,omitempty" xorm:"TEXT 'error'"`
Started int64 `json:"started,omitempty" xorm:"started"`
Finished int64 `json:"finished,omitempty" xorm:"finished"`
AgentID int64 `json:"agent_id,omitempty" xorm:"agent_id"`
Platform string `json:"platform,omitempty" xorm:"platform"`
Environ map[string]string `json:"environ,omitempty" xorm:"json 'environ'"`
DependsOn []int64 `json:"depends_on,omitempty" xorm:"json 'depends_on'"`
DependsOnNames []string `json:"-" xorm:"-"`
AxisID int `json:"-" xorm:"axis_id"`
Children []*Step `json:"children,omitempty" xorm:"-"`
}

// TableName return database table name for xorm.
Expand Down
18 changes: 10 additions & 8 deletions server/pipeline/items.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,14 +180,15 @@ func setPipelineStepsOnPipeline(pipeline *model.Pipeline, pipelineItems []*stepb
for _, step := range stage.Steps {
pidSequence++
step := &model.Step{
Name: step.Name,
UUID: step.UUID,
PipelineID: pipeline.ID,
PID: pidSequence,
PPID: item.Workflow.PID,
State: model.StatusPending,
Failure: step.Failure,
Type: model.StepType(step.Type),
Name: step.Name,
UUID: step.UUID,
PipelineID: pipeline.ID,
PID: pidSequence,
PPID: item.Workflow.PID,
State: model.StatusPending,
Failure: step.Failure,
Type: model.StepType(step.Type),
DependsOnNames: step.DependsOn,
Copy link
Member

@6543 6543 Feb 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why not name it DependsOn like the rest of the codebase?!?

Copy link
Contributor Author

@lafriks lafriks Feb 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because you need to resolve string DependsOn (step names) to IDs but they are only known when all records are inserted, model has both DopendsOnNames -> string array and DependsOn -> int64 array that will be populated after saving all workflows and steps to database from DependsOnNames and then we can update depends_on column with depends_on step ID's

Copy link
Member

@6543 6543 Feb 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can we not do it DependsOn -> string array and DependsOnIDs -> int64 array to be consistent with the rest?

}
if item.Workflow.State == model.StatusSkipped {
step.State = model.StatusSkipped
Expand All @@ -202,6 +203,7 @@ func setPipelineStepsOnPipeline(pipeline *model.Pipeline, pipelineItems []*stepb
item.Workflow.State = model.StatusBlocked
}
item.Workflow.PipelineID = pipeline.ID
item.Workflow.DependsOnNames = item.DependsOn
pipeline.Workflows = append(pipeline.Workflows, item.Workflow)
}

Expand Down
47 changes: 44 additions & 3 deletions server/store/datastore/workflow.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,15 +52,56 @@ func (s storage) WorkflowsCreate(workflows []*model.Workflow) error {
}

func (s storage) workflowsCreate(sess *xorm.Session, workflows []*model.Workflow) error {
for i := range workflows {
// matrix workflows will have same name, so will have multiple IDs
workflowMap := make(map[string][]int64, len(workflows))

for _, wf := range workflows {
// only Insert on single object ref set auto created ID back to object
if err := s.stepCreate(sess, workflows[i].Children); err != nil {
if err := s.stepCreate(sess, wf.Children); err != nil {
return err
}
if _, err := sess.Insert(workflows[i]); err != nil {
if _, err := sess.Insert(wf); err != nil {
return err
}

workflowMap[wf.Name] = append(workflowMap[wf.Name], wf.ID)
}

for _, wf := range workflows {
if len(wf.DependsOnNames) > 0 {
ids := make([]int64, 0, len(wf.DependsOnNames))
for _, name := range wf.DependsOnNames {
ids = append(ids, workflowMap[name]...)
}
wf.DependsOn = ids

if _, err := sess.ID(wf.ID).Cols("depends_on").Update(wf); err != nil {
return err
}
}

stepMap := make(map[string]int64, len(wf.Children))
for _, child := range wf.Children {
stepMap[child.Name] = child.ID
}

for _, step := range wf.Children {
if len(step.DependsOnNames) > 0 {
ids := make([]int64, 0, len(step.DependsOnNames))
for _, name := range step.DependsOnNames {
if id, ok := stepMap[name]; ok {
ids = append(ids, id)
}
}
step.DependsOn = ids

if _, err := sess.ID(step.ID).Cols("depends_on").Update(step); err != nil {
return err
}
}
}
}

return nil
}

Expand Down
105 changes: 99 additions & 6 deletions server/store/datastore/workflow_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,15 +71,17 @@ func TestWorkflowGetTree(t *testing.T) {
PipelineID: 1,
PID: 2,
PPID: 1,
Name: "test",
State: "success",
},
{
UUID: "2bf387f7-2913-4907-814c-c9ada88707c0",
PipelineID: 1,
PID: 3,
PPID: 1,
Name: "build",
State: "success",
UUID: "2bf387f7-2913-4907-814c-c9ada88707c0",
PipelineID: 1,
PID: 3,
PPID: 1,
Name: "build",
DependsOnNames: []string{"test"},
State: "success",
},
},
}
Expand All @@ -93,6 +95,97 @@ func TestWorkflowGetTree(t *testing.T) {
assert.Len(t, workflowGet.Children, 2)
assert.Equal(t, 2, workflowGet.Children[0].PID)
assert.Equal(t, 3, workflowGet.Children[1].PID)
assert.EqualValues(t, []int64{workflowGet.Children[0].ID}, workflowGet.Children[1].DependsOn)
}

func TestWorkflowGetTreeMatrix(t *testing.T) {
store, closer := newTestStore(t, new(model.Step), new(model.Pipeline), new(model.Workflow))
defer closer()

buildAxis1 := &model.Workflow{
PipelineID: 1,
PID: 1,
Name: "build",
AxisID: 1,
Children: []*model.Step{
{
UUID: "aaaaaaaa-0001-0001-0001-000000000001",
PipelineID: 1,
PID: 2,
PPID: 1,
Name: "compile-linux-amd64",
State: "pending",
},
{
UUID: "aaaaaaaa-0001-0001-0001-000000000002",
PipelineID: 1,
PID: 3,
PPID: 1,
Name: "strip-binary",
DependsOnNames: []string{"compile-linux-amd64"},
State: "pending",
},
},
}
buildAxis2 := &model.Workflow{
PipelineID: 1,
PID: 4,
Name: "build",
AxisID: 2,
Children: []*model.Step{
{
UUID: "bbbbbbbb-0002-0002-0002-000000000001",
PipelineID: 1,
PID: 5,
PPID: 4,
Name: "compile-linux-arm64",
State: "pending",
},
{
UUID: "bbbbbbbb-0002-0002-0002-000000000002",
PipelineID: 1,
PID: 6,
PPID: 4,
Name: "strip-binary",
DependsOnNames: []string{"compile-linux-arm64"},
State: "pending",
},
},
}
deploy := &model.Workflow{
PipelineID: 1,
PID: 7,
Name: "deploy",
DependsOnNames: []string{"build"},
Children: []*model.Step{
{
UUID: "cccccccc-0003-0003-0003-000000000001",
PipelineID: 1,
PID: 8,
PPID: 7,
Name: "deploy",
State: "pending",
},
},
}

assert.NoError(t, store.WorkflowsCreate([]*model.Workflow{buildAxis1, buildAxis2, deploy}))

workflowsGet, err := store.WorkflowGetTree(&model.Pipeline{ID: 1})
assert.NoError(t, err)
assert.Len(t, workflowsGet, 3)

// deploy should depend on both build axis IDs
var deployWF *model.Workflow
for _, wf := range workflowsGet {
if wf.Name == "deploy" {
deployWF = wf
}
}
assert.NotNil(t, deployWF)
assert.Len(t, deployWF.DependsOn, 2, "deploy should depend on both matrix axes of build")
assert.Contains(t, deployWF.DependsOn, buildAxis1.ID)
assert.Contains(t, deployWF.DependsOn, buildAxis2.ID)
}

func TestWorkflowUpdate(t *testing.T) {
Expand Down