Skip to content

Commit 8c752e3

Browse files
martindbrvchmouel
martindbrv
authored and
chmouel
committed
feat: add relative TaskRef fetching functionality
The current implementation doesn't allow to reference tasks relative to a remote pipeline (e.g., the task definition is in the same remote location as the pipeline). The issue is described here: https://issues.redhat.com/browse/SRVKP-4332 This patch introduces the necessary functionality to enable the behaviour above. It also includes: - a multi-PipelineRun unit test - adapted docs Signed-off-by: Chmouel Boudjnah <[email protected]>
1 parent f4b1be2 commit 8c752e3

File tree

7 files changed

+314
-15
lines changed

7 files changed

+314
-15
lines changed

‎docs/content/docs/guide/resolver.md

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,70 @@ out and not process the pipeline.
201201

202202
If the object fetched cannot be parsed as a Tekton `Task` it will error out.
203203

204+
### Relative Tasks
205+
206+
`Pipeline-as-Code` also supports fetching relative
207+
to a remote pipeline tasks (see Section [Remote Pipeline Annotations](#remote-pipeline-annotations)).
208+
209+
Consider the following scenario:
210+
211+
* Repository A (where the event is originating from) contains:
212+
213+
```yaml
214+
# .tekton/pipelinerun.yaml
215+
216+
apiVersion: tekton.dev/v1
217+
kind: PipelineRun
218+
metadata:
219+
name: hello-world
220+
annotations:
221+
pipelinesascode.tekton.dev/on-target-branch: "[main]"
222+
pipelinesascode.tekton.dev/on-event: "[push]"
223+
pipelinesascode.tekton.dev/pipeline: "https://github.com/user/repositoryb/blob/main/pipeline.yaml"
224+
spec:
225+
pipelineRef:
226+
name: hello-world
227+
```
228+
229+
* Repository B contains:
230+
231+
```yaml
232+
# pipeline.yaml
233+
234+
apiVersion: tekton.dev/v1beta1
235+
kind: Pipeline
236+
metadata:
237+
name: hello-world
238+
annotations:
239+
pipelinesascode.tekton.dev/task: "./task.yaml"
240+
spec:
241+
tasks:
242+
- name: say-hello
243+
taskRef:
244+
name: hello
245+
```
246+
247+
```yaml
248+
# task.yaml
249+
250+
apiVersion: tekton.dev/v1beta1
251+
kind: Task
252+
metadata:
253+
name: hello
254+
spec:
255+
steps:
256+
- name: echo
257+
image: alpine
258+
script: |
259+
#!/bin/sh
260+
echo "Hello, World!"
261+
```
262+
263+
The Resolver will fetch the remote pipeline and then attempt to retrieve each task.
264+
The task paths are relative to the path where the remote pipeline, referencing them
265+
resides (if the pipeline is at `/foo/bar/pipeline.yaml`, and the specified task path is `../task.yaml`, the
266+
assembled target URL for fetching the task is `/foo/task.yaml`).
267+
204268
## Remote Pipeline annotations
205269

206270
Remote Pipeline can be referenced by annotation, allowing you to share a Pipeline across multiple repositories.

‎pkg/resolve/remote.go

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ package resolve
33
import (
44
"context"
55
"fmt"
6+
"net/url"
7+
"path"
68

79
"github.com/openshift-pipelines/pipelines-as-code/pkg/matcher"
810
tektonv1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1"
@@ -19,6 +21,38 @@ func alreadyFetchedResource[T NamedItem](resources map[string]T, resourceName st
1921
return false
2022
}
2123

24+
// Tries to assemble task FQDNs based on the base URL
25+
// of a remote pipeline.
26+
//
27+
// If there isn't a remote pipeline reference for the current
28+
// run, tasks are returned as they are. Any task with an already
29+
// valid URL is skipped.
30+
func assembleTaskFQDNs(pipelineURL string, tasks []string) ([]string, error) {
31+
if pipelineURL == "" {
32+
return tasks, nil // no pipeline URL, return tasks as is
33+
}
34+
35+
pURL, err := url.Parse(pipelineURL)
36+
if err != nil {
37+
return tasks, err
38+
}
39+
// pop the pipeline file path from the URL
40+
pURL.Path = path.Dir(pURL.Path)
41+
42+
taskURLS := make([]string, len(tasks))
43+
for i, t := range tasks {
44+
tURL, err := url.Parse(t)
45+
if err == nil && tURL.Scheme != "" && tURL.Host != "" {
46+
taskURLS[i] = t
47+
continue // it's already an absolute URL
48+
}
49+
tURL = pURL
50+
tURL = tURL.JoinPath(t)
51+
taskURLS[i] = tURL.String()
52+
}
53+
return taskURLS, nil
54+
}
55+
2256
// resolveRemoteResources will get remote tasks or Pipelines from annotations.
2357
//
2458
// It already has some tasks or pipeline coming from the tekton directory stored in [types]
@@ -41,7 +75,8 @@ func resolveRemoteResources(ctx context.Context, rt *matcher.RemoteTasks, types
4175
for _, pipelinerun := range types.PipelineRuns {
4276
// contain Resources specific to run
4377
fetchedResourcesForPipelineRun := FetchedResourcesForRun{
44-
Tasks: map[string]*tektonv1.Task{},
78+
Tasks: map[string]*tektonv1.Task{},
79+
PipelineURL: "",
4580
}
4681
var pipeline *tektonv1.Pipeline
4782
var err error
@@ -76,6 +111,8 @@ func resolveRemoteResources(ctx context.Context, rt *matcher.RemoteTasks, types
76111
}
77112
// add the pipeline to the Resources fetched for the Event
78113
fetchedResourcesForEvent.Pipelines[remotePipeline] = pipeline
114+
// add the pipeline URL to the run specific Resources
115+
fetchedResourcesForPipelineRun.PipelineURL = remotePipeline
79116
}
80117
}
81118
}
@@ -98,6 +135,11 @@ func resolveRemoteResources(ctx context.Context, rt *matcher.RemoteTasks, types
98135
if err != nil {
99136
return []*tektonv1.PipelineRun{}, fmt.Errorf("error getting remote task from pipeline annotations: %w", err)
100137
}
138+
// check for relative task references and assemble FQDNs
139+
pipelineTasks, err = assembleTaskFQDNs(fetchedResourcesForPipelineRun.PipelineURL, pipelineTasks)
140+
if err != nil {
141+
return []*tektonv1.PipelineRun{}, err
142+
}
101143
}
102144
}
103145

‎pkg/resolve/remote_test.go

Lines changed: 133 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,62 @@ func TestRemote(t *testing.T) {
7575
pipelinewithTaskRefYamlB, err := yaml.Marshal(pipelinewithTaskRef)
7676
assert.NilError(t, err)
7777

78+
pipelineRelativeTaskRef := []tektonv1.PipelineTask{
79+
{
80+
Name: remoteTaskName + "-a",
81+
TaskRef: &tektonv1.TaskRef{
82+
Name: remoteTaskName + "-a",
83+
},
84+
},
85+
{
86+
Name: remoteTaskName + "-b",
87+
TaskRef: &tektonv1.TaskRef{
88+
Name: remoteTaskName + "-b",
89+
},
90+
},
91+
{
92+
Name: remoteTaskName + "-c",
93+
TaskRef: &tektonv1.TaskRef{
94+
Name: remoteTaskName + "-c",
95+
},
96+
},
97+
{
98+
Name: remoteTaskName + "-d",
99+
TaskRef: &tektonv1.TaskRef{
100+
Name: remoteTaskName + "-d",
101+
},
102+
},
103+
}
104+
pipelineWithRelativeTaskRef := ttkn.MakePipeline(remotePipelineName, pipelineRelativeTaskRef[:3], map[string]string{
105+
apipac.Task: "./" + remoteTaskName + "-a",
106+
apipac.Task + "-1": "../" + remoteTaskName + "-b",
107+
apipac.Task + "-2": "../../../../" + remoteTaskName + "-c",
108+
})
109+
110+
pipelineWithRelativeTaskRefYamlB, err := yaml.Marshal(pipelineWithRelativeTaskRef)
111+
assert.NilError(t, err)
112+
113+
pipelineWithRelativeTaskRef1 := ttkn.MakePipeline(remotePipelineName, pipelineRelativeTaskRef[1:], map[string]string{
114+
apipac.Task: remoteTaskName + "-b",
115+
apipac.Task + "-1": "utils/" + remoteTaskName + "-c",
116+
apipac.Task + "-2": " " + remoteTaskName + "-d",
117+
})
118+
119+
pipelineWithRelativeTaskRefYamlB1, err := yaml.Marshal(pipelineWithRelativeTaskRef1)
120+
assert.NilError(t, err)
121+
122+
singleRelativeTaskBa, err := ttkn.MakeTaskB(remoteTaskName+"-a", taskFromPipelineSpec)
123+
assert.NilError(t, err)
124+
125+
singleRelativeTaskBb, err := ttkn.MakeTaskB(remoteTaskName+"-b", taskFromPipelineSpec)
126+
assert.NilError(t, err)
127+
128+
singleRelativeTaskBc, err := ttkn.MakeTaskB(remoteTaskName+"-c", taskFromPipelineSpec)
129+
assert.NilError(t, err)
130+
131+
singleRelativeTaskBd, err := ttkn.MakeTaskB(remoteTaskName+"-d", taskFromPipelineSpec)
132+
assert.NilError(t, err)
133+
78134
singleTask := ttkn.MakeTask(remoteTaskName, taskFromPipelineSpec)
79135
singleTaskB, err := yaml.Marshal(singleTask)
80136
assert.NilError(t, err)
@@ -92,7 +148,7 @@ func TestRemote(t *testing.T) {
92148
remoteURLS map[string]map[string]string
93149
expectedLogsSnippets []string
94150
expectedTaskSpec tektonv1.TaskSpec
95-
expectedPipelineRun string
151+
expectedPipelineRun []string
96152
noPipelineRun bool
97153
}{
98154
{
@@ -127,7 +183,66 @@ func TestRemote(t *testing.T) {
127183
fmt.Sprintf("successfully fetched %s from remote https url", remotePipelineURL),
128184
fmt.Sprintf("successfully fetched %s from remote https url", remoteTaskURL),
129185
},
130-
expectedPipelineRun: "remote-pipeline-with-remote-task-from-pipeline.yaml",
186+
expectedPipelineRun: []string{"remote-pipeline-with-remote-task-from-pipeline.yaml"},
187+
},
188+
{
189+
name: "remote pipelines with relative tasks",
190+
pipelineruns: []*tektonv1.PipelineRun{
191+
ttkn.MakePR(randomPipelineRunName, map[string]string{
192+
apipac.Pipeline: remotePipelineURL,
193+
},
194+
tektonv1.PipelineRunSpec{
195+
PipelineRef: &tektonv1.PipelineRef{
196+
Name: remotePipelineName,
197+
},
198+
},
199+
),
200+
ttkn.MakePR(randomPipelineRunName, map[string]string{
201+
apipac.Pipeline: remotePipelineURL + "-1",
202+
},
203+
tektonv1.PipelineRunSpec{
204+
PipelineRef: &tektonv1.PipelineRef{
205+
Name: remotePipelineName,
206+
},
207+
},
208+
),
209+
},
210+
remoteURLS: map[string]map[string]string{
211+
remotePipelineURL: {
212+
"body": string(pipelineWithRelativeTaskRefYamlB),
213+
"code": "200",
214+
},
215+
remotePipelineURL + "-1": {
216+
"body": string(pipelineWithRelativeTaskRefYamlB1),
217+
"code": "200",
218+
},
219+
remoteTaskURL + "-a": {
220+
"body": string(singleRelativeTaskBa),
221+
"code": "200",
222+
},
223+
remoteTaskURL + "-b": {
224+
"body": string(singleRelativeTaskBb),
225+
"code": "200",
226+
},
227+
remoteTaskURL + "-c": {
228+
"body": string(singleRelativeTaskBc),
229+
"code": "200",
230+
},
231+
"http://remote/utils/remote-task-c": {
232+
"body": string(singleRelativeTaskBc),
233+
"code": "200",
234+
},
235+
remoteTaskURL + "-d": {
236+
"body": string(singleRelativeTaskBd),
237+
"code": "200",
238+
},
239+
},
240+
expectedTaskSpec: taskFromPipelineSpec,
241+
expectedLogsSnippets: []string{},
242+
expectedPipelineRun: []string{
243+
"remote-pipeline-with-relative-tasks.yaml",
244+
"remote-pipeline-with-relative-tasks-1.yaml",
245+
},
131246
},
132247
{
133248
name: "remote pipeline with remote task in pipeline overridden from pipelinerun",
@@ -162,7 +277,7 @@ func TestRemote(t *testing.T) {
162277
fmt.Sprintf("successfully fetched %s from remote https url", remotePipelineURL),
163278
fmt.Sprintf("successfully fetched %s from remote https url", taskFromPipelineRunURL),
164279
},
165-
expectedPipelineRun: "remote-pipeline-with-remote-task-from-pipelinerun.yaml",
280+
expectedPipelineRun: []string{"remote-pipeline-with-remote-task-from-pipelinerun.yaml"},
166281
},
167282
{
168283
name: "remote pipelinerun no annotations",
@@ -222,7 +337,7 @@ func TestRemote(t *testing.T) {
222337
fmt.Sprintf("successfully fetched %s from remote https url", remotePipelineURL),
223338
fmt.Sprintf("successfully fetched %s from remote https url", remoteTaskURL),
224339
},
225-
expectedPipelineRun: "skip-fetching-multiple-tasks-of-the-same-name-from-pipelinerun-annotations-and-pipeline-annotation.yaml",
340+
expectedPipelineRun: []string{"skip-fetching-multiple-tasks-of-the-same-name-from-pipelinerun-annotations-and-pipeline-annotation.yaml"},
226341
},
227342
{
228343
name: "skip fetching multiple tasks of the same name from pipelinerun annotations and tektondir",
@@ -258,7 +373,7 @@ func TestRemote(t *testing.T) {
258373
fmt.Sprintf("skipping remote task %s as already fetched task %s for pipelinerun %s", remoteTaskURL, remoteTaskName, randomPipelineRunName),
259374
fmt.Sprintf("overriding task %s coming from .tekton directory by an annotation task for pipelinerun %s", remoteTaskName, randomPipelineRunName),
260375
},
261-
expectedPipelineRun: "skip-fetching-multiple-tasks-of-the-same-name-from-pipelinerun-annotations-and-tektondir.yaml",
376+
expectedPipelineRun: []string{"skip-fetching-multiple-tasks-of-the-same-name-from-pipelinerun-annotations-and-tektondir.yaml"},
262377
},
263378
{
264379
name: "skip fetching multiple pipelines of the same name from pipelinerun annotations and tektondir",
@@ -293,7 +408,7 @@ func TestRemote(t *testing.T) {
293408
fmt.Sprintf("successfully fetched %s from remote https url", remoteTaskURL),
294409
fmt.Sprintf("skipping remote task %s as already fetched task %s for pipelinerun %s", remoteTaskURL, remoteTaskName, randomPipelineRunName),
295410
},
296-
expectedPipelineRun: "skip-fetching-multiple-pipelines-of-the-same-name-from-pipelinerun-annotations-and-tektondir.yaml",
411+
expectedPipelineRun: []string{"skip-fetching-multiple-pipelines-of-the-same-name-from-pipelinerun-annotations-and-tektondir.yaml"},
297412
},
298413
}
299414
for _, tt := range tests {
@@ -334,12 +449,18 @@ func TestRemote(t *testing.T) {
334449
assert.Assert(t, len(ret) == 0, "not expecting any pipelinerun")
335450
return
336451
}
337-
expectedData, err := os.ReadFile("testdata/" + tt.expectedPipelineRun)
338-
assert.NilError(t, err)
339-
pipelineRun := &tektonv1.PipelineRun{}
340-
err = yaml.Unmarshal(expectedData, pipelineRun)
341-
assert.NilError(t, err)
342-
assert.DeepEqual(t, pipelineRun, ret[0])
452+
for i, pr := range ret {
453+
if len(tt.expectedPipelineRun) < len(ret) {
454+
assert.NilError(t, fmt.Errorf("insufficient amount of expectedPipelineRuns was provided, got %d but want %d; or set noPipelineRun to true",
455+
len(tt.expectedPipelineRun), len(ret)))
456+
}
457+
expectedData, err := os.ReadFile("testdata/" + tt.expectedPipelineRun[i])
458+
assert.NilError(t, err)
459+
pipelineRun := &tektonv1.PipelineRun{}
460+
err = yaml.Unmarshal(expectedData, pipelineRun)
461+
assert.NilError(t, err)
462+
assert.DeepEqual(t, pipelineRun, pr)
463+
}
343464
})
344465
}
345466
}

‎pkg/resolve/resolve.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,9 @@ type FetchedResources struct {
3838

3939
// Contains Fetched Resources for Run, with key equals to resource name from metadata.name field.
4040
type FetchedResourcesForRun struct {
41-
Tasks map[string]*tektonv1.Task
42-
Pipeline *tektonv1.Pipeline
41+
Tasks map[string]*tektonv1.Task
42+
Pipeline *tektonv1.Pipeline
43+
PipelineURL string
4344
}
4445

4546
func NewTektonTypes() TektonTypes {

‎pkg/resolve/testdata/remote-pipeline-with-relative-tasks-1.yaml

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
apiVersion: tekton.dev/v1
2+
kind: PipelineRun
3+
metadata:
4+
annotations:
5+
pipelinesascode.tekton.dev/pipeline: http://remote/remote-pipeline-1
6+
generateName: pipelinerun-abc-
7+
spec:
8+
pipelineSpec:
9+
tasks:
10+
- name: remote-task-b
11+
taskSpec:
12+
steps:
13+
- name: step1
14+
image: scratch
15+
command:
16+
- "true"
17+
- name: remote-task-c
18+
taskSpec:
19+
steps:
20+
- name: step1
21+
image: scratch
22+
command:
23+
- "true"
24+
- name: remote-task-d
25+
taskSpec:
26+
steps:
27+
- name: step1
28+
image: scratch
29+
command:
30+
- "true"
31+
finally: []

0 commit comments

Comments
 (0)