From 7afc0271961fc46a7bbd27c8ffe335b1a68d527b Mon Sep 17 00:00:00 2001 From: Akshay Pant Date: Wed, 12 Nov 2025 13:11:00 +0530 Subject: [PATCH] feat(cel): add direct custom param variable access in expressions Allow custom parameters from Repository CR to be accessed directly as CEL variables without template expansion. Parameters can now be used as: param_name == "value" on top of "{{param_name}}" == "value". Jira: https://issues.redhat.com/browse/SRVKP-9118 Signed-off-by: Akshay Pant --- docs/content/docs/guide/customparams.md | 70 ++++++ docs/content/docs/guide/matchingevents.md | 86 ++++++- pkg/matcher/annotation_matcher.go | 45 +++- pkg/matcher/annotation_matcher_test.go | 284 +++++++++++++++++++++- pkg/matcher/cel.go | 20 +- pkg/test/repository/repository.go | 6 + 6 files changed, 492 insertions(+), 19 deletions(-) diff --git a/docs/content/docs/guide/customparams.md b/docs/content/docs/guide/customparams.md index df40f0be27..57a6ef83d5 100644 --- a/docs/content/docs/guide/customparams.md +++ b/docs/content/docs/guide/customparams.md @@ -122,3 +122,73 @@ and a pull request event. - [GitHub Documentation for webhook events](https://docs.github.com/webhooks-and-events/webhooks/webhook-events-and-payloads?actionType=auto_merge_disabled#pull_request) - [GitLab Documentation for webhook events](https://docs.gitlab.com/ee/user/project/integrations/webhook_events.html) {{< /hint >}} + +### Using custom parameters in CEL matching expressions + +In addition to template expansion (`{{ param }}`), custom parameters defined in the Repository CR are available as CEL variables in the `on-cel-expression` annotation. This allows you to control which PipelineRuns are triggered based on repository-specific configuration. + +For example, with this Repository CR configuration: + +```yaml +apiVersion: pipelinesascode.tekton.dev/v1alpha1 +kind: Repository +metadata: + name: my-repo +spec: + url: "https://github.com/owner/repo" + params: + - name: enable_ci + value: "true" + - name: environment + value: "staging" +``` + +You can use these parameters directly in your PipelineRun's CEL expression: + +```yaml +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + name: my-pipeline + annotations: + pipelinesascode.tekton.dev/on-cel-expression: | + event == "push" && enable_ci == "true" && environment == "staging" +spec: + # ... pipeline spec +``` + +This approach is particularly useful for: + +- **Conditional CI**: Enable or disable CI for specific repositories without changing PipelineRun files +- **Environment-specific matching**: Run different pipelines based on environment configuration +- **Feature flags**: Control which pipelines run using repository-level feature flags + +Custom parameters from secrets are also available: + +```yaml +apiVersion: pipelinesascode.tekton.dev/v1alpha1 +kind: Repository +metadata: + name: my-repo +spec: + url: "https://github.com/owner/repo" + params: + - name: api_key + secret_ref: + name: my-secret + key: key +``` + +```yaml +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + name: my-pipeline-with-secret + annotations: + pipelinesascode.tekton.dev/on-cel-expression: | + event == "push" && api_key != "" +spec: + # ... pipeline spec +``` + +For more information on CEL expressions and event matching, see the [Advanced event matching using CEL]({{< relref "/docs/guide/matchingevents#advanced-event-matching-using-cel" >}}) documentation. diff --git a/docs/content/docs/guide/matchingevents.md b/docs/content/docs/guide/matchingevents.md index a25138a7f8..b4cfa7878d 100644 --- a/docs/content/docs/guide/matchingevents.md +++ b/docs/content/docs/guide/matchingevents.md @@ -301,18 +301,20 @@ pipelinesascode.tekton.dev/on-cel-expression: | The fields available are: -| **Field** | **Description** | -| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `event` | `push`, `pull_request` or `incoming`. | -| `target_branch` | The branch we are targeting. | -| `source_branch` | The branch where this pull_request comes from. (On `push`, this is the same as `target_branch`.) | -| `target_url` | The URL of the repository we are targeting. | -| `source_url` | The URL of the repository where this pull_request comes from. (On `push`, this is the same as `target_url`.) | -| `event_title` | Matches the title of the event. For `push`, it matches the commit title. For PR, it matches the Pull/Merge Request title. (Only supported for `GitHub`, `GitLab`, and `BitbucketCloud` providers.) | -| `body` | The full body as passed by the Git provider. Example: `body.pull_request.number` retrieves the pull request number on GitHub. | -| `headers` | The full set of headers as passed by the Git provider. Example: `headers['x-github-event']` retrieves the event type on GitHub. | -| `.pathChanged` | A suffix function to a string that can be a glob of a path to check if changed. (Supported only for `GitHub` and `GitLab` providers.) | -| `files` | The list of files that changed in the event (`all`, `added`, `deleted`, `modified`, and `renamed`). Example: `files.all` or `files.deleted`. For pull requests, every file belonging to the pull request will be listed. | +| **Field** | **Description** | +| --- | --- | +| `event` | `push`, `pull_request` or `incoming`. | +| `event_type` | The event type from the webhook payload header. Provider-specific (e.g., GitHub sends `pull_request`, GitLab is `Merge Request`, etc). | +| `target_branch` | The branch we are targeting. | +| `source_branch` | The branch where this pull_request comes from. (On `push`, this is the same as `target_branch`.) | +| `target_url` | The URL of the repository we are targeting. | +| `source_url` | The URL of the repository where this pull_request comes from. (On `push`, this is the same as `target_url`.) | +| `event_title` | Matches the title of the event. For `push`, it matches the commit title. For PR, it matches the Pull/Merge Request title. (Only supported for `GitHub`, `GitLab`, and `BitbucketCloud` providers.) | +| `body` | The full body as passed by the Git provider. Example: `body.pull_request.number` retrieves the pull request number on GitHub. | +| `headers` | The full set of headers as passed by the Git provider. Example: `headers['x-github-event']` retrieves the event type on GitHub. | +| `.pathChanged` | A suffix function to a string that can be a glob of a path to check if changed. (Supported only for `GitHub` and `GitLab` providers.) | +| `files` | The list of files that changed in the event (`all`, `added`, `deleted`, `modified`, and `renamed`). Example: `files.all` or `files.deleted`. For pull requests, every file belonging to the pull request will be listed. | +| Custom params | Any [custom parameters]({{< relref "/docs/guide/customparams" >}}) provided from the Repository CR `spec.params` are available as CEL variables. Example: `enable_ci == "true"`. See [Using custom parameters in CEL expressions: limitations](#using-custom-parameters-in-cel-expressions-limitations) below for important details. | CEL expressions let you do more complex filtering compared to the simple `on-target` annotation matching and enable more advanced scenarios. @@ -330,6 +332,66 @@ You can find more information about the CEL language spec here: {{< /hint >}} +### Using custom parameters in CEL expressions: limitations + +#### Filtered custom parameters and CEL evaluation + +When using a custom parameter with a `filter` in a CEL expression, be aware that if the filter condition +is **not met**, the parameter will be **undefined**, causing a CEL evaluation error rather than evaluating to false. + +For example, consider this Repository CR: + +```yaml +apiVersion: pipelinesascode.tekton.dev/v1alpha1 +kind: Repository +metadata: + name: my-repo +spec: + url: "https://github.com/owner/repo" + params: + - name: docker_registry + value: "registry.staging.example.com" + filter: pac.event_type == "pull_request" +``` + +And this PipelineRun: + +```yaml +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + name: my-pipeline + annotations: + pipelinesascode.tekton.dev/on-cel-expression: | + docker_registry == "registry.staging.example.com" +spec: + # ... pipeline spec +``` + +On a **push event**, the `docker_registry` parameter will not be defined (since the filter only matches pull +requests), and the CEL expression will produce an **error**, not `false`. The PipelineRun will not be +evaluated and an error will be reported. + +To avoid undefined parameter errors, ensure your CEL expressions only reference custom parameters when their +filter conditions match, or use parameters without filters for CEL matching. We recommend testing your CEL +expressions with different event types using the [tkn pac cel]({{< relref "/docs/guide/cli#tkn-pac-cel" >}}) +command to verify they work correctly across all scenarios + +#### Custom parameters do not override standard CEL variables + +Custom parameters defined in the Repository CR cannot override the built-in CEL variables provided by +Pipelines-as-Code, such as: + +* `event` (or `event_type`) +* `target_branch` +* `source_branch` +* `trigger_target` +* And other default variables documented in the table above + +If you define a custom parameter with the same name as a standard CEL variable, the standard variable will +take precedence in CEL expressions. Custom parameters should use unique names that don't conflict with +built-in variables. + ### Matching a PipelineRun to a branch with a regex In a CEL expression, you can match a field name using a regular expression. For diff --git a/pkg/matcher/annotation_matcher.go b/pkg/matcher/annotation_matcher.go index 4e8d9f8756..dc5d9be330 100644 --- a/pkg/matcher/annotation_matcher.go +++ b/pkg/matcher/annotation_matcher.go @@ -9,9 +9,11 @@ import ( "github.com/openshift-pipelines/pipelines-as-code/pkg/apis/pipelinesascode" "github.com/openshift-pipelines/pipelines-as-code/pkg/apis/pipelinesascode/keys" apipac "github.com/openshift-pipelines/pipelines-as-code/pkg/apis/pipelinesascode/v1alpha1" + "github.com/openshift-pipelines/pipelines-as-code/pkg/customparams" pacerrors "github.com/openshift-pipelines/pipelines-as-code/pkg/errors" "github.com/openshift-pipelines/pipelines-as-code/pkg/events" "github.com/openshift-pipelines/pipelines-as-code/pkg/formatting" + "github.com/openshift-pipelines/pipelines-as-code/pkg/kubeinteraction" "github.com/openshift-pipelines/pipelines-as-code/pkg/opscomments" "github.com/openshift-pipelines/pipelines-as-code/pkg/params" "github.com/openshift-pipelines/pipelines-as-code/pkg/params/info" @@ -210,6 +212,12 @@ func MatchPipelinerunByAnnotation(ctx context.Context, logger *zap.SugaredLogger } logger.Info(infomsg) + // Resolve custom params once for all PipelineRuns (for use in CEL expressions) + customParams := resolveCustomParamsForCEL(ctx, repo, event, cs, vcx, eventEmitter, logger) + if len(customParams) > 0 { + logger.Debugf("resolved %d custom params from repo for CEL", len(customParams)) + } + celValidationErrors := []*pacerrors.PacYamlValidations{} for _, prun := range pruns { prMatch := Match{ @@ -280,7 +288,7 @@ func MatchPipelinerunByAnnotation(ctx context.Context, logger *zap.SugaredLogger if celExpr, ok := prun.GetObjectMeta().GetAnnotations()[keys.OnCelExpression]; ok { checkPipelineRunAnnotation(prun, eventEmitter, repo) - out, err := celEvaluate(ctx, celExpr, event, vcx) + out, err := celEvaluate(ctx, celExpr, event, vcx, customParams) if err != nil { logger.Errorf("there was an error evaluating the CEL expression, skipping: %v", err) if checkIfCELEvaluateError(err) { @@ -508,3 +516,38 @@ func MatchRunningPipelineRunForIncomingWebhook(eventType, incomingPipelineRun st } return nil } + +// resolveCustomParamsForCEL resolves custom parameters from the Repository CR for use in CEL expressions. +// It returns a map of parameter names to values, excluding reserved keywords. +// All parameters are returned as strings, including those from secret_ref. +func resolveCustomParamsForCEL(ctx context.Context, repo *apipac.Repository, event *info.Event, cs *params.Run, vcx provider.Interface, eventEmitter *events.EventEmitter, logger *zap.SugaredLogger) map[string]string { + if repo == nil || repo.Spec.Params == nil { + return map[string]string{} + } + + // Create kubeinteraction interface + kinteract, err := kubeinteraction.NewKubernetesInteraction(cs) + if err != nil { + logger.Warnf("failed to create kubernetes interaction for custom params: %s", err.Error()) + return map[string]string{} + } + + // Use existing customparams package to resolve all params + cp := customparams.NewCustomParams(event, repo, cs, kinteract, eventEmitter, vcx) + allParams, _, err := cp.GetParams(ctx) + if err != nil { + eventEmitter.EmitMessage(repo, zap.WarnLevel, "CustomParamsCELError", + fmt.Sprintf("failed to resolve custom params for CEL: %s", err.Error())) + return map[string]string{} + } + + // Filter to only include params defined in repo.Spec.Params (not standard PAC params) + result := make(map[string]string) + for _, param := range *repo.Spec.Params { + if value, ok := allParams[param.Name]; ok { + result[param.Name] = value + } + } + + return result +} diff --git a/pkg/matcher/annotation_matcher_test.go b/pkg/matcher/annotation_matcher_test.go index d36d682408..b2730166db 100644 --- a/pkg/matcher/annotation_matcher_test.go +++ b/pkg/matcher/annotation_matcher_test.go @@ -1346,6 +1346,274 @@ func TestMatchPipelinerunAnnotationAndRepositories(t *testing.T) { }, }, }, + { + name: "cel/custom-params-simple", + args: annotationTestArgs{ + pruns: []*tektonv1.PipelineRun{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: pipelineTargetNSName, + Annotations: map[string]string{ + keys.OnCelExpression: "test_enabled == \"true\"", + }, + }, + }, + }, + runevent: info.Event{ + URL: targetURL, + TriggerTarget: "push", + BaseBranch: mainBranch, + EventType: "push", + }, + data: testclient.Data{ + Repositories: []*v1alpha1.Repository{ + testnewrepo.NewRepo( + testnewrepo.RepoTestcreationOpts{ + Name: "test-good", + URL: targetURL, + InstallNamespace: targetNamespace, + Params: &[]v1alpha1.Params{ + { + Name: "test_enabled", + Value: "true", + }, + }, + }, + ), + }, + }, + }, + wantPRName: pipelineTargetNSName, + wantErr: false, + }, + { + name: "cel/custom-params-multiple", + args: annotationTestArgs{ + pruns: []*tektonv1.PipelineRun{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: pipelineTargetNSName, + Annotations: map[string]string{ + keys.OnCelExpression: "environment == \"production\" && deploy_enabled == \"true\"", + }, + }, + }, + }, + runevent: info.Event{ + URL: targetURL, + TriggerTarget: "push", + BaseBranch: mainBranch, + EventType: "push", + }, + data: testclient.Data{ + Repositories: []*v1alpha1.Repository{ + testnewrepo.NewRepo( + testnewrepo.RepoTestcreationOpts{ + Name: "test-good", + URL: targetURL, + InstallNamespace: targetNamespace, + Params: &[]v1alpha1.Params{ + { + Name: "environment", + Value: "production", + }, + { + Name: "deploy_enabled", + Value: "true", + }, + }, + }, + ), + }, + }, + }, + wantPRName: pipelineTargetNSName, + wantErr: false, + }, + { + name: "cel/custom-params-not-matching", + args: annotationTestArgs{ + pruns: []*tektonv1.PipelineRun{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: pipelineTargetNSName, + Annotations: map[string]string{ + keys.OnCelExpression: "deploy_enabled == \"true\"", + }, + }, + }, + }, + runevent: info.Event{ + URL: targetURL, + TriggerTarget: "push", + BaseBranch: mainBranch, + EventType: "push", + }, + data: testclient.Data{ + Repositories: []*v1alpha1.Repository{ + testnewrepo.NewRepo( + testnewrepo.RepoTestcreationOpts{ + Name: "test-good", + URL: targetURL, + InstallNamespace: targetNamespace, + Params: &[]v1alpha1.Params{ + { + Name: "deploy_enabled", + Value: "false", + }, + }, + }, + ), + }, + }, + }, + wantPRName: "", + wantErr: true, + }, + { + name: "cel/custom-params-with-reserved-keyword", + args: annotationTestArgs{ + pruns: []*tektonv1.PipelineRun{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: pipelineTargetNSName, + Annotations: map[string]string{ + keys.OnCelExpression: "event == \"push\"", + }, + }, + }, + }, + runevent: info.Event{ + URL: targetURL, + TriggerTarget: "push", + BaseBranch: mainBranch, + EventType: "push", + }, + data: testclient.Data{ + Repositories: []*v1alpha1.Repository{ + testnewrepo.NewRepo( + testnewrepo.RepoTestcreationOpts{ + Name: "test-good", + URL: targetURL, + InstallNamespace: targetNamespace, + Params: &[]v1alpha1.Params{ + { + Name: "event", // Reserved keyword - should be ignored + Value: "invalid", + }, + }, + }, + ), + }, + }, + }, + wantPRName: pipelineTargetNSName, + wantErr: false, + }, + { + name: "cel/custom-params-combined-with-builtin", + args: annotationTestArgs{ + pruns: []*tektonv1.PipelineRun{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: pipelineTargetNSName, + Annotations: map[string]string{ + keys.OnCelExpression: "event == \"pull_request\" && target_branch == \"" + mainBranch + "\" && run_tests == \"true\"", + }, + }, + }, + }, + runevent: info.Event{ + URL: targetURL, + TriggerTarget: "pull_request", + BaseBranch: mainBranch, + HeadBranch: "feature-branch", + EventType: "pull_request", + PullRequestNumber: 123, + }, + data: testclient.Data{ + Repositories: []*v1alpha1.Repository{ + testnewrepo.NewRepo( + testnewrepo.RepoTestcreationOpts{ + Name: "test-good", + URL: targetURL, + InstallNamespace: targetNamespace, + Params: &[]v1alpha1.Params{ + { + Name: "run_tests", + Value: "true", + }, + }, + }, + ), + }, + }, + }, + wantPRName: pipelineTargetNSName, + wantErr: false, + }, + { + name: "cel/custom-params-from-secret", + args: annotationTestArgs{ + pruns: []*tektonv1.PipelineRun{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: pipelineTargetNSName, + Annotations: map[string]string{ + keys.OnCelExpression: "event == \"push\" && api_key != \"\" && api_token == \"secret-token-value\"", + }, + }, + }, + }, + runevent: info.Event{ + URL: targetURL, + TriggerTarget: "push", + BaseBranch: mainBranch, + EventType: "push", + }, + data: testclient.Data{ + Secret: []*corev1.Secret{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "my-secret", + Namespace: targetNamespace, + }, + Data: map[string][]byte{ + "key": []byte("secret-key-value"), + "token": []byte("secret-token-value"), + }, + }, + }, + Repositories: []*v1alpha1.Repository{ + testnewrepo.NewRepo( + testnewrepo.RepoTestcreationOpts{ + Name: "test-good", + URL: targetURL, + InstallNamespace: targetNamespace, + Params: &[]v1alpha1.Params{ + { + Name: "api_key", + SecretRef: &v1alpha1.Secret{ + Name: "my-secret", + Key: "key", + }, + }, + { + Name: "api_token", + SecretRef: &v1alpha1.Secret{ + Name: "my-secret", + Key: "token", + }, + }, + }, + }, + ), + }, + }, + }, + wantPRName: pipelineTargetNSName, + wantErr: false, + }, } for _, tt := range tests { @@ -1414,14 +1682,24 @@ func runTest(ctx context.Context, t *testing.T, tt annotationTest, vcx provider. logger := zap.New(observer).Sugar() vcx.SetLogger(logger) client := ¶ms.Run{ - Clients: clients.Clients{PipelineAsCode: cs.PipelineAsCode}, - Info: info.Info{}, + Clients: clients.Clients{ + PipelineAsCode: cs.PipelineAsCode, + Kube: cs.Kube, + }, + Info: info.Info{}, } eventEmitter := events.NewEventEmitter(cs.Kube, logger) + + // Get the repository for custom params resolution + var repo *v1alpha1.Repository + if len(tt.args.data.Repositories) > 0 { + repo = tt.args.data.Repositories[0] + } + matches, err := MatchPipelinerunByAnnotation(ctx, logger, tt.args.pruns, - client, &tt.args.runevent, vcx, eventEmitter, nil, + client, &tt.args.runevent, vcx, eventEmitter, repo, ) if tt.wantLog != "" { diff --git a/pkg/matcher/cel.go b/pkg/matcher/cel.go index ce48902d38..634c3c9135 100644 --- a/pkg/matcher/cel.go +++ b/pkg/matcher/cel.go @@ -22,7 +22,7 @@ const ( reChangedFilesTags = `files\.` ) -func celEvaluate(ctx context.Context, expr string, event *info.Event, vcx provider.Interface) (ref.Val, error) { +func celEvaluate(ctx context.Context, expr string, event *info.Event, vcx provider.Interface, customParams map[string]string) (ref.Val, error) { eventTitle := event.PullRequestTitle if event.TriggerTarget == triggertype.Push { eventTitle = event.SHATitle @@ -72,6 +72,7 @@ func celEvaluate(ctx context.Context, expr string, event *info.Event, vcx provid data := map[string]any{ "event": event.TriggerTarget.String(), + "event_type": event.EventType, "event_title": eventTitle, "target_branch": event.BaseBranch, "source_branch": event.HeadBranch, @@ -87,10 +88,12 @@ func celEvaluate(ctx context.Context, expr string, event *info.Event, vcx provid "renamed": changedFiles.Renamed, }, } - env, err := cel.NewEnv( + + varDecls := []cel.EnvOption{ cel.Lib(celPac{vcx, ctx, event}), cel.VariableDecls( decls.NewVariable("event", types.StringType), + decls.NewVariable("event_type", types.StringType), decls.NewVariable("headers", types.NewMapType(types.StringType, types.DynType)), decls.NewVariable("body", types.NewMapType(types.StringType, types.DynType)), decls.NewVariable("event_title", types.StringType), @@ -99,7 +102,18 @@ func celEvaluate(ctx context.Context, expr string, event *info.Event, vcx provid decls.NewVariable("target_url", types.StringType), decls.NewVariable("source_url", types.StringType), decls.NewVariable("files", types.NewMapType(types.StringType, types.DynType)), - )) + ), + } + // Add declarations for custom params (all as strings) + for k, v := range customParams { + // Don't overwrite standard params + if _, ok := data[k]; !ok { + data[k] = v + varDecls = append(varDecls, cel.VariableDecls(decls.NewVariable(k, types.StringType))) + } + } + + env, err := cel.NewEnv(varDecls...) if err != nil { return nil, err } diff --git a/pkg/test/repository/repository.go b/pkg/test/repository/repository.go index 41d11b6036..10192c5bc5 100644 --- a/pkg/test/repository/repository.go +++ b/pkg/test/repository/repository.go @@ -20,6 +20,7 @@ type RepoTestcreationOpts struct { RepoStatus []v1alpha1.RepositoryRunStatus ConcurrencyLimit int Settings *v1alpha1.Settings + Params *[]v1alpha1.Params } func NewRepo(opts RepoTestcreationOpts) *v1alpha1.Repository { @@ -97,5 +98,10 @@ func NewRepo(opts RepoTestcreationOpts) *v1alpha1.Repository { Name: opts.WebhookSecretName, } } + + if opts.Params != nil { + repo.Spec.Params = opts.Params + } + return repo }