Merge pull request #198 from DevanshMathur19/CI-18487

feat: [CI-18487]: Glob pattern support for PLUGIN_STRIP_PREFIX

Author: DevanshMathur19

README.md

@@ -61,6 +61,47 @@ docker run --rm \
   plugins/s3 --dry-run
 ```
 
+### Wildcard strip_prefix
+
+You can strip dynamic, runtime-generated directory prefixes from S3 keys using shell-style wildcards in `strip_prefix`.
+
+Supported patterns:
+
+- `*` matches exactly one path segment (no `/`).
+- `**` matches any depth (including zero segments).
+- `?` matches exactly one character (no `/`).
+
+The pattern is anchored at the start of the path. Use `/` to delimit directory boundaries. We recommend ending directory patterns with a trailing `/` to strip whole directory segments.
+
+Examples:
+
+- Pattern: `/harness/artifacts/*/`
+  - Path: `/harness/artifacts/build-123/module/app.zip`
+  - Result key suffix: `module/app.zip`
+
+- Pattern: `/harness/artifacts/**/`
+  - Path: `/harness/artifacts/build-123/deep/nested/file.zip`
+  - Result key suffix: `file.zip`
+
+- Pattern: `/harness/artifacts/*/services/`
+  - Path: `/harness/artifacts/build-123/services/auth/auth.zip`
+  - Result key suffix: `auth/auth.zip`
+
+Trailing slash semantics:
+
+- A trailing `/` indicates you are stripping up to a directory boundary.
+- Without a trailing `/`, the match may end mid-segment. For directory prefix stripping, prefer a trailing `/`.
+
+Windows notes:
+
+- `strip_prefix` must start with `/`. Backslashes are accepted and normalized to `/` internally (e.g. `\\harness\\artifacts\\*/` is allowed).
+- Windows drive letters like `C:\\...` are not supported and will be rejected.
+
+Dry-run logging:
+
+- With `--dry-run` or `PLUGIN_DRY_RUN=true`, the plugin logs what would be uploaded.
+- Fields include `name` (source), `target` (S3 key), `strip_pattern`, and `removed_prefix` (the portion stripped from the path when the pattern matches).
+
 * For Download
 ```
 docker run --rm \

main.go

@@ -86,7 +86,7 @@ func main() {
 		},
 		cli.StringFlag{
 			Name:   "strip-prefix",
-			Usage:  "used to add or remove a prefix from the source/target path",
+			Usage:  "prefix to strip from source path (supports wildcards: *, **, ?)",
 			EnvVar: "PLUGIN_STRIP_PREFIX",
 		},
 		cli.StringSliceFlag{
@@ -163,7 +163,6 @@ func run(c *cli.Context) error {
 		_ = godotenv.Load(c.String("env-file"))
 	}
 
-
 	plugin := Plugin{
 		Endpoint:              c.String("endpoint"),
 		Key:                   c.String("access-key"),
@@ -193,4 +192,3 @@ func run(c *cli.Context) error {
 
 	return plugin.Exec()
 }
-

plugin.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"io"
 	"mime"
 	"os"
@@ -84,7 +85,7 @@ type Plugin struct {
 	Source string
 	Target string
 
-	// Strip the prefix from the target path
+	// Strip the prefix from the target path (supports wildcards)
 	StripPrefix string
 
 	// Exclude files matching this pattern.
@@ -138,13 +139,83 @@ func (p *Plugin) Exec() error {
 		return err
 	}
 
+	// Validate strip prefix pattern and precompile regex once
+	normalizedStrip := strings.ReplaceAll(p.StripPrefix, "\\", "/")
+	if p.StripPrefix != "" && strings.HasPrefix(normalizedStrip, "/") {
+		if err := validateStripPrefix(p.StripPrefix); err != nil {
+			log.WithFields(log.Fields{
+				"error":   err,
+				"pattern": p.StripPrefix,
+			}).Error("Invalid strip_prefix pattern")
+			return err
+		}
+	}
+
+	var compiled *regexp.Regexp
+	if normalizedStrip != "" && strings.HasPrefix(normalizedStrip, "/") && strings.ContainsAny(normalizedStrip, "*?") {
+		var err error
+		compiled, err = patternToRegex(normalizedStrip)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"error":   err,
+				"pattern": p.StripPrefix,
+			}).Error("Failed to compile strip_prefix pattern")
+			return err
+		}
+	}
+
+	anyMatched := false
+
 	for _, match := range matches {
 		// skip directories
 		if isDir(match, matches) {
 			continue
 		}
 
-		target := resolveKey(p.Target, match, p.StripPrefix)
+		// Preview stripping (wildcard for absolute patterns, literal for relative patterns)
+		stripped := match
+		matched := false
+		if normalizedStrip != "" {
+			if strings.HasPrefix(normalizedStrip, "/") {
+				var err error
+				stripped, matched, err = stripWildcardPrefixWithRegex(match, normalizedStrip, compiled)
+				if err != nil {
+					log.WithFields(log.Fields{
+						"error":   err,
+						"path":    match,
+						"pattern": p.StripPrefix,
+					}).Warn("Failed to strip prefix, using original path")
+					stripped = match
+				}
+			} else {
+				// Backward-compat: literal TrimPrefix for relative strip_prefix (no leading '/')
+				m := filepath.ToSlash(match)
+				trimmed := strings.TrimPrefix(m, normalizedStrip)
+				if trimmed != m {
+					matched = true
+					stripped = trimmed
+				} else {
+					stripped = match
+				}
+			}
+		}
+		if matched {
+			anyMatched = true
+		}
+
+		// Build final key
+		var target string
+		if normalizedStrip != "" && !strings.HasPrefix(normalizedStrip, "/") {
+			// Relative strip_prefix: use master resolveKey behavior
+			target = resolveKey(p.Target, filepath.ToSlash(match), p.StripPrefix)
+		} else {
+			// Absolute strip_prefix (wildcards): join stripped suffix under target
+			rel := strings.TrimPrefix(filepath.ToSlash(stripped), "/")
+			target = filepath.ToSlash(filepath.Join(p.Target, rel))
+			if !strings.HasPrefix(target, "/") {
+				target = "/" + target
+			}
+		}
 
 		contentType := matchExtension(match, p.ContentType)
 		contentEncoding := matchExtension(match, p.ContentEncoding)
@@ -165,9 +236,22 @@ func (p *Plugin) Exec() error {
 			"target": target,
 		}).Info("Uploading file")
 
-		// when executing a dry-run we exit because we don't actually want to
-		// upload the file to S3.
+		// when executing a dry-run print what would be stripped and skip upload.
 		if p.DryRun {
+			removed := ""
+			if matched {
+				// removed prefix = original - stripped suffix
+				orig := filepath.ToSlash(match)
+				rem := strings.TrimSuffix(orig, filepath.ToSlash(stripped))
+				removed = rem
+			}
+			log.WithFields(log.Fields{
+				"name":           match,
+				"bucket":         p.Bucket,
+				"target":         target,
+				"strip_pattern":  p.StripPrefix,
+				"removed_prefix": removed,
+			}).Info("Dry-run: would upload")
 			continue
 		}
 
@@ -226,6 +310,12 @@ func (p *Plugin) Exec() error {
 		f.Close()
 	}
 
+	if normalizedStrip != "" && !anyMatched {
+		log.WithFields(log.Fields{
+			"pattern": p.StripPrefix,
+		}).Warn("strip_prefix did not match any paths; keys will include original path")
+	}
+
 	return nil
 }
 
@@ -370,7 +460,6 @@ func (p *Plugin) downloadS3Object(client *s3.S3, sourceDir, key, target string)
 
 	// Create the destination file path
 	destination := filepath.Join(p.Target, target)
-	log.Println("Destination: ", destination)
 
 	// Extract the directory from the destination path
 	dir := filepath.Dir(destination)
@@ -466,7 +555,6 @@ func (p *Plugin) createS3Client() *s3.S3 {
 		log.Warn("AWS Key and/or Secret not provided (falling back to ec2 instance profile)")
 	}
 
-
 	// Create session with primary credentials
 	sess, err = session.NewSession(conf)
 	if err != nil {
@@ -479,8 +567,8 @@ func (p *Plugin) createS3Client() *s3.S3 {
 	// Handle secondary role assumption if UserRoleArn is provided
 	if len(p.UserRoleArn) > 0 {
 		log.WithField("UserRoleArn", p.UserRoleArn).Info("Using user role ARN")
-		
-		// Create credentials using the existing session for role assumption 
+
+		// Create credentials using the existing session for role assumption
 		// by assuming the UserRoleArn (with ExternalID when provided)
 		creds := stscreds.NewCredentials(sess, p.UserRoleArn, func(provider *stscreds.AssumeRoleProvider) {
 			if p.UserRoleExternalID != "" {
@@ -508,3 +596,113 @@ func assumeRoleWithWebIdentity(sess *session.Session, roleArn, roleSessionName,
 	}
 	return credentials.NewStaticCredentials(*result.Credentials.AccessKeyId, *result.Credentials.SecretAccessKey, *result.Credentials.SessionToken), nil
 }
+
+// validateStripPrefix validates a strip prefix pattern with wildcards
+func validateStripPrefix(pattern string) error {
+	// Normalize Windows backslashes to forward slashes for validation (OS-independent)
+	pattern = strings.ReplaceAll(pattern, "\\", "/")
+
+	// Pattern must start with /
+	if !strings.HasPrefix(pattern, "/") {
+		return fmt.Errorf("strip_prefix must start with '/'")
+	}
+
+	// Reject Windows drive-letter prefixes like C:/...
+	if len(pattern) >= 2 && pattern[1] == ':' {
+		return fmt.Errorf("strip_prefix must be an absolute POSIX-style path (e.g. '/root/...'), drive letters are not supported")
+	}
+
+	// Check length limit
+	if len(pattern) > 256 {
+		return fmt.Errorf("strip_prefix pattern too long (max 256 characters)")
+	}
+
+	// Count wildcards
+	wildcardCount := strings.Count(pattern, "*") + strings.Count(pattern, "?")
+	if wildcardCount > 20 {
+		return fmt.Errorf("strip_prefix pattern contains too many wildcards (max 20)")
+	}
+
+	// Check for empty segments
+	if strings.Contains(pattern, "//") {
+		return fmt.Errorf("strip_prefix pattern contains empty segment '//'")
+	}
+
+	// Check for invalid ** usage (must be standalone segment)
+	parts := strings.Split(pattern, "/")
+	for _, part := range parts {
+		if strings.Contains(part, "**") && part != "**" {
+			return fmt.Errorf("'**' must be a standalone directory segment")
+		}
+	}
+
+	return nil
+}
+
+// patternToRegex converts shell-style wildcards to regex
+func patternToRegex(pattern string) (*regexp.Regexp, error) {
+	// Escape special regex characters except our wildcards
+	escaped := regexp.QuoteMeta(pattern)
+
+	// Replace escaped wildcards with regex equivalents
+	// Order matters: ** must be replaced before *
+	escaped = strings.ReplaceAll(escaped, `\*\*`, "(.+)")  // ** -> (.+) any depth
+	escaped = strings.ReplaceAll(escaped, `\*`, "([^/]+)") // * -> ([^/]+) one segment
+	escaped = strings.ReplaceAll(escaped, `\?`, "([^/])")  // ? -> ([^/]) one character
+
+	// Anchor at start
+	escaped = "^" + escaped
+
+	return regexp.Compile(escaped)
+}
+
+// stripWildcardPrefixWithRegex strips prefix using wildcard pattern matching, reusing
+// a precompiled regex when provided. It returns the possibly stripped path, whether
+// the pattern matched, and any error if stripping would remove the entire key.
+func stripWildcardPrefixWithRegex(path, pattern string, re *regexp.Regexp) (string, bool, error) {
+	if pattern == "" {
+		return path, false, nil
+	}
+
+	// Normalize paths to forward slashes (OS-independent)
+	path = strings.ReplaceAll(path, "\\", "/")
+	pattern = strings.ReplaceAll(pattern, "\\", "/")
+
+	// Literal prefix (no wildcards)
+	if !strings.ContainsAny(pattern, "*?") {
+		if !strings.HasPrefix(path, pattern) {
+			return path, false, nil
+		}
+		stripped := strings.TrimPrefix(path, pattern)
+		if stripped == "" || stripped == "/" || strings.TrimPrefix(stripped, "/") == "" {
+			return path, true, fmt.Errorf("strip_prefix removes entire path for '%s'", filepath.Base(path))
+		}
+		return stripped, true, nil
+	}
+
+	// Wildcard pattern
+	var err error
+	if re == nil {
+		re, err = patternToRegex(pattern)
+		if err != nil {
+			return path, false, fmt.Errorf("invalid pattern: %v", err)
+		}
+	}
+
+	m := re.FindStringSubmatch(path)
+	if len(m) == 0 {
+		return path, false, nil
+	}
+	full := m[0]
+	stripped := strings.TrimPrefix(path, full)
+	if stripped == "" || stripped == "/" || strings.TrimPrefix(stripped, "/") == "" {
+		return path, true, fmt.Errorf("strip_prefix removes entire path for '%s'", filepath.Base(path))
+	}
+	return stripped, true, nil
+}
+
+// stripWildcardPrefix strips prefix using wildcard pattern matching
+func stripWildcardPrefix(path, pattern string) (string, error) {
+	stripped, _, err := stripWildcardPrefixWithRegex(path, pattern, nil)
+	return stripped, err
+}