feat: support recursive pattern matching in file fetching (#405)

* feat: support recursive pattern matching in file fetching

- Introduced the github.com/bmatcuk/doublestar/v4 dependency to support recursive pattern matching in file fetching.
- Updated fetch_test.go to include tests for new pattern matching capabilities, including subdirectory and wildcard patterns.
- Adjusted file paths in tests to reflect new structure.

Signed-off-by: Zhao Chen <zhaochen.zju@gmail.com>

* refactor: updated the dragonfly function to utilize doublestar.Match

- Updated the fetchByDragonfly function to utilize doublestar.Match, allowing for both simple and recursive pattern matching in file fetching.

Signed-off-by: Zhao Chen <zhaochen.zju@gmail.com>

* refactor: replace doublestar.Match with doublestar.PathMatch for improved pattern matching

- Updated the fetchByDragonfly and Fetch functions to use doublestar.PathMatch, enhancing pattern matching capabilities to support system-native path separators and recursive patterns.

Signed-off-by: Zhao Chen <zhaochen.zju@gmail.com>

---------

Signed-off-by: Zhao Chen <zhaochen.zju@gmail.com>

Author: aftersnow

go.mod

@@ -62,6 +62,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect
 	github.com/aws/smithy-go v1.24.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/clipperhouse/stringish v0.1.1 // indirect

go.sum

@@ -77,6 +77,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs=
+github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
 github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=

pkg/backend/fetch.go

@@ -20,8 +20,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"path/filepath"
 
+	"github.com/bmatcuk/doublestar/v4"
 	legacymodelspec "github.com/dragonflyoss/model-spec/specs-go/v1"
 	modelspec "github.com/modelpack/model-spec/specs-go/v1"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -78,7 +78,10 @@ func (b *backend) Fetch(ctx context.Context, target string, cfg *config.Fetch) e
 				if path == "" {
 					path = anno[legacymodelspec.AnnotationFilepath]
 				}
-				matched, err := filepath.Match(pattern, path)
+				// Use doublestar.PathMatch for pattern matching to support ** recursive matching
+				// PathMatch uses the system's native path separator (like filepath.Match) while
+				// also supporting recursive patterns like **/*.json
+				matched, err := doublestar.PathMatch(pattern, path)
 				if err != nil {
 					return fmt.Errorf("failed to match pattern: %w", err)
 				}

pkg/backend/fetch_by_d7y.go

@@ -28,6 +28,7 @@ import (
 	common "d7y.io/api/v2/pkg/apis/common/v2"
 	dfdaemon "d7y.io/api/v2/pkg/apis/dfdaemon/v2"
 	"github.com/avast/retry-go/v4"
+	"github.com/bmatcuk/doublestar/v4"
 	legacymodelspec "github.com/dragonflyoss/model-spec/specs-go/v1"
 	modelspec "github.com/modelpack/model-spec/specs-go/v1"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -81,7 +82,10 @@ func (b *backend) fetchByDragonfly(ctx context.Context, target string, cfg *conf
 				if path == "" {
 					path = anno[legacymodelspec.AnnotationFilepath]
 				}
-				matched, err := filepath.Match(pattern, path)
+				// Use doublestar.PathMatch for pattern matching to support ** recursive matching
+				// PathMatch uses the system's native path separator (like filepath.Match) while
+				// also supporting recursive patterns like **/*.json
+				matched, err := doublestar.PathMatch(pattern, path)
 				if err != nil {
 					return fmt.Errorf("failed to match pattern: %w", err)
 				}

pkg/backend/fetch_test.go

@@ -73,7 +73,7 @@ func TestFetch(t *testing.T) {
 						Digest:    file2Digest,
 						Size:      int64(len(file2Content)),
 						Annotations: map[string]string{
-							modelspec.AnnotationFilepath: "file2.txt",
+							modelspec.AnnotationFilepath: "subdir/file2.txt",
 						},
 					},
 				},
@@ -118,11 +118,33 @@ func TestFetch(t *testing.T) {
 			expectError: false,
 		},
 		{
-			name:   "fetch with pattern matching both files",
+			name:   "fetch with pattern matching subdirectory file",
 			target: url + "/test/model:latest",
 			cfg: &config.Fetch{
 				Output:      tempDir,
-				Patterns:    []string{"file*.txt"},
+				Patterns:    []string{"subdir/*.txt"},
+				PlainHTTP:   true,
+				Concurrency: 2,
+			},
+			expectError: false,
+		},
+		{
+			name:   "fetch with recursive pattern matching all txt files",
+			target: url + "/test/model:latest",
+			cfg: &config.Fetch{
+				Output:      tempDir,
+				Patterns:    []string{"**/*.txt"},
+				PlainHTTP:   true,
+				Concurrency: 2,
+			},
+			expectError: false,
+		},
+		{
+			name:   "fetch with wildcard pattern (old behavior)",
+			target: url + "/test/model:latest",
+			cfg: &config.Fetch{
+				Output:      tempDir,
+				Patterns:    []string{"*.txt"},
 				PlainHTTP:   true,
 				Concurrency: 2,
 			},