diff --git a/.github/workflows/async-upload-test.yml b/.github/workflows/async-upload-test.yml index fdf65c515a..53b16e8285 100644 --- a/.github/workflows/async-upload-test.yml +++ b/.github/workflows/async-upload-test.yml @@ -39,7 +39,7 @@ jobs: run: working-directory: jobs/async-upload steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.0 - name: Set up Python uses: actions/setup-python@v6 with: @@ -66,7 +66,7 @@ jobs: run: working-directory: jobs/async-upload steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.0 - name: Set up Python uses: actions/setup-python@v6 with: @@ -89,7 +89,7 @@ jobs: working-directory: jobs/async-upload steps: - name: Check out the repository - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.0 with: fetch-depth: 0 - name: Set up Python diff --git a/.github/workflows/build-and-push-async-upload.yml b/.github/workflows/build-and-push-async-upload.yml index 1a5d59f597..9868b289f9 100644 --- a/.github/workflows/build-and-push-async-upload.yml +++ b/.github/workflows/build-and-push-async-upload.yml @@ -31,7 +31,7 @@ jobs: packages: write steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/build-and-push-csi-image.yml b/.github/workflows/build-and-push-csi-image.yml index 4d958a85d8..07f76db02c 100644 --- a/.github/workflows/build-and-push-csi-image.yml +++ b/.github/workflows/build-and-push-csi-image.yml @@ -38,7 +38,7 @@ jobs: if: github.head_ref == '' && github.ref == 'refs/heads/main' run: echo "BUILD_CONTEXT=main" >> $GITHUB_ENV # checkout branch - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.0 # set image version - name: Set main-branch environment if: env.BUILD_CONTEXT == 'main' diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml index 2796c0683a..ea2fc5cc47 100644 --- a/.github/workflows/build-and-push-image.yml +++ b/.github/workflows/build-and-push-image.yml @@ -42,7 +42,7 @@ jobs: if: github.head_ref == '' && github.ref == 'refs/heads/main' run: echo "BUILD_CONTEXT=main" >> $GITHUB_ENV # checkout branch - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.0 # Set up QEMU for multi-architecture builds - name: Set up QEMU uses: docker/setup-qemu-action@v3 diff --git a/.github/workflows/build-and-push-ui-images-standalone.yml b/.github/workflows/build-and-push-ui-images-standalone.yml index e180e878fe..b7c2ee90da 100644 --- a/.github/workflows/build-and-push-ui-images-standalone.yml +++ b/.github/workflows/build-and-push-ui-images-standalone.yml @@ -29,7 +29,7 @@ jobs: packages: write steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/build-and-push-ui-images.yml b/.github/workflows/build-and-push-ui-images.yml index 2d4766fe83..76ede95a35 100644 --- a/.github/workflows/build-and-push-ui-images.yml +++ b/.github/workflows/build-and-push-ui-images.yml @@ -29,7 +29,7 @@ jobs: packages: write steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/build-image-pr.yml b/.github/workflows/build-image-pr.yml index 5a225deb75..fa5806d388 100644 --- a/.github/workflows/build-image-pr.yml +++ b/.github/workflows/build-image-pr.yml @@ -25,7 +25,7 @@ jobs: build-and-test-image: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.0 - name: Generate Tag shell: bash id: tags diff --git a/.github/workflows/build-image-ui-pr.yml b/.github/workflows/build-image-ui-pr.yml index f941073ee6..a76e54ebce 100644 --- a/.github/workflows/build-image-ui-pr.yml +++ b/.github/workflows/build-image-ui-pr.yml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest steps: # checkout branch - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.0 - name: Build UI Image shell: bash run: ./scripts/build_deploy.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e575bd92f5..f2d23a045b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.0 - name: Setup Go uses: actions/setup-go@v6 with: diff --git a/.github/workflows/check-db-schema-structs.yaml b/.github/workflows/check-db-schema-structs.yaml index cc3f990e44..056755b7d2 100644 --- a/.github/workflows/check-db-schema-structs.yaml +++ b/.github/workflows/check-db-schema-structs.yaml @@ -14,7 +14,7 @@ jobs: check-mysql-schema-structs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.0 - name: Setup Go uses: actions/setup-go@v6 with: @@ -34,7 +34,7 @@ jobs: check-postgres-schema-structs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.0 - name: Setup Go uses: actions/setup-go@v6 with: diff --git a/.github/workflows/check-gitattributes.yaml b/.github/workflows/check-gitattributes.yaml index 24cd4d424e..edbcafbb3d 100644 --- a/.github/workflows/check-gitattributes.yaml +++ b/.github/workflows/check-gitattributes.yaml @@ -8,6 +8,6 @@ jobs: validate: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.0 - name: Validate OpenAPI spec run: ./scripts/gen_gitattributes.sh --check diff --git a/.github/workflows/check-openapi-spec-pr.yaml b/.github/workflows/check-openapi-spec-pr.yaml index 7a63e635c9..cd96a78057 100644 --- a/.github/workflows/check-openapi-spec-pr.yaml +++ b/.github/workflows/check-openapi-spec-pr.yaml @@ -12,7 +12,7 @@ jobs: validate: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.0 - name: Validate OpenAPI spec run: | make openapi/validate diff --git a/.github/workflows/controller-test.yml b/.github/workflows/controller-test.yml index e57be6d30a..f0ecb23237 100644 --- a/.github/workflows/controller-test.yml +++ b/.github/workflows/controller-test.yml @@ -39,7 +39,7 @@ jobs: echo "tag=${tag}" >> $GITHUB_OUTPUT - name: Clone the code - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.0 - name: Setup Go uses: actions/setup-go@v6 diff --git a/.github/workflows/csi-test.yml b/.github/workflows/csi-test.yml index 8b16160291..652f39413b 100644 --- a/.github/workflows/csi-test.yml +++ b/.github/workflows/csi-test.yml @@ -35,7 +35,7 @@ jobs: build-and-test-csi-image: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.0 - name: Generate tag shell: bash diff --git a/.github/workflows/first-time-contributor-pr.yml b/.github/workflows/first-time-contributor-pr.yml index 87312d8117..7c0729cfff 100644 --- a/.github/workflows/first-time-contributor-pr.yml +++ b/.github/workflows/first-time-contributor-pr.yml @@ -15,13 +15,14 @@ permissions: # set contents: read at top-level, per OpenSSF ScoreCard rule Token jobs: welcome: + if: ${{ github.actor != 'dependabot[bot]' }} runs-on: ubuntu-latest permissions: pull-requests: write issues: write steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.0 - name: Install PyYAML run: pip3 install pyyaml diff --git a/.github/workflows/go-mod-tidy-diff-check.yml b/.github/workflows/go-mod-tidy-diff-check.yml index 1771f16f6e..50f8bc9b9b 100644 --- a/.github/workflows/go-mod-tidy-diff-check.yml +++ b/.github/workflows/go-mod-tidy-diff-check.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.0 - name: Set up Go uses: actions/setup-go@v6 diff --git a/.github/workflows/prepare.yml b/.github/workflows/prepare.yml index 46352e0430..e862b841d6 100644 --- a/.github/workflows/prepare.yml +++ b/.github/workflows/prepare.yml @@ -8,7 +8,7 @@ jobs: prepare: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.0 - name: Setup Go uses: actions/setup-go@v6 with: diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index 329ba4b9a2..13fc10106b 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -16,7 +16,7 @@ jobs: FORCE_COLOR: "1" steps: - name: Check out the repository - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.0 with: fetch-depth: 0 - name: Set up Python diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index a9d28c1441..8661f6f8e3 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -31,7 +31,7 @@ jobs: FORCE_COLOR: "1" steps: - name: Check out the repository - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.0 - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v6 with: @@ -74,7 +74,7 @@ jobs: nodejs: ["20"] steps: - name: Check out the repository - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.0 with: fetch-depth: 0 - name: Set up Python @@ -148,7 +148,7 @@ jobs: DEPLOY_MANIFEST_DB: "${{ matrix.manifest-db }}" steps: - name: Check out the repository - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.0 - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v6 with: @@ -227,7 +227,7 @@ jobs: run: | kubectl port-forward -n ${{ env.MR_NAMESPACE }} service/model-registry-service 8080:8080 & kubectl port-forward -n minio svc/minio 9000:9000 & - kubectl port-forward service/distribution-registry-test-service 5001:5001 & + kubectl port-forward service/distribution-registry-test-service -n local-oci-registry-ns 5001:5001 & sleep 2 nox --python=${{ matrix.python }} --session=e2e -- --cov-report=xml - name: Upload coverage report # we upload coverage stats for py once, regardless if previous step failed @@ -251,7 +251,7 @@ jobs: run: | kubectl port-forward -n ${{ env.MR_NAMESPACE }} service/model-registry-service 8080:8080 > /dev/null 2>&1 & kubectl port-forward -n minio svc/minio 9000:9000 > /dev/null 2>&1 & - kubectl port-forward service/distribution-registry-test-service 5001:5001 > /dev/null 2>&1 & + kubectl port-forward service/distribution-registry-test-service -n local-oci-registry-ns 5001:5001 > /dev/null 2>&1 & sleep 2 nox --python=${{ matrix.python }} --session=fuzz @@ -268,7 +268,7 @@ jobs: FORCE_COLOR: "1" steps: - name: Check out the repository - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.0 - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v6 with: diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 7a139b4e9e..3b3a5550c7 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -31,7 +31,7 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.0 + uses: actions/checkout@v6.0.0 # unify for Dependabot bump with: persist-credentials: false diff --git a/.github/workflows/test-fuzz.yml b/.github/workflows/test-fuzz.yml index b5dd618498..7d18ff4b34 100644 --- a/.github/workflows/test-fuzz.yml +++ b/.github/workflows/test-fuzz.yml @@ -53,7 +53,7 @@ jobs: } - name: Checkout PR - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.0 with: ref: ${{ fromJson(steps.pr.outputs.result).sha }} diff --git a/.github/workflows/trivy-image-scanning.yaml b/.github/workflows/trivy-image-scanning.yaml index e829bce857..373c453c03 100644 --- a/.github/workflows/trivy-image-scanning.yaml +++ b/.github/workflows/trivy-image-scanning.yaml @@ -24,7 +24,7 @@ jobs: ] steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.0 - name: Sanitize image name for SARIF filename run: | diff --git a/.github/workflows/ui-bff-build.yml b/.github/workflows/ui-bff-build.yml index 46dd17a715..1d5bd4cda0 100644 --- a/.github/workflows/ui-bff-build.yml +++ b/.github/workflows/ui-bff-build.yml @@ -26,7 +26,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.0 - name: Setup Go uses: actions/setup-go@v6 diff --git a/.github/workflows/ui-frontend-build.yml b/.github/workflows/ui-frontend-build.yml index 8f7e3689ef..3875175cd4 100644 --- a/.github/workflows/ui-frontend-build.yml +++ b/.github/workflows/ui-frontend-build.yml @@ -26,7 +26,7 @@ jobs: test-and-build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.0 - name: Set up Node.js uses: actions/setup-node@v6 diff --git a/catalog/README.md b/catalog/README.md index bf196f2da1..0b5b765573 100644 --- a/catalog/README.md +++ b/catalog/README.md @@ -13,6 +13,7 @@ The catalog service operates as a **metadata aggregation layer** that: ### Supported Catalog Sources - **YAML Catalog** - Static YAML files containing model metadata +- **HuggingFace Hub** - Discover models from HuggingFace's model repository ## REST API @@ -75,6 +76,76 @@ catalogs: path: "./models" ``` +### HuggingFace Source Configuration + +The HuggingFace catalog source allows you to discover and import models from the HuggingFace Hub. To configure a HuggingFace source: + +#### 1. Set Your API Key + +The HuggingFace provider requires an API key for authentication. By default, the service reads the API key from the `HF_API_KEY` environment variable: + +```bash +export HF_API_KEY="your-huggingface-api-key-here" +``` + +**Getting a HuggingFace API Key:** +1. Sign up or log in to [HuggingFace](https://huggingface.co) +2. Go to your [Settings > Access Tokens](https://huggingface.co/settings/tokens) +3. Create a new token with "Read" permissions +4. Copy the token and set it as an environment variable + +**For Kubernetes deployments:** +- Store the API key in a Kubernetes Secret +- Reference it in your deployment configuration +- The catalog service will read it from the configured environment variable (defaults to `HF_API_KEY`) + +**Custom Environment Variable Name:** +You can configure a custom environment variable name per source by setting the `apiKeyEnvVar` property in your source configuration (see below). This is useful when you need different API keys for different sources. + +**Important Notes:** +- **Private Models**: For private models, the API key must belong to an account that has been granted access to the model. Without proper access, the catalog service will not be able to retrieve model information. +- **Gated Models**: For gated models (models with usage restrictions), you must accept the model's terms of service on HuggingFace before the catalog service can access all available model information. Visit the model's page on HuggingFace and accept the terms to ensure full metadata is available. + +#### 2. Configure the Source + +Add a HuggingFace source to your `catalog-sources.yaml`: + +```yaml +catalogs: + - name: "HuggingFace Hub" + id: "huggingface" + type: "hf" + enabled: true + # Required: List of model identifiers to include + # Format: "organization/model-name" or "username/model-name" + includedModels: + - "meta-llama/Llama-3.1-8B-Instruct" + - "ibm-granite/granite-4.0-h-small" + - "microsoft/phi-2" + + # Optional: Exclude specific models or patterns + # Supports exact matches or patterns ending with "*" + excludedModels: + - "some-org/unwanted-model" + - "another-org/test-*" # Excludes all models starting with "test-" + + # Optional: Configure a custom environment variable name for the API key + # Defaults to "HF_API_KEY" if not specified + properties: + apiKeyEnvVar: "MY_CUSTOM_API_KEY_VAR" +``` + +#### Model Filtering + +Both `includedModels` and `excludedModels` are top-level properties (not nested under `properties`): + +- **`includedModels`** (required): List of model identifiers to fetch from HuggingFace. Format: `"organization/model-name"` or `"username/model-name"` +- **`excludedModels`** (optional): List of models or patterns to exclude from the results + +The `excludedModels` property supports: +- **Exact matches**: `"meta-llama/Llama-3.1-8B-Instruct"` - excludes this specific model +- **Pattern matching**: `"test-*"` - excludes all models starting with "test-" + ## Development ### Prerequisites diff --git a/catalog/internal/catalog/assets/catalog_logo.svg b/catalog/internal/catalog/assets/catalog_logo.svg new file mode 100644 index 0000000000..d1fb11bd47 --- /dev/null +++ b/catalog/internal/catalog/assets/catalog_logo.svg @@ -0,0 +1,2 @@ + + diff --git a/catalog/internal/catalog/catalog_test.go b/catalog/internal/catalog/catalog_test.go index c398dce05d..cba92bb2b8 100644 --- a/catalog/internal/catalog/catalog_test.go +++ b/catalog/internal/catalog/catalog_test.go @@ -354,7 +354,14 @@ func TestLoadCatalogSourcesWithMockRepositories(t *testing.T) { l := NewLoader(services, []string{}) ctx := context.Background() - err := l.updateDatabase(ctx, "test-path", testConfig) + // First call updateSources to populate the SourceCollection + // (updateDatabase now uses merged sources from the collection) + err := l.updateSources("test-path", testConfig) + if err != nil { + t.Fatalf("updateSources() error = %v", err) + } + + err = l.updateDatabase(ctx) if err != nil { t.Fatalf("updateDatabase() error = %v", err) } @@ -440,9 +447,15 @@ func TestLoadCatalogSourcesWithRepositoryErrors(t *testing.T) { l := NewLoader(services, []string{}) ctx := context.Background() + // First call updateSources to populate the SourceCollection + err := l.updateSources("test-path", testConfig) + if err != nil { + t.Fatalf("updateSources() error = %v", err) + } + // This should not return an error even if repository operations fail // (errors are logged but don't stop the loading process) - err := l.updateDatabase(ctx, "test-path", testConfig) + err = l.updateDatabase(ctx) if err != nil { t.Fatalf("updateDatabase() should not fail even with repository errors, got error = %v", err) } diff --git a/catalog/internal/catalog/hf_catalog.go b/catalog/internal/catalog/hf_catalog.go index 0ae837b749..00b459b338 100644 --- a/catalog/internal/catalog/hf_catalog.go +++ b/catalog/internal/catalog/hf_catalog.go @@ -2,78 +2,587 @@ package catalog import ( "context" + _ "embed" + "encoding/base64" + "encoding/json" "fmt" + "io" "net/http" + "os" + "strconv" "strings" "time" "github.com/golang/glog" - "github.com/kubeflow/model-registry/catalog/pkg/openapi" - model "github.com/kubeflow/model-registry/catalog/pkg/openapi" + dbmodels "github.com/kubeflow/model-registry/catalog/internal/db/models" + apimodels "github.com/kubeflow/model-registry/catalog/pkg/openapi" + "github.com/kubeflow/model-registry/internal/db/models" ) -type hfCatalogImpl struct { - client *http.Client - apiKey string - baseURL string +const ( + defaultHuggingFaceURL = "https://huggingface.co" + defaultAPIKeyEnvVar = "HF_API_KEY" + urlKey = "url" + apiKeyEnvVarKey = "apiKeyEnvVar" +) + +// gatedString is a custom type that can unmarshal both boolean and string values from JSON +// It converts booleans to strings (false -> "false", true -> "true") +type gatedString string + +// UnmarshalJSON implements json.Unmarshaler to handle both boolean and string values +func (g *gatedString) UnmarshalJSON(data []byte) error { + // Handle null/empty + if len(data) == 0 || string(data) == "null" { + *g = gatedString("") + return nil + } + + // Try to unmarshal as boolean first (handles true/false) + var b bool + if err := json.Unmarshal(data, &b); err == nil { + *g = gatedString(strconv.FormatBool(b)) + return nil + } + + // If not a boolean, try as string (handles quoted strings) + var s string + if err := json.Unmarshal(data, &s); err != nil { + return fmt.Errorf("gated field must be boolean or string, got: %s", string(data)) + } + *g = gatedString(s) + return nil } -var _ APIProvider = &hfCatalogImpl{} +// String returns the string value +func (g gatedString) String() string { + return string(g) +} -const ( - defaultHuggingFaceURL = "https://huggingface.co" +// hfModel implements apimodels.CatalogModel and populates it from HuggingFace API data +type hfModel struct { + apimodels.CatalogModel +} + +type hfModelProvider struct { + client *http.Client + sourceId string + apiKey string + baseURL string + includedModels []string + filter *ModelFilter +} + +// hfModelInfo represents the structure of HuggingFace API model information +type hfModelInfo struct { + ID string `json:"id"` + Author string `json:"author,omitempty"` + Sha string `json:"sha,omitempty"` + CreatedAt string `json:"createdAt,omitempty"` + UpdatedAt string `json:"updatedAt,omitempty"` + Private bool `json:"private,omitempty"` + Gated gatedString `json:"gated,omitempty"` + Downloads int `json:"downloads,omitempty"` + Tags []string `json:"tags,omitempty"` + PipelineTag string `json:"pipeline_tag,omitempty"` + LibraryName string `json:"library_name,omitempty"` + ModelID string `json:"modelId,omitempty"` + Task string `json:"task,omitempty"` + Siblings []hfFile `json:"siblings,omitempty"` + Config *hfConfig `json:"config,omitempty"` + CardData *hfCard `json:"cardData,omitempty"` +} + +type hfFile struct { + RFileName string `json:"rfilename"` +} + +type hfConfig struct { + Architectures []string `json:"architectures,omitempty"` + ModelType string `json:"model_type,omitempty"` +} + +type hfCard struct { + Data map[string]interface{} `json:"data,omitempty"` +} + +//go:embed assets/catalog_logo.svg +var catalogLogoSVG []byte + +var ( + catalogModelLogo = "data:image/svg+xml;base64," + base64.StdEncoding.EncodeToString(catalogLogoSVG) ) -func (h *hfCatalogImpl) GetModel(ctx context.Context, modelName string, sourceID string) (*openapi.CatalogModel, error) { - // TODO: Implement HuggingFace model retrieval - return nil, fmt.Errorf("HuggingFace model retrieval not yet implemented") +// populateFromHFInfo populates the hfModel's CatalogModel fields from HuggingFace API data +func (hfm *hfModel) populateFromHFInfo(ctx context.Context, provider *hfModelProvider, hfInfo *hfModelInfo, sourceId string, originalModelName string) { + // Set model name + modelName := hfInfo.ID + if modelName == "" { + modelName = hfInfo.ModelID + } + if modelName == "" { + modelName = originalModelName + } + hfm.Name = modelName + + // Set ExternalId + if hfInfo.ID != "" { + hfm.ExternalId = &hfInfo.ID + } + + // Set SourceId + if sourceId != "" { + hfm.SourceId = &sourceId + } + + // Convert timestamps + if hfInfo.CreatedAt != "" { + if createTime, err := parseHFTime(hfInfo.CreatedAt); err == nil { + createTimeStr := strconv.FormatInt(createTime, 10) + hfm.CreateTimeSinceEpoch = &createTimeStr + } + } + if hfInfo.UpdatedAt != "" { + if updateTime, err := parseHFTime(hfInfo.UpdatedAt); err == nil { + updateTimeStr := strconv.FormatInt(updateTime, 10) + hfm.LastUpdateTimeSinceEpoch = &updateTimeStr + } + } + + // Extract license from tags + // Skip license tags in custom properties to avoid duplication + var filteredTags []string + if len(hfInfo.Tags) > 0 { + filteredTags = make([]string, 0, len(hfInfo.Tags)) + for _, tag := range hfInfo.Tags { + if strings.HasPrefix(tag, "license:") { + // Extract license (only first one) + if hfm.License == nil { + license := strings.TrimPrefix(tag, "license:") + if license != "" { + hfm.License = &license + } + } + } else { + filteredTags = append(filteredTags, tag) + } + } + } + + // Extract README from sibling files first (preferred source) + // Check for common README filenames + readmeFilenames := []string{"README.md", "readme.md", "Readme.md", "README", "readme"} + + for _, sibling := range hfInfo.Siblings { + for _, readmeFilename := range readmeFilenames { + if sibling.RFileName == readmeFilename { + if readmeContent, err := provider.fetchFileContent(ctx, modelName, readmeFilename); err == nil { + hfm.Readme = &readmeContent + break + } else { + glog.V(2).Infof("Failed to fetch README from sibling file %s for model %s: %v", readmeFilename, modelName, err) + } + } + } + if hfm.Readme != nil { + break + } + } + + // Extract description from cardData if available + if hfInfo.CardData != nil && hfInfo.CardData.Data != nil { + // Extract description from cardData if available + if desc, ok := hfInfo.CardData.Data["description"].(string); ok && desc != "" { + hfm.Description = &desc + } + + // Extract language from cardData if available + if langData, ok := hfInfo.CardData.Data["language"].([]interface{}); ok && len(langData) > 0 { + languages := make([]string, 0, len(langData)) + for _, lang := range langData { + if langStr, ok := lang.(string); ok && langStr != "" { + languages = append(languages, langStr) + } + } + if len(languages) > 0 { + hfm.Language = languages + } + } + + // Extract license link from cardData if available + // Check common field names for license link/URL + if hfm.LicenseLink == nil { + licenseLinkFields := []string{"license_link", "licenseLink", "license_url", "licenseUrl", "license"} + for _, field := range licenseLinkFields { + if link, ok := hfInfo.CardData.Data[field].(string); ok && link != "" { + if strings.HasPrefix(link, "http://") || strings.HasPrefix(link, "https://") { + hfm.LicenseLink = &link + break + } + } + } + } + + } + + // Set provider from author + if hfInfo.Author != "" { + hfm.Provider = &hfInfo.Author + } + + // Set library name + if hfInfo.LibraryName != "" { + hfm.LibraryName = &hfInfo.LibraryName + } + + // Set logo + hfm.Logo = &catalogModelLogo + + // Convert tasks + var tasks []string + if hfInfo.Task != "" { + tasks = append(tasks, hfInfo.Task) + } + if hfInfo.PipelineTag != "" && hfInfo.PipelineTag != hfInfo.Task { + tasks = append(tasks, hfInfo.PipelineTag) + } + if len(tasks) > 0 { + hfm.Tasks = tasks + } + + // Convert tags and other metadata to custom properties + customProps := make(map[string]apimodels.MetadataValue) + + customProps["hf_private"] = apimodels.MetadataValue{ + MetadataStringValue: &apimodels.MetadataStringValue{ + StringValue: strconv.FormatBool(hfInfo.Private), + }, + } + + customProps["hf_gated"] = apimodels.MetadataValue{ + MetadataStringValue: &apimodels.MetadataStringValue{ + StringValue: hfInfo.Gated.String(), + }, + } + + if len(filteredTags) > 0 { + if tagsJSON, err := json.Marshal(filteredTags); err == nil { + customProps["hf_tags"] = apimodels.MetadataValue{ + MetadataStringValue: &apimodels.MetadataStringValue{ + StringValue: string(tagsJSON), + }, + } + } + } + + if hfInfo.Config != nil { + if len(hfInfo.Config.Architectures) > 0 { + if archJSON, err := json.Marshal(hfInfo.Config.Architectures); err == nil { + customProps["hf_architectures"] = apimodels.MetadataValue{ + MetadataStringValue: &apimodels.MetadataStringValue{ + StringValue: string(archJSON), + }, + } + } + } + if hfInfo.Config.ModelType != "" { + customProps["hf_model_type"] = apimodels.MetadataValue{ + MetadataStringValue: &apimodels.MetadataStringValue{ + StringValue: hfInfo.Config.ModelType, + }, + } + } + } + + if len(customProps) > 0 { + hfm.SetCustomProperties(customProps) + } } -func (h *hfCatalogImpl) ListModels(ctx context.Context, params ListModelsParams) (model.CatalogModelList, error) { - // TODO: Implement HuggingFace model listing - // For now, return empty list to satisfy interface - return model.CatalogModelList{ - Items: []model.CatalogModel{}, - PageSize: 0, - Size: 0, - }, nil +func (p *hfModelProvider) Models(ctx context.Context) (<-chan ModelProviderRecord, error) { + // Read the catalog and report errors + catalog, err := p.getModelsFromHF(ctx) + if err != nil { + return nil, err + } + + ch := make(chan ModelProviderRecord) + go func() { + defer close(ch) + + // Send the initial list right away. + p.emit(ctx, catalog, ch) + }() + + return ch, nil } -func (h *hfCatalogImpl) GetArtifacts(ctx context.Context, modelName string, sourceID string, params ListArtifactsParams) (openapi.CatalogArtifactList, error) { - // TODO: Implement HuggingFace model artifacts retrieval - // For now, return empty list to satisfy interface - return openapi.CatalogArtifactList{ - Items: []openapi.CatalogArtifact{}, - PageSize: 0, - Size: 0, - }, nil +func (p *hfModelProvider) getModelsFromHF(ctx context.Context) ([]ModelProviderRecord, error) { + var records []ModelProviderRecord + + for _, modelName := range p.includedModels { + // Skip if excluded - check before fetching to avoid unnecessary API calls + if !p.filter.Allows(modelName) { + glog.V(2).Infof("Skipping excluded model: %s", modelName) + continue + } + + modelInfo, err := p.fetchModelInfo(ctx, modelName) + if err != nil { + glog.Errorf("Failed to fetch model info for %s: %v", modelName, err) + continue + } + + record := p.convertHFModelToRecord(ctx, modelInfo, modelName) + + // Additional safety check: verify the final model name is not excluded + // (in case the model name changed during conversion, e.g., from hfInfo.ID) + if record.Model.GetAttributes() != nil && record.Model.GetAttributes().Name != nil { + finalModelName := *record.Model.GetAttributes().Name + if !p.filter.Allows(finalModelName) { + glog.V(2).Infof("Skipping excluded model (after conversion): %s", finalModelName) + continue + } + } + + records = append(records, record) + } + + return records, nil } -func (h *hfCatalogImpl) GetFilterOptions(ctx context.Context) (*openapi.FilterOptionsList, error) { - // TODO: Implement HuggingFace filter options retrieval - // For now, return empty options to satisfy interface - emptyFilters := make(map[string]openapi.FilterOption) - return &openapi.FilterOptionsList{ - Filters: &emptyFilters, - }, nil +func (p *hfModelProvider) fetchModelInfo(ctx context.Context, modelName string) (*hfModelInfo, error) { + // The HF API requires the full model identifier: org/model-name (aka repo/model-name) + + // Normalize the model name (remove any leading/trailing slashes) + modelName = strings.Trim(modelName, "/") + + // Construct the API URL with the full model identifier + apiURL := fmt.Sprintf("%s/api/models/%s", p.baseURL, modelName) + + glog.V(2).Infof("Fetching HuggingFace model info from: %s", apiURL) + + req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + // Set User-Agent header (HuggingFace API expects this) + req.Header.Set("User-Agent", "model-registry-catalog") + + if p.apiKey != "" { + req.Header.Set("Authorization", "Bearer "+p.apiKey) + } + + resp, err := p.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch model info for %s: %w", modelName, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("HuggingFace API returned status %d for model %s: %s", resp.StatusCode, modelName, string(bodyBytes)) + } + + var modelInfo hfModelInfo + if err := json.NewDecoder(resp.Body).Decode(&modelInfo); err != nil { + return nil, fmt.Errorf("failed to decode model info for %s: %w", modelName, err) + } + + // Ensure ID is set from modelName if not present in API response + if modelInfo.ID == "" { + modelInfo.ID = modelName + } + + return &modelInfo, nil +} + +// fetchFileContent fetches the content of a file from HuggingFace repository +func (p *hfModelProvider) fetchFileContent(ctx context.Context, modelName string, filename string) (string, error) { + // Normalize the model name (remove any leading/trailing slashes) + modelName = strings.Trim(modelName, "/") + + // Construct the API URL for raw file content + // HuggingFace API endpoint: {baseURL}/{model_id}/raw/main/{filename} + apiURL := fmt.Sprintf("%s/%s/raw/main/%s", p.baseURL, modelName, filename) + + req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil) + if err != nil { + return "", fmt.Errorf("failed to create request: %w", err) + } + + // Set User-Agent header + req.Header.Set("User-Agent", "model-registry-catalog") + + if p.apiKey != "" { + req.Header.Set("Authorization", "Bearer "+p.apiKey) + } + + resp, err := p.client.Do(req) + if err != nil { + return "", fmt.Errorf("failed to fetch file %s for model %s: %w", filename, modelName, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("HuggingFace API returned status %d for file %s in model %s: %s", resp.StatusCode, filename, modelName, string(bodyBytes)) + } + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read file content for %s in model %s: %w", filename, modelName, err) + } + + return string(bodyBytes), nil +} + +func (p *hfModelProvider) convertHFModelToRecord(ctx context.Context, hfInfo *hfModelInfo, originalModelName string) ModelProviderRecord { + // Create hfModel and populate it from HF API data + hfm := &hfModel{} + hfm.populateFromHFInfo(ctx, p, hfInfo, p.sourceId, originalModelName) + + // Convert to database model + model := dbmodels.CatalogModelImpl{} + + // Convert model attributes + modelName := hfm.Name + attrs := &dbmodels.CatalogModelAttributes{ + Name: &modelName, + ExternalID: hfm.ExternalId, + } + + // Convert timestamps if available + if hfm.CreateTimeSinceEpoch != nil { + if createTime, err := strconv.ParseInt(*hfm.CreateTimeSinceEpoch, 10, 64); err == nil { + attrs.CreateTimeSinceEpoch = &createTime + } + } + if hfm.LastUpdateTimeSinceEpoch != nil { + if updateTime, err := strconv.ParseInt(*hfm.LastUpdateTimeSinceEpoch, 10, 64); err == nil { + attrs.LastUpdateTimeSinceEpoch = &updateTime + } + } + + model.Attributes = attrs + + // Convert model properties + properties, customProperties := convertHFModelProperties(&hfm.CatalogModel) + if len(properties) > 0 { + model.Properties = &properties + } + if len(customProperties) > 0 { + model.CustomProperties = &customProperties + } + + return ModelProviderRecord{ + Model: &model, + Artifacts: []dbmodels.CatalogArtifact{}, // HF models don't have artifacts from the API + } +} + +// convertHFModelProperties converts CatalogModel properties to database format +func convertHFModelProperties(catalogModel *apimodels.CatalogModel) ([]models.Properties, []models.Properties) { + var properties []models.Properties + var customProperties []models.Properties + + // Regular properties + if catalogModel.Description != nil { + properties = append(properties, models.NewStringProperty("description", *catalogModel.Description, false)) + } + if catalogModel.Readme != nil { + properties = append(properties, models.NewStringProperty("readme", *catalogModel.Readme, false)) + } + if catalogModel.Provider != nil { + properties = append(properties, models.NewStringProperty("provider", *catalogModel.Provider, false)) + } + if catalogModel.License != nil { + properties = append(properties, models.NewStringProperty("license", *catalogModel.License, false)) + } + if catalogModel.LicenseLink != nil { + properties = append(properties, models.NewStringProperty("license_link", *catalogModel.LicenseLink, false)) + } + if catalogModel.LibraryName != nil { + properties = append(properties, models.NewStringProperty("library_name", *catalogModel.LibraryName, false)) + } + if catalogModel.Logo != nil { + properties = append(properties, models.NewStringProperty("logo", *catalogModel.Logo, false)) + } + if catalogModel.SourceId != nil { + properties = append(properties, models.NewStringProperty("source_id", *catalogModel.SourceId, false)) + } + + // Convert array properties + if len(catalogModel.Tasks) > 0 { + if tasksJSON, err := json.Marshal(catalogModel.Tasks); err == nil { + properties = append(properties, models.NewStringProperty("tasks", string(tasksJSON), false)) + } + } + if len(catalogModel.Language) > 0 { + if languageJSON, err := json.Marshal(catalogModel.Language); err == nil { + properties = append(properties, models.NewStringProperty("language", string(languageJSON), false)) + } + } + + // Convert custom properties from the CatalogModel + if catalogModel.CustomProperties != nil { + for key, value := range catalogModel.GetCustomProperties() { + prop := convertMetadataValueToProperty(key, value) + customProperties = append(customProperties, prop) + } + } + + return properties, customProperties } -// validateCredentials checks if the HuggingFace API credentials are valid -func (h *hfCatalogImpl) validateCredentials(ctx context.Context) error { +// parseHFTime parses HuggingFace timestamp format (ISO 8601) +func parseHFTime(timeStr string) (int64, error) { + t, err := time.Parse(time.RFC3339, timeStr) + if err != nil { + return 0, err + } + return t.UnixMilli(), nil +} + +func (p *hfModelProvider) emit(ctx context.Context, models []ModelProviderRecord, out chan<- ModelProviderRecord) { + done := ctx.Done() + for _, model := range models { + // Check if model should be excluded by name + if model.Model.GetAttributes() != nil && model.Model.GetAttributes().Name != nil { + modelName := *model.Model.GetAttributes().Name + if !p.filter.Allows(modelName) { + glog.V(2).Infof("Skipping excluded model in emit: %s", modelName) + continue + } + } + + select { + case out <- model: + case <-done: + return + } + } +} + +// validateCredentials checks if the HuggingFace API key credentials are valid +func (p *hfModelProvider) validateCredentials(ctx context.Context) error { glog.Infof("Validating HuggingFace API credentials") // Make a simple API call to validate credentials - apiURL := h.baseURL + "/api/whoami-v2" + apiURL := p.baseURL + "/api/whoami-v2" req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil) if err != nil { return fmt.Errorf("failed to create validation request: %w", err) } - if h.apiKey != "" { - req.Header.Set("Authorization", "Bearer "+h.apiKey) + req.Header.Set("User-Agent", "model-registry-catalog") + + if p.apiKey != "" { + req.Header.Set("Authorization", "Bearer "+p.apiKey) } - resp, err := h.client.Do(req) + resp, err := p.client.Do(req) if err != nil { return fmt.Errorf("failed to validate HuggingFace credentials: %w", err) } @@ -83,46 +592,71 @@ func (h *hfCatalogImpl) validateCredentials(ctx context.Context) error { return fmt.Errorf("invalid HuggingFace API credentials") } if resp.StatusCode != http.StatusOK { - return fmt.Errorf("HuggingFace API validation failed with status: %d", resp.StatusCode) + bodyBytes, _ := io.ReadAll(resp.Body) + return fmt.Errorf("HuggingFace API validation failed with status: %d: %s", resp.StatusCode, string(bodyBytes)) } glog.Infof("HuggingFace credentials validated successfully") return nil } -// newHfCatalog creates a new HuggingFace catalog source -func newHfCatalog(source *Source, reldir string) (APIProvider, error) { - apiKey, ok := source.Properties["apiKey"].(string) - if !ok || apiKey == "" { - return nil, fmt.Errorf("missing or invalid 'apiKey' property for HuggingFace catalog") - } +func newHFModelProvider(ctx context.Context, source *Source, reldir string) (<-chan ModelProviderRecord, error) { + p := &hfModelProvider{} + p.client = &http.Client{Timeout: 30 * time.Second} - baseURL := defaultHuggingFaceURL - if url, ok := source.Properties["url"].(string); ok && url != "" { - baseURL = strings.TrimSuffix(url, "/") + // Parse Source ID + sourceId := source.GetId() + if sourceId == "" { + return nil, fmt.Errorf("missing source ID for HuggingFace catalog") } + p.sourceId = sourceId - // Optional model limit for future implementation - modelLimit := 100 - if limit, ok := source.Properties["modelLimit"].(int); ok && limit > 0 { - modelLimit = limit + // Parse API key from environment variable + // Allow the environment variable name to be configured via properties, defaulting to HF_API_KEY + apiKeyEnvVar := defaultAPIKeyEnvVar + if envVar, ok := source.Properties[apiKeyEnvVarKey].(string); ok && envVar != "" { + apiKeyEnvVar = envVar } + apiKey := os.Getenv(apiKeyEnvVar) + if apiKey == "" { + return nil, fmt.Errorf("missing %s environment variable for HuggingFace catalog", apiKeyEnvVar) + } + p.apiKey = apiKey - glog.Infof("Configuring HuggingFace catalog with URL: %s, modelLimit: %d", baseURL, modelLimit) - - h := &hfCatalogImpl{ - client: &http.Client{Timeout: 30 * time.Second}, - apiKey: apiKey, - baseURL: baseURL, + // Parse base URL (optional, defaults to huggingface.co) + // This allows tests to use mock servers by providing a custom URL + p.baseURL = defaultHuggingFaceURL + if url, ok := source.Properties[urlKey].(string); ok && url != "" { + p.baseURL = strings.TrimSuffix(url, "/") // Remove trailing slash if present } - // Validate credentials during initialization (as required by Jira ticket) - ctx := context.Background() - if err := h.validateCredentials(ctx); err != nil { + // Validate credentials before proceeding + if err := p.validateCredentials(ctx); err != nil { glog.Errorf("HuggingFace catalog credential validation failed: %v", err) return nil, fmt.Errorf("failed to validate HuggingFace catalog credentials: %w", err) } - glog.Infof("HuggingFace catalog source configured successfully") - return h, nil + // Use top-level IncludedModels from Source as the list of models to fetch + // These can be specific model names (required for HF API) or patterns + if len(source.IncludedModels) == 0 { + return nil, fmt.Errorf("includedModels cannot be empty for HuggingFace catalog") + } + + p.includedModels = source.IncludedModels + + // Create ModelFilter from source configuration (handles IncludedModels/ExcludedModels from Source) + // Note: IncludedModels are used both for fetching and filtering + filter, err := NewModelFilterFromSource(source, nil, nil) + if err != nil { + return nil, err + } + p.filter = filter + + return p.Models(ctx) +} + +func init() { + if err := RegisterModelProvider("hf", newHFModelProvider); err != nil { + panic(err) + } } diff --git a/catalog/internal/catalog/hf_catalog_test.go b/catalog/internal/catalog/hf_catalog_test.go index e21b384ed7..7cf032f22b 100644 --- a/catalog/internal/catalog/hf_catalog_test.go +++ b/catalog/internal/catalog/hf_catalog_test.go @@ -3,172 +3,556 @@ package catalog import ( "context" "net/http" - "net/http/httptest" - "strings" "testing" - "github.com/kubeflow/model-registry/catalog/pkg/openapi" + apimodels "github.com/kubeflow/model-registry/catalog/pkg/openapi" + "github.com/kubeflow/model-registry/internal/db/models" ) -func TestNewHfCatalog_MissingAPIKey(t *testing.T) { - source := &Source{ - CatalogSource: openapi.CatalogSource{ - Id: "test_hf", - Name: "Test HF", +func TestPopulateFromHFInfo(t *testing.T) { + tests := []struct { + name string + hfInfo *hfModelInfo + sourceId string + originalModelName string + expectedName string + expectedExternalID *string + expectedSourceID *string + expectedProvider *string + expectedLicense *string + expectedLibrary *string + hasReadme bool + hasDescription bool + hasTasks bool + hasCustomProps bool + }{ + { + name: "complete model info", + hfInfo: &hfModelInfo{ + ID: "test-org/test-model", + Author: "test-author", + Sha: "abc123", + CreatedAt: "2023-01-01T00:00:00Z", + UpdatedAt: "2023-01-02T00:00:00Z", + Downloads: 1000, + Tags: []string{"license:mit", "transformers", "pytorch"}, + PipelineTag: "text-generation", + Task: "text-generation", + LibraryName: "transformers", + Config: &hfConfig{ + Architectures: []string{"GPT2LMHeadModel"}, + ModelType: "gpt2", + }, + CardData: &hfCard{ + Data: map[string]interface{}{ + "description": "A test model description", + }, + }, + }, + sourceId: "test-source-id", + originalModelName: "test-org/test-model", + expectedName: "test-org/test-model", + expectedProvider: stringPtr("test-author"), + expectedLicense: stringPtr("mit"), + expectedLibrary: stringPtr("transformers"), + hasTasks: true, + hasCustomProps: true, + hasReadme: false, // No README fetching in unit tests }, - Type: "hf", - Properties: map[string]any{ - "url": "https://huggingface.co", + { + name: "model with ModelID fallback", + hfInfo: &hfModelInfo{ + ModelID: "fallback-model-id", + Author: "another-author", + }, + sourceId: "source-2", + originalModelName: "original-name", + expectedName: "fallback-model-id", + expectedProvider: stringPtr("another-author"), + }, + { + name: "model with original name fallback", + hfInfo: &hfModelInfo{ + Author: "author-3", + }, + sourceId: "source-3", + originalModelName: "fallback-original-name", + expectedName: "fallback-original-name", + expectedProvider: stringPtr("author-3"), + }, + { + name: "model with license in tags", + hfInfo: &hfModelInfo{ + ID: "test/licensed-model", + Tags: []string{"license:apache-2.0", "other-tag"}, + }, + sourceId: "source-4", + originalModelName: "test/licensed-model", + expectedName: "test/licensed-model", + expectedLicense: stringPtr("apache-2.0"), + hasCustomProps: true, + }, + { + name: "model with tasks", + hfInfo: &hfModelInfo{ + ID: "test/task-model", + Task: "text-classification", + PipelineTag: "sentiment-analysis", + }, + sourceId: "source-5", + originalModelName: "test/task-model", + expectedName: "test/task-model", + hasTasks: true, + }, + { + name: "model with description in cardData", + hfInfo: &hfModelInfo{ + ID: "test/desc-model", + CardData: &hfCard{ + Data: map[string]interface{}{ + "description": "This is a test description", + }, + }, + }, + sourceId: "source-6", + originalModelName: "test/desc-model", + expectedName: "test/desc-model", + hasDescription: true, + }, + { + name: "minimal model info", + hfInfo: &hfModelInfo{ + ID: "minimal/model", + }, + sourceId: "source-7", + originalModelName: "minimal/model", + expectedName: "minimal/model", }, } - _, err := newHfCatalog(source, "") - if err == nil { - t.Fatal("Expected error for missing API key, got nil") - } - if err.Error() != "missing or invalid 'apiKey' property for HuggingFace catalog" { - t.Fatalf("Expected specific error message, got: %s", err.Error()) - } -} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a mock provider with HTTP client to avoid nil pointer + // Note: README fetching will fail, but that's expected in unit tests + provider := &hfModelProvider{ + sourceId: tt.sourceId, + client: &http.Client{}, + } -func TestNewHfCatalog_WithValidCredentials(t *testing.T) { - // Create mock server that returns valid response for credential validation - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Check for authorization header - auth := r.Header.Get("Authorization") - if auth != "Bearer test-api-key" { - w.WriteHeader(http.StatusUnauthorized) - return - } - - switch r.URL.Path { - case "/api/whoami-v2": - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"name": "test-user", "type": "user"}`)) - default: - w.WriteHeader(http.StatusNotFound) - } - })) - defer server.Close() - - source := &Source{ - CatalogSource: openapi.CatalogSource{ - Id: "test_hf", - Name: "Test HF", - }, - Type: "hf", - Properties: map[string]any{ - "apiKey": "test-api-key", - "url": server.URL, - "modelLimit": 10, - }, - } + // Create hfModel and populate it + hfm := &hfModel{} + ctx := context.Background() + hfm.populateFromHFInfo(ctx, provider, tt.hfInfo, tt.sourceId, tt.originalModelName) - catalog, err := newHfCatalog(source, "") - if err != nil { - t.Fatalf("Failed to create HF catalog: %v", err) - } + // Verify name + if hfm.Name != tt.expectedName { + t.Errorf("Name = %v, want %v", hfm.Name, tt.expectedName) + } - hfCatalog := catalog.(*hfCatalogImpl) + // Verify ExternalID + if tt.expectedExternalID != nil { + if hfm.ExternalId == nil || *hfm.ExternalId != *tt.expectedExternalID { + t.Errorf("ExternalId = %v, want %v", hfm.ExternalId, tt.expectedExternalID) + } + } else if tt.hfInfo.ID != "" { + // If hfInfo has ID, ExternalId should be set + if hfm.ExternalId == nil || *hfm.ExternalId != tt.hfInfo.ID { + t.Errorf("ExternalId = %v, want %v", hfm.ExternalId, tt.hfInfo.ID) + } + } - // Test that methods return appropriate responses for stub implementation - ctx := context.Background() + // Verify SourceID + if tt.expectedSourceID != nil { + if hfm.SourceId == nil || *hfm.SourceId != *tt.expectedSourceID { + t.Errorf("SourceId = %v, want %v", hfm.SourceId, tt.expectedSourceID) + } + } else if tt.sourceId != "" { + if hfm.SourceId == nil || *hfm.SourceId != tt.sourceId { + t.Errorf("SourceId = %v, want %v", hfm.SourceId, tt.sourceId) + } + } - // Test GetModel - should return not implemented error - model, err := hfCatalog.GetModel(ctx, "test-model", "") - if err == nil { - t.Fatal("Expected not implemented error, got nil") - } - if model != nil { - t.Fatal("Expected nil model, got non-nil") - } + // Verify Provider + if tt.expectedProvider != nil { + if hfm.Provider == nil || *hfm.Provider != *tt.expectedProvider { + t.Errorf("Provider = %v, want %v", hfm.Provider, tt.expectedProvider) + } + } - // Test ListModels - should return empty list - listParams := ListModelsParams{ - Query: "", - OrderBy: openapi.ORDERBYFIELD_NAME, - SortOrder: openapi.SORTORDER_ASC, - } - modelList, err := hfCatalog.ListModels(ctx, listParams) - if err != nil { - t.Fatalf("Failed to list models: %v", err) + // Verify License + if tt.expectedLicense != nil { + if hfm.License == nil || *hfm.License != *tt.expectedLicense { + t.Errorf("License = %v, want %v", hfm.License, tt.expectedLicense) + } + } + + // Verify LibraryName + if tt.expectedLibrary != nil { + if hfm.LibraryName == nil || *hfm.LibraryName != *tt.expectedLibrary { + t.Errorf("LibraryName = %v, want %v", hfm.LibraryName, tt.expectedLibrary) + } + } + + // Verify Tasks + if tt.hasTasks { + if len(hfm.Tasks) == 0 { + t.Error("Expected tasks to be set, but got empty slice") + } + } + + // Verify Description + if tt.hasDescription { + if hfm.Description == nil { + t.Error("Expected description to be set, but got nil") + } + } + + // Verify CustomProperties + if tt.hasCustomProps { + if hfm.GetCustomProperties() == nil || len(hfm.GetCustomProperties()) == 0 { + t.Error("Expected custom properties to be set, but got nil or empty") + } + } + + // Verify timestamps if present + if tt.hfInfo.CreatedAt != "" { + if hfm.CreateTimeSinceEpoch == nil { + t.Error("Expected CreateTimeSinceEpoch to be set") + } + } + if tt.hfInfo.UpdatedAt != "" { + if hfm.LastUpdateTimeSinceEpoch == nil { + t.Error("Expected LastUpdateTimeSinceEpoch to be set") + } + } + }) } - if len(modelList.Items) != 0 { - t.Fatalf("Expected 0 models, got %d", len(modelList.Items)) +} + +func TestConvertHFModelToRecord(t *testing.T) { + tests := []struct { + name string + hfInfo *hfModelInfo + originalModelName string + sourceId string + verifyFunc func(t *testing.T, record ModelProviderRecord) + }{ + { + name: "complete model conversion", + hfInfo: &hfModelInfo{ + ID: "test-org/complete-model", + Author: "test-author", + CreatedAt: "2023-01-01T00:00:00Z", + UpdatedAt: "2023-01-02T00:00:00Z", + Tags: []string{"license:mit"}, + LibraryName: "transformers", + Task: "text-generation", + CardData: &hfCard{ + Data: map[string]interface{}{ + "description": "A complete test model", + }, + }, + }, + originalModelName: "test-org/complete-model", + sourceId: "test-source", + verifyFunc: func(t *testing.T, record ModelProviderRecord) { + if record.Model == nil { + t.Fatal("Model should not be nil") + } + attrs := record.Model.GetAttributes() + if attrs == nil { + t.Fatal("Attributes should not be nil") + } + if attrs.Name == nil || *attrs.Name != "test-org/complete-model" { + t.Errorf("Name = %v, want 'test-org/complete-model'", attrs.Name) + } + if attrs.ExternalID == nil || *attrs.ExternalID != "test-org/complete-model" { + t.Errorf("ExternalID = %v, want 'test-org/complete-model'", attrs.ExternalID) + } + if attrs.CreateTimeSinceEpoch == nil { + t.Error("CreateTimeSinceEpoch should be set") + } + if attrs.LastUpdateTimeSinceEpoch == nil { + t.Error("LastUpdateTimeSinceEpoch should be set") + } + if record.Model.GetProperties() == nil || len(*record.Model.GetProperties()) == 0 { + t.Error("Properties should be set") + } + if len(record.Artifacts) != 0 { + t.Errorf("Artifacts should be empty, got %d", len(record.Artifacts)) + } + }, + }, + { + name: "minimal model conversion", + hfInfo: &hfModelInfo{ + ID: "minimal/model", + }, + originalModelName: "minimal/model", + sourceId: "source-1", + verifyFunc: func(t *testing.T, record ModelProviderRecord) { + if record.Model == nil { + t.Fatal("Model should not be nil") + } + attrs := record.Model.GetAttributes() + if attrs == nil || attrs.Name == nil { + t.Fatal("Attributes and Name should not be nil") + } + if *attrs.Name != "minimal/model" { + t.Errorf("Name = %v, want 'minimal/model'", attrs.Name) + } + }, + }, } - // Test GetArtifacts - should return empty list - artifacts, err := hfCatalog.GetArtifacts(ctx, "test-model", "", ListArtifactsParams{}) - if err != nil { - t.Fatalf("Failed to get artifacts: %v", err) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + provider := &hfModelProvider{ + sourceId: tt.sourceId, + } + ctx := context.Background() + record := provider.convertHFModelToRecord(ctx, tt.hfInfo, tt.originalModelName) + tt.verifyFunc(t, record) + }) } - if artifacts.Items == nil { - t.Fatal("Expected artifacts list, got nil") +} + +func TestConvertHFModelProperties(t *testing.T) { + tests := []struct { + name string + catalogModel *apimodels.CatalogModel + wantProps bool + wantCustom bool + verifyFunc func(t *testing.T, props []models.Properties, customProps []models.Properties) + }{ + { + name: "model with all properties", + catalogModel: &apimodels.CatalogModel{ + Name: "test-model", + Description: stringPtr("Test description"), + Readme: stringPtr("# Test README"), + Provider: stringPtr("test-provider"), + License: stringPtr("mit"), + LibraryName: stringPtr("transformers"), + SourceId: stringPtr("source-1"), + Tasks: []string{"text-generation"}, + }, + wantProps: true, + wantCustom: false, + verifyFunc: func(t *testing.T, props []models.Properties, customProps []models.Properties) { + if len(props) == 0 { + t.Error("Expected properties to be set") + } + }, + }, + { + name: "model with custom properties", + catalogModel: func() *apimodels.CatalogModel { + model := &apimodels.CatalogModel{ + Name: "test-model", + } + customProps := map[string]apimodels.MetadataValue{ + "hf_tags": { + MetadataStringValue: &apimodels.MetadataStringValue{ + StringValue: `["tag1","tag2"]`, + }, + }, + } + model.SetCustomProperties(customProps) + return model + }(), + wantProps: false, + wantCustom: true, + verifyFunc: func(t *testing.T, props []models.Properties, customProps []models.Properties) { + if len(customProps) == 0 { + t.Error("Expected custom properties to be set") + } + }, + }, + { + name: "model with minimal properties", + catalogModel: &apimodels.CatalogModel{ + Name: "minimal-model", + }, + wantProps: false, + wantCustom: false, + verifyFunc: func(t *testing.T, props []models.Properties, customProps []models.Properties) { + if len(props) != 0 { + t.Errorf("Expected no properties, got %d", len(props)) + } + if len(customProps) != 0 { + t.Errorf("Expected no custom properties, got %d", len(customProps)) + } + }, + }, + { + name: "model with tasks", + catalogModel: &apimodels.CatalogModel{ + Name: "task-model", + Tasks: []string{"classification", "generation"}, + }, + wantProps: true, + verifyFunc: func(t *testing.T, props []models.Properties, customProps []models.Properties) { + // Should have tasks property + foundTasks := false + for _, prop := range props { + if prop.Name == "tasks" { + foundTasks = true + break + } + } + if !foundTasks { + t.Error("Expected tasks property to be present") + } + }, + }, } - if len(artifacts.Items) != 0 { - t.Fatalf("Expected 0 artifacts, got %d", len(artifacts.Items)) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + props, customProps := convertHFModelProperties(tt.catalogModel) + if (len(props) > 0) != tt.wantProps { + t.Errorf("Properties presence = %v, want %v", len(props) > 0, tt.wantProps) + } + if (len(customProps) > 0) != tt.wantCustom { + t.Errorf("Custom properties presence = %v, want %v", len(customProps) > 0, tt.wantCustom) + } + if tt.verifyFunc != nil { + tt.verifyFunc(t, props, customProps) + } + }) } } -func TestNewHfCatalog_InvalidCredentials(t *testing.T) { - // Create mock server that returns 401 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusUnauthorized) - })) - defer server.Close() - - source := &Source{ - CatalogSource: openapi.CatalogSource{ - Id: "test_hf", - Name: "Test HF", +func TestHFModelProviderWithModelFilter(t *testing.T) { + tests := []struct { + name string + includedModels []string + excludedModels []string + modelName string + wantAllowed bool + description string + }{ + { + name: "model matches included pattern", + includedModels: []string{"ibm-granite/*"}, + excludedModels: nil, + modelName: "ibm-granite/granite-4.0-h-small", + wantAllowed: true, + description: "Model matching included pattern should be allowed", }, - Type: "hf", - Properties: map[string]any{ - "apiKey": "invalid-key", - "url": server.URL, + { + name: "model does not match included pattern", + includedModels: []string{"ibm-granite/*"}, + excludedModels: nil, + modelName: "meta-llama/Llama-3.2-1B", + wantAllowed: false, + description: "Model not matching included pattern should be excluded", + }, + { + name: "model matches excluded pattern", + includedModels: []string{"ibm-granite/*"}, + excludedModels: []string{"*-beta"}, + modelName: "ibm-granite/granite-4.0-h-beta", + wantAllowed: false, + description: "Model matching excluded pattern should be excluded even if it matches included", + }, + { + name: "model matches included but not excluded", + includedModels: []string{"ibm-granite/*"}, + excludedModels: []string{"*-beta"}, + modelName: "ibm-granite/granite-4.0-h-small", + wantAllowed: true, + description: "Model matching included but not excluded should be allowed", + }, + { + name: "case insensitive matching", + includedModels: []string{"IBM-Granite/*"}, + excludedModels: nil, + modelName: "ibm-granite/granite-4.0-h-small", + wantAllowed: true, + description: "Filtering should be case-insensitive", + }, + { + name: "no included patterns allows all", + includedModels: nil, + excludedModels: []string{"*-beta"}, + modelName: "test/model", + wantAllowed: true, + description: "No included patterns means all models are allowed (unless excluded)", }, } - _, err := newHfCatalog(source, "") - if err == nil { - t.Fatal("Expected error for invalid credentials, got nil") - } - if !strings.Contains(err.Error(), "invalid HuggingFace API credentials") { - t.Fatalf("Expected credential validation error, got: %s", err.Error()) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a filter with the test patterns + filter, err := NewModelFilter(tt.includedModels, tt.excludedModels) + if err != nil { + t.Fatalf("NewModelFilter() error = %v, want nil", err) + } + + // Create a provider with the filter + provider := &hfModelProvider{ + filter: filter, + } + + // Test that the filter works correctly + got := provider.filter.Allows(tt.modelName) + if got != tt.wantAllowed { + t.Errorf("ModelFilter.Allows(%q) = %v, want %v. %s", tt.modelName, got, tt.wantAllowed, tt.description) + } + }) } } -func TestNewHfCatalog_DefaultConfiguration(t *testing.T) { - // Create mock server for default HuggingFace URL - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"name": "test-user"}`)) - })) - defer server.Close() - - source := &Source{ - CatalogSource: openapi.CatalogSource{ - Id: "test_hf", - Name: "Test HF", - }, - Type: "hf", - Properties: map[string]any{ - "apiKey": "test-key", - "url": server.URL, // Override default for testing +func TestPopulateFromHFInfoWithCustomProperties(t *testing.T) { + hfInfo := &hfModelInfo{ + ID: "test/custom-props-model", + Sha: "sha123", + Downloads: 5000, + Tags: []string{"tag1", "tag2", "license:apache-2.0"}, + Config: &hfConfig{ + Architectures: []string{"BertModel", "BertForSequenceClassification"}, + ModelType: "bert", }, } - catalog, err := newHfCatalog(source, "") - if err != nil { - t.Fatalf("Failed to create HF catalog with defaults: %v", err) + provider := &hfModelProvider{ + sourceId: "test-source", + } + + hfm := &hfModel{} + ctx := context.Background() + hfm.populateFromHFInfo(ctx, provider, hfInfo, "test-source", "test/custom-props-model") + + customProps := hfm.GetCustomProperties() + if customProps == nil { + t.Fatal("Custom properties should not be nil") + } + + // Verify hf_tags + if tagsVal, ok := customProps["hf_tags"]; !ok { + t.Error("Expected hf_tags in custom properties") + } else if tagsVal.MetadataStringValue == nil { + t.Error("hf_tags should be a string value") } - hfCatalog := catalog.(*hfCatalogImpl) - if hfCatalog.apiKey != "test-key" { - t.Fatalf("Expected apiKey 'test-key', got '%s'", hfCatalog.apiKey) + // Verify hf_architectures + if archVal, ok := customProps["hf_architectures"]; !ok { + t.Error("Expected hf_architectures in custom properties") + } else if archVal.MetadataStringValue == nil { + t.Error("hf_architectures should be a string value") } - if hfCatalog.baseURL != server.URL { - t.Fatalf("Expected baseURL '%s', got '%s'", server.URL, hfCatalog.baseURL) + + // Verify hf_model_type + if modelTypeVal, ok := customProps["hf_model_type"]; !ok { + t.Error("Expected hf_model_type in custom properties") + } else if modelTypeVal.MetadataStringValue == nil || modelTypeVal.MetadataStringValue.StringValue != "bert" { + t.Errorf("hf_model_type = %v, want 'bert'", modelTypeVal.MetadataStringValue) } } + +// Helper function to create string pointers +func stringPtr(s string) *string { + return &s +} diff --git a/catalog/internal/catalog/loader.go b/catalog/internal/catalog/loader.go index 3cb5b56634..5189b224df 100644 --- a/catalog/internal/catalog/loader.go +++ b/catalog/internal/catalog/loader.go @@ -12,7 +12,6 @@ import ( dbmodels "github.com/kubeflow/model-registry/catalog/internal/db/models" "github.com/kubeflow/model-registry/catalog/internal/db/service" apimodels "github.com/kubeflow/model-registry/catalog/pkg/openapi" - "github.com/kubeflow/model-registry/internal/apiutils" mrmodels "github.com/kubeflow/model-registry/internal/db/models" "k8s.io/apimachinery/pkg/util/yaml" ) @@ -57,6 +56,11 @@ type Source struct { // Properties used for configuring the catalog connection based on catalog implementation Properties map[string]any `json:"properties,omitempty"` + + // Origin is the absolute path of the config file this source was loaded from. + // This is set automatically during loading and used for resolving relative paths. + // It is not read from YAML; it's set programmatically. + Origin string `json:"-" yaml:"-"` } type Loader struct { @@ -66,20 +70,33 @@ type Loader struct { // Labels contains current labels loaded from the configuration files. Labels *LabelCollection - paths []string - services service.Services - closersMu sync.Mutex - closers map[string]func() - handlers []LoaderEventHandler + paths []string + services service.Services + closersMu sync.Mutex + closer func() // cancels the current model loading goroutines + handlers []LoaderEventHandler + loadedSources map[string]bool // tracks which source IDs have been loaded } func NewLoader(services service.Services, paths []string) *Loader { + // Convert paths to absolute for consistent origin ordering. + // This matches how loadOne converts paths before calling Merge. + absPaths := make([]string, 0, len(paths)) + for _, p := range paths { + absPath, err := filepath.Abs(p) + if err != nil { + // Fall back to original path if conversion fails + absPath = p + } + absPaths = append(absPaths, absPath) + } + return &Loader{ - Sources: NewSourceCollection(), - Labels: NewLabelCollection(), - paths: paths, - services: services, - closers: map[string]func(){}, + Sources: NewSourceCollection(absPaths...), + Labels: NewLabelCollection(), + paths: paths, + services: services, + loadedSources: map[string]bool{}, } } @@ -94,26 +111,33 @@ func (l *Loader) RegisterEventHandler(fn LoaderEventHandler) { // Start processes the sources YAML files. Background goroutines will be // stopped when the context is canceled. func (l *Loader) Start(ctx context.Context) error { + // Phase 1: Parse all config files and merge sources/labels + // This must happen BEFORE loading models so that sparse overrides work correctly for _, path := range l.paths { - err := l.loadOne(ctx, path) + err := l.parseAndMerge(path) if err != nil { return fmt.Errorf("%s: %w", path, err) } + } + // Phase 2: Load models from merged sources (once, after all merging is complete) + err := l.loadAllModels(ctx) + if err != nil { + return err + } + + // Phase 3: Set up file watchers for hot-reload + for _, path := range l.paths { go func(path string) { changes, err := getMonitor().Path(ctx, path) if err != nil { glog.Errorf("unable to watch sources file (%s): %v", path, err) - // Not fatal, we just won't get automatic updates. + return } for range changes { glog.Infof("Reloading sources %s", path) - - err = l.loadOne(ctx, path) - if err != nil { - glog.Errorf("unable to load sources: %v", err) - } + l.reloadAll(ctx) } }(path) } @@ -121,9 +145,8 @@ func (l *Loader) Start(ctx context.Context) error { return nil } -// loadOne processes (or re-processes) a sources config file. -func (l *Loader) loadOne(ctx context.Context, path string) error { - // Get absolute path of the catalog config file +// parseAndMerge parses a config file and merges its sources/labels into the collections. +func (l *Loader) parseAndMerge(path string) error { path, err := filepath.Abs(path) if err != nil { return fmt.Errorf("failed to get absolute path for %s: %v", path, err) @@ -134,17 +157,35 @@ func (l *Loader) loadOne(ctx context.Context, path string) error { return err } - err = l.updateSources(path, config) - if err != nil { + if err = l.updateSources(path, config); err != nil { return err } - err = l.updateLabels(path, config) - if err != nil { - return err + return l.updateLabels(path, config) +} + +// loadAllModels loads models from all merged sources. +func (l *Loader) loadAllModels(ctx context.Context) error { + // Clear the loaded sources tracker for a fresh load + l.loadedSources = map[string]bool{} + + return l.updateDatabase(ctx) +} + +// reloadAll re-parses all config files and reloads all models. +// Called when any config file changes. +func (l *Loader) reloadAll(ctx context.Context) { + // Re-parse all config files + for _, path := range l.paths { + if err := l.parseAndMerge(path); err != nil { + glog.Errorf("unable to reload sources from %s: %v", path, err) + } } - return l.updateDatabase(ctx, path, config) + // Reload all models + if err := l.loadAllModels(ctx); err != nil { + glog.Errorf("unable to reload models: %v", err) + } } func (l *Loader) read(path string) (*sourceConfig, error) { @@ -158,31 +199,16 @@ func (l *Loader) read(path string) (*sourceConfig, error) { return nil, err } - enabledSources := make([]Source, 0, len(config.Catalogs)) - - // Remove disabled sources and explicitly set enabled on the others. - for _, source := range config.Catalogs { - // If enabled is explicitly set to false, skip - if source.HasEnabled() && *source.Enabled == false { - continue - } - // If not explicitly set, default to enabled - source.CatalogSource.Enabled = apiutils.Of(true) - - // Default to an empty labels list - if source.Labels == nil { - source.Labels = []string{} - } - - enabledSources = append(enabledSources, source) - } - config.Catalogs = enabledSources + // Note: We intentionally do NOT filter disabled sources or apply defaults here. + // This allows field-level merging in SourceCollection to work correctly: + // - A base source with enabled=false can be enabled by a user override with just id + enabled=true + // - Defaults are applied after merging in SourceCollection.merged() return config, nil } func (l *Loader) updateSources(path string, config *sourceConfig) error { - sources := make(map[string]apimodels.CatalogSource, len(config.Catalogs)) + sources := make(map[string]Source, len(config.Catalogs)) for _, source := range config.Catalogs { glog.Infof("reading config type %s...", source.Type) @@ -194,13 +220,15 @@ func (l *Loader) updateSources(path string, config *sourceConfig) error { return fmt.Errorf("invalid source: duplicate id %s", id) } - // Validate includedModels/excludedModels patterns early + // Validate includedModels/excludedModels patterns early (only if set) if err := ValidateSourceFilters(source.IncludedModels, source.ExcludedModels); err != nil { return fmt.Errorf("invalid source %s: %w", id, err) } - sources[id] = source.CatalogSource - + // Set the origin path so relative paths in properties can be resolved + // relative to this config file's directory + source.Origin = path + sources[id] = source glog.Infof("loaded source %s of type %s", id, source.Type) } @@ -224,17 +252,20 @@ func (l *Loader) updateLabels(path string, config *sourceConfig) error { return l.Labels.Merge(path, config.Labels) } -func (l *Loader) updateDatabase(ctx context.Context, path string, config *sourceConfig) error { +func (l *Loader) updateDatabase(ctx context.Context) error { ctx, cancel := context.WithCancel(ctx) l.closersMu.Lock() - if l.closers[path] != nil { - l.closers[path]() + if l.closer != nil { + l.closer() } - l.closers[path] = cancel + l.closer = cancel l.closersMu.Unlock() - records := l.readProviderRecords(ctx, path, config) + // Use merged sources from SourceCollection instead of per-file config. + // This enables sparse overrides to work: a user can enable a disabled source + // with just "id" and "enabled: true", inheriting Type and Properties from the base. + records := l.readProviderRecords(ctx) go func() { for record := range records { @@ -291,16 +322,33 @@ func (l *Loader) updateDatabase(ctx context.Context, path string, config *source return nil } -// readProviderRecords calls the provider for every configured source and -// merges the returned channels together. The returned channel is closed when -// the last provider channel is closed. -func (l *Loader) readProviderRecords(ctx context.Context, path string, config *sourceConfig) <-chan ModelProviderRecord { - configDir := filepath.Dir(path) +// readProviderRecords calls the provider for every merged source that hasn't +// been loaded yet, and merges the returned channels together. The returned +// channel is closed when the last provider channel is closed. +func (l *Loader) readProviderRecords(ctx context.Context) <-chan ModelProviderRecord { ch := make(chan ModelProviderRecord) var wg sync.WaitGroup - for _, source := range config.Catalogs { + // Get all enabled sources from the merged collection. + // This allows sparse overrides to work: a user can enable a disabled source + // with just "id" and "enabled: true", inheriting Type and Properties from the base. + mergedSources := l.Sources.AllSources() + + for _, source := range mergedSources { + // Skip sources that have already been loaded + if l.loadedSources[source.Id] { + continue + } + + if source.Type == "" { + glog.Errorf("source %s has no type defined, skipping", source.Id) + continue + } + + // Mark this source as loaded + l.loadedSources[source.Id] = true + glog.Infof("Reading models from %s source %s", source.Type, source.Id) registerFunc, ok := registeredModelProviders[source.Type] @@ -309,7 +357,12 @@ func (l *Loader) readProviderRecords(ctx context.Context, path string, config *s continue } - records, err := registerFunc(ctx, &source, configDir) + // Use the source's origin directory for resolving relative paths. + // This allows sources from different config files (e.g., mounted from + // different configmaps) to use relative paths correctly. + sourceDir := filepath.Dir(source.Origin) + + records, err := registerFunc(ctx, &source, sourceDir) if err != nil { glog.Errorf("error reading catalog type %s with id %s: %v", source.Type, source.Id, err) continue diff --git a/catalog/internal/catalog/sources.go b/catalog/internal/catalog/sources.go index 5c92432888..d7c86d6f5e 100644 --- a/catalog/internal/catalog/sources.go +++ b/catalog/internal/catalog/sources.go @@ -1,74 +1,208 @@ package catalog import ( - "fmt" "maps" "slices" "strings" "sync" model "github.com/kubeflow/model-registry/catalog/pkg/openapi" + "github.com/kubeflow/model-registry/internal/apiutils" ) -type SourceCollection struct { - mu sync.RWMutex - - // origins keeps track of where a source came from by some name (intended to be a file path). - origins map[string][]string +// originEntry holds sources from a single origin (config file). +type originEntry struct { + origin string + sources map[string]Source +} - sources map[string]model.CatalogSource +// SourceCollection manages catalog sources from multiple origins with priority-based merging. +// Later entries in the slice take precedence over earlier ones. +type SourceCollection struct { + mu sync.RWMutex + entries []originEntry } -func NewSourceCollection() *SourceCollection { - return &SourceCollection{ - origins: map[string][]string{}, - sources: map[string]model.CatalogSource{}, +// NewSourceCollection creates a new SourceCollection with the given origin order. +// Origins listed later in the order take precedence over earlier ones. +// For example, if originOrder is ["default.yaml", "user.yaml"], sources from +// "user.yaml" will override sources with the same ID from "default.yaml". +func NewSourceCollection(originOrder ...string) *SourceCollection { + entries := make([]originEntry, len(originOrder)) + for i, origin := range originOrder { + entries[i] = originEntry{origin: origin, sources: nil} } + return &SourceCollection{entries: entries} } // Merge adds sources from one origin (ordinarily, a file path--but any unique // string will do), completely replacing anything that was previously from that // origin. -func (sc *SourceCollection) Merge(origin string, sources map[string]model.CatalogSource) error { +// +// If a source with the same ID exists in multiple origins, fields from +// higher-priority origins (listed later in entries) override fields from +// lower-priority origins. Fields that are not set (zero value for strings, +// nil for pointers/slices/maps) in the override are inherited from the base. +func (sc *SourceCollection) Merge(origin string, sources map[string]Source) error { sc.mu.Lock() defer sc.mu.Unlock() - // Remove everything that was set before for this origin so that - // unreferenced sources are deleted. - for _, id := range sc.origins[origin] { - delete(sc.sources, id) + // Find existing entry for this origin + for i := range sc.entries { + if sc.entries[i].origin == origin { + sc.entries[i].sources = sources + return nil + } + } + + // Origin not found, append it (dynamic registration) + sc.entries = append(sc.entries, originEntry{origin: origin, sources: sources}) + return nil +} + +// mergeSources performs field-level merging of two Source structs. +// Fields from 'override' take precedence over 'base' when they are explicitly set. +// A field is considered "set" if: +// - For strings: non-empty +// - For pointers: non-nil +// - For slices: non-nil (empty slice is considered explicitly set to "no items") +// - For maps: non-nil (empty map is considered explicitly set) +func mergeSources(base, override Source) Source { + result := base + + // Id is always taken from override (it's the key) + result.Id = override.Id + + // Name: override if non-empty + if override.Name != "" { + result.Name = override.Name + } + + // Enabled: override if non-nil + if override.Enabled != nil { + result.Enabled = override.Enabled + } + + // Labels: override if non-nil (empty slice means "explicitly no labels") + if override.Labels != nil { + result.Labels = override.Labels + } + + // IncludedModels: override if non-nil + if override.IncludedModels != nil { + result.IncludedModels = override.IncludedModels + } + + // ExcludedModels: override if non-nil + if override.ExcludedModels != nil { + result.ExcludedModels = override.ExcludedModels + } + + // Type: override if non-empty + if override.Type != "" { + result.Type = override.Type + } + + // Properties: override if non-nil (complete replacement, not deep merge) + if override.Properties != nil { + result.Properties = override.Properties + } + + // Origin: use override's origin if Properties are overridden (since relative + // paths in Properties should resolve relative to where they were defined). + // Otherwise, keep base origin (where Type and original Properties came from). + if override.Properties != nil && override.Origin != "" { + result.Origin = override.Origin + } + + return result +} + +// applyDefaults applies default values to an Source for fields that are not set. +func applyDefaults(source Source) Source { + // Default Enabled to true if not set + if source.Enabled == nil { + source.Enabled = apiutils.Of(true) + } + + // Default Labels to empty slice if not set + if source.Labels == nil { + source.Labels = []string{} } - sc.origins[origin] = slices.Collect(maps.Keys(sources)) - for sourceID, source := range sources { - // Everything was deleted above, so if there's a source that - // already exists it must have come from another origin (file). - if _, exists := sc.sources[sourceID]; exists { - return fmt.Errorf("source %s exists from multiple origins", sourceID) + return source +} + +// merged computes the merged view of all sources with field-level merging. +// Must be called with lock held. +func (sc *SourceCollection) merged() map[string]Source { + result := map[string]Source{} + + for _, entry := range sc.entries { + for id, source := range entry.sources { + if existing, ok := result[id]; ok { + // Field-level merge: existing is base, source is override + result[id] = mergeSources(existing, source) + } else { + result[id] = source + } } + } - sc.sources[sourceID] = source + // Apply defaults to all merged sources + for id, source := range result { + result[id] = applyDefaults(source) } - return nil + return result } -func (sc *SourceCollection) All() map[string]model.CatalogSource { +// AllSources returns all merged sources including Type and Properties. +// This is used by the loader to get complete source information. +// Only enabled sources are returned. +func (sc *SourceCollection) AllSources() map[string]Source { sc.mu.RLock() defer sc.mu.RUnlock() - return sc.sources + result := map[string]Source{} + for id, source := range sc.merged() { + if source.Enabled != nil && *source.Enabled { + result[id] = source + } + } + return result +} + +// All returns all enabled sources as CatalogSource (for the API). +// This excludes internal fields like Type and Properties. +func (sc *SourceCollection) All() map[string]model.CatalogSource { + result := map[string]model.CatalogSource{} + for id, source := range sc.AllSources() { + result[id] = source.CatalogSource + } + return result } +// Get returns a source by name if it exists and is enabled. func (sc *SourceCollection) Get(name string) (src model.CatalogSource, ok bool) { sc.mu.RLock() defer sc.mu.RUnlock() - src, ok = sc.sources[name] - return + // Get from merged view (which includes field-level merging and defaults) + merged := sc.merged() + source, exists := merged[name] + if !exists { + return model.CatalogSource{}, false + } + + // Only return if enabled + if source.Enabled != nil && *source.Enabled { + return source.CatalogSource, true + } + return model.CatalogSource{}, false } -// ByLabel returns sources that have any of the labels provided. The matching +// ByLabel returns enabled sources that have any of the labels provided. The matching // is case insensitive. // // If a label is "null", every source without a label is returned. @@ -82,20 +216,29 @@ func (sc *SourceCollection) ByLabel(labels []string) []model.CatalogSource { } matches := map[string]model.CatalogSource{} + sources := sc.merged() if _, hasNull := labelMap["null"]; hasNull { - for _, source := range sc.sources { + for _, source := range sources { + // Skip disabled sources + if source.Enabled == nil || !*source.Enabled { + continue + } if len(source.Labels) == 0 { - matches[source.Id] = source + matches[source.Id] = source.CatalogSource } } } OUTER: - for _, source := range sc.sources { + for _, source := range sources { + // Skip disabled sources + if source.Enabled == nil || !*source.Enabled { + continue + } for _, label := range source.Labels { if _, match := labelMap[strings.ToLower(label)]; match { - matches[source.Id] = source + matches[source.Id] = source.CatalogSource continue OUTER } } diff --git a/catalog/internal/catalog/sources_test.go b/catalog/internal/catalog/sources_test.go index 906d340b89..63a34cf170 100644 --- a/catalog/internal/catalog/sources_test.go +++ b/catalog/internal/catalog/sources_test.go @@ -11,36 +11,47 @@ import ( func TestSourceCollection_ByLabel(t *testing.T) { // Create test sources with various labels - sources := map[string]model.CatalogSource{ + // Note: source3 is disabled and should not appear in results + sources := map[string]Source{ "source1": { - Id: "source1", - Name: "Source 1", - Enabled: apiutils.Of(true), - Labels: []string{"frontend", "production"}, + CatalogSource: model.CatalogSource{ + Id: "source1", + Name: "Source 1", + Enabled: apiutils.Of(true), + Labels: []string{"frontend", "production"}, + }, }, "source2": { - Id: "source2", - Name: "Source 2", - Enabled: apiutils.Of(true), - Labels: []string{"Backend", "Development"}, // Mixed case to test case insensitivity + CatalogSource: model.CatalogSource{ + Id: "source2", + Name: "Source 2", + Enabled: apiutils.Of(true), + Labels: []string{"Backend", "Development"}, // Mixed case to test case insensitivity + }, }, "source3": { - Id: "source3", - Name: "Source 3", - Enabled: apiutils.Of(false), - Labels: []string{"analytics", "PRODUCTION"}, // Mixed case + CatalogSource: model.CatalogSource{ + Id: "source3", + Name: "Source 3", + Enabled: apiutils.Of(false), // Disabled - should not appear in results + Labels: []string{"analytics", "PRODUCTION"}, // Mixed case + }, }, "source4": { - Id: "source4", - Name: "Source 4", - Enabled: apiutils.Of(true), - Labels: []string{"testing", "staging"}, + CatalogSource: model.CatalogSource{ + Id: "source4", + Name: "Source 4", + Enabled: apiutils.Of(true), + Labels: []string{"testing", "staging", "analytics"}, // Added analytics to test this label with enabled source + }, }, "source5": { - Id: "source5", - Name: "Source 5", - Enabled: apiutils.Of(true), - Labels: []string{}, // No labels + CatalogSource: model.CatalogSource{ + Id: "source5", + Name: "Source 5", + Enabled: apiutils.Of(true), + Labels: []string{}, // No labels + }, }, } @@ -70,9 +81,9 @@ func TestSourceCollection_ByLabel(t *testing.T) { expectedSources: []string{"source1", "source2"}, }, { - name: "production label case insensitive", + name: "production label case insensitive - disabled source excluded", labels: []string{"production"}, - expectedSources: []string{"source1", "source3"}, + expectedSources: []string{"source1"}, // source3 is disabled }, { name: "no matching labels", @@ -85,9 +96,9 @@ func TestSourceCollection_ByLabel(t *testing.T) { expectedSources: nil, }, { - name: "multiple different labels", + name: "multiple different labels - disabled source excluded", labels: []string{"analytics", "testing"}, - expectedSources: []string{"source3", "source4"}, + expectedSources: []string{"source4"}, // source3 is disabled, source4 has both labels }, } @@ -170,12 +181,14 @@ func TestSourceCollection_ByLabel_NilLabels(t *testing.T) { sc := NewSourceCollection() // Add a source with nil labels (edge case) - sources := map[string]model.CatalogSource{ + sources := map[string]Source{ "source1": { - Id: "source1", - Name: "Source 1", - Enabled: apiutils.Of(true), - Labels: nil, // nil labels + CatalogSource: model.CatalogSource{ + Id: "source1", + Name: "Source 1", + Enabled: apiutils.Of(true), + Labels: nil, // nil labels + }, }, } @@ -211,32 +224,820 @@ func TestSourceCollection_ByLabel_NilLabels(t *testing.T) { } } +func TestSourceCollection_MergeOverride(t *testing.T) { + tests := []struct { + name string + originOrder []string + mergeSequence []struct { + origin string + sources map[string]Source + } + expectedSources map[string]model.CatalogSource + }{ + { + name: "later origin overrides earlier origin", + originOrder: []string{"default.yaml", "user.yaml"}, + mergeSequence: []struct { + origin string + sources map[string]Source + }{ + { + origin: "default.yaml", + sources: map[string]Source{ + "hf": {CatalogSource: model.CatalogSource{ + Id: "hf", + Name: "Hugging Face", + Enabled: apiutils.Of(true), + Labels: []string{"default"}, + ExcludedModels: []string{"model-a"}, + }}, + }, + }, + { + origin: "user.yaml", + sources: map[string]Source{ + "hf": {CatalogSource: model.CatalogSource{ + Id: "hf", + Name: "Hugging Face Custom", + Enabled: apiutils.Of(true), + Labels: []string{"custom"}, + ExcludedModels: []string{"model-a", "DeepSeek"}, + }}, + }, + }, + }, + expectedSources: map[string]model.CatalogSource{ + "hf": { + Id: "hf", + Name: "Hugging Face Custom", + Enabled: apiutils.Of(true), + Labels: []string{"custom"}, + ExcludedModels: []string{"model-a", "DeepSeek"}, + }, + }, + }, + { + name: "source from single origin behaves as before", + originOrder: []string{"default.yaml", "user.yaml"}, + mergeSequence: []struct { + origin string + sources map[string]Source + }{ + { + origin: "default.yaml", + sources: map[string]Source{ + "hf": {CatalogSource: model.CatalogSource{ + Id: "hf", + Name: "Hugging Face", + Enabled: apiutils.Of(true), + Labels: []string{"default"}, + }}, + }, + }, + { + origin: "user.yaml", + sources: map[string]Source{}, + }, + }, + expectedSources: map[string]model.CatalogSource{ + "hf": { + Id: "hf", + Name: "Hugging Face", + Enabled: apiutils.Of(true), + Labels: []string{"default"}, + }, + }, + }, + { + name: "multiple sources with partial override", + originOrder: []string{"default.yaml", "user.yaml"}, + mergeSequence: []struct { + origin string + sources map[string]Source + }{ + { + origin: "default.yaml", + sources: map[string]Source{ + "hf": {CatalogSource: model.CatalogSource{ + Id: "hf", + Name: "Hugging Face", + Enabled: apiutils.Of(true), + Labels: []string{}, + }}, + "local": {CatalogSource: model.CatalogSource{ + Id: "local", + Name: "Local Files", + Enabled: apiutils.Of(true), + Labels: []string{}, + }}, + }, + }, + { + origin: "user.yaml", + sources: map[string]Source{ + "hf": {CatalogSource: model.CatalogSource{ + Id: "hf", + Name: "Hugging Face", + Enabled: apiutils.Of(true), + Labels: []string{}, + ExcludedModels: []string{"DeepSeek"}, + }}, + }, + }, + }, + expectedSources: map[string]model.CatalogSource{ + "hf": { + Id: "hf", + Name: "Hugging Face", + Enabled: apiutils.Of(true), + Labels: []string{}, + ExcludedModels: []string{"DeepSeek"}, + }, + "local": { + Id: "local", + Name: "Local Files", + Enabled: apiutils.Of(true), + Labels: []string{}, + }, + }, + }, + { + name: "three origins with cascading override", + originOrder: []string{"base.yaml", "team.yaml", "user.yaml"}, + mergeSequence: []struct { + origin string + sources map[string]Source + }{ + { + origin: "base.yaml", + sources: map[string]Source{ + "hf": {CatalogSource: model.CatalogSource{ + Id: "hf", + Name: "Base HF", + Enabled: apiutils.Of(true), + Labels: []string{"base"}, + }}, + }, + }, + { + origin: "team.yaml", + sources: map[string]Source{ + "hf": {CatalogSource: model.CatalogSource{ + Id: "hf", + Name: "Team HF", + Enabled: apiutils.Of(true), + Labels: []string{"team"}, + }}, + }, + }, + { + origin: "user.yaml", + sources: map[string]Source{ + "hf": {CatalogSource: model.CatalogSource{ + Id: "hf", + Name: "User HF", + Enabled: apiutils.Of(true), + Labels: []string{"user"}, + }}, + }, + }, + }, + expectedSources: map[string]model.CatalogSource{ + "hf": { + Id: "hf", + Name: "User HF", + Enabled: apiutils.Of(true), + Labels: []string{"user"}, + }, + }, + }, + { + name: "user can disable a source from default - disabled sources not returned", + originOrder: []string{"default.yaml", "user.yaml"}, + mergeSequence: []struct { + origin string + sources map[string]Source + }{ + { + origin: "default.yaml", + sources: map[string]Source{ + "hf": {CatalogSource: model.CatalogSource{ + Id: "hf", + Name: "Hugging Face", + Enabled: apiutils.Of(true), + Labels: []string{}, + }}, + }, + }, + { + origin: "user.yaml", + sources: map[string]Source{ + "hf": {CatalogSource: model.CatalogSource{ + Id: "hf", + Name: "Hugging Face", + Enabled: apiutils.Of(false), + Labels: []string{}, + }}, + }, + }, + }, + // Disabled sources are not returned by All() + expectedSources: map[string]model.CatalogSource{}, + }, + { + name: "sparse override: user enables disabled source with just id and enabled", + originOrder: []string{"default.yaml", "user.yaml"}, + mergeSequence: []struct { + origin string + sources map[string]Source + }{ + { + origin: "default.yaml", + sources: map[string]Source{ + "models_x": {CatalogSource: model.CatalogSource{ + Id: "models_x", + Name: "Models X Catalog", + Enabled: apiutils.Of(false), // Disabled in default + Labels: []string{"enterprise"}, + IncludedModels: []string{"model-a", "model-b"}, + ExcludedModels: []string{"model-c"}, + }}, + }, + }, + { + origin: "user.yaml", + sources: map[string]Source{ + // Sparse override: only id and enabled + "models_x": {CatalogSource: model.CatalogSource{ + Id: "models_x", + Enabled: apiutils.Of(true), // Enable it + // Name, Labels, IncludedModels, ExcludedModels are nil/empty + }}, + }, + }, + }, + expectedSources: map[string]model.CatalogSource{ + "models_x": { + Id: "models_x", + Name: "Models X Catalog", // Inherited from default + Enabled: apiutils.Of(true), // Overridden by user + Labels: []string{"enterprise"}, // Inherited from default + IncludedModels: []string{"model-a", "model-b"}, // Inherited from default + ExcludedModels: []string{"model-c"}, // Inherited from default + }, + }, + }, + { + name: "sparse override: user changes only excluded models", + originOrder: []string{"default.yaml", "user.yaml"}, + mergeSequence: []struct { + origin string + sources map[string]Source + }{ + { + origin: "default.yaml", + sources: map[string]Source{ + "hf": {CatalogSource: model.CatalogSource{ + Id: "hf", + Name: "Hugging Face", + Enabled: apiutils.Of(true), + Labels: []string{"public"}, + ExcludedModels: []string{"model-a"}, + }}, + }, + }, + { + origin: "user.yaml", + sources: map[string]Source{ + "hf": {CatalogSource: model.CatalogSource{ + Id: "hf", + // Only override ExcludedModels + ExcludedModels: []string{"model-a", "DeepSeek", "banned-model"}, + }}, + }, + }, + }, + expectedSources: map[string]model.CatalogSource{ + "hf": { + Id: "hf", + Name: "Hugging Face", // Inherited + Enabled: apiutils.Of(true), // Inherited + Labels: []string{"public"}, // Inherited + ExcludedModels: []string{"model-a", "DeepSeek", "banned-model"}, // Overridden + }, + }, + }, + { + name: "sparse override: user clears labels with empty slice", + originOrder: []string{"default.yaml", "user.yaml"}, + mergeSequence: []struct { + origin string + sources map[string]Source + }{ + { + origin: "default.yaml", + sources: map[string]Source{ + "hf": {CatalogSource: model.CatalogSource{ + Id: "hf", + Name: "Hugging Face", + Enabled: apiutils.Of(true), + Labels: []string{"public", "ai"}, + }}, + }, + }, + { + origin: "user.yaml", + sources: map[string]Source{ + "hf": {CatalogSource: model.CatalogSource{ + Id: "hf", + Labels: []string{}, // Explicitly clear labels + }}, + }, + }, + }, + expectedSources: map[string]model.CatalogSource{ + "hf": { + Id: "hf", + Name: "Hugging Face", // Inherited + Enabled: apiutils.Of(true), // Inherited + Labels: []string{}, // Overridden to empty + }, + }, + }, + { + name: "defaults applied: enabled defaults to true, labels defaults to empty", + originOrder: []string{"default.yaml"}, + mergeSequence: []struct { + origin string + sources map[string]Source + }{ + { + origin: "default.yaml", + sources: map[string]Source{ + "hf": {CatalogSource: model.CatalogSource{ + Id: "hf", + Name: "Hugging Face", + // Enabled and Labels are nil + }}, + }, + }, + }, + expectedSources: map[string]model.CatalogSource{ + "hf": { + Id: "hf", + Name: "Hugging Face", + Enabled: apiutils.Of(true), // Default applied + Labels: []string{}, // Default applied + }, + }, + }, + { + name: "sparse override: type and properties are inherited", + originOrder: []string{"default.yaml", "user.yaml"}, + mergeSequence: []struct { + origin string + sources map[string]Source + }{ + { + origin: "default.yaml", + sources: map[string]Source{ + "models_x": { + CatalogSource: model.CatalogSource{ + Id: "models_x", + Name: "Models X Catalog", + Enabled: apiutils.Of(false), // Disabled in default + Labels: []string{"enterprise"}, + }, + Type: "yaml", + Properties: map[string]any{ + "yamlCatalogPath": "models-x.yaml", + }, + }, + }, + }, + { + origin: "user.yaml", + sources: map[string]Source{ + // Sparse override: only id and enabled + "models_x": { + CatalogSource: model.CatalogSource{ + Id: "models_x", + Enabled: apiutils.Of(true), // Enable it + }, + // Type and Properties are empty/nil - should be inherited + }, + }, + }, + }, + expectedSources: map[string]model.CatalogSource{ + "models_x": { + Id: "models_x", + Name: "Models X Catalog", // Inherited from default + Enabled: apiutils.Of(true), // Overridden by user + Labels: []string{"enterprise"}, // Inherited from default + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sc := NewSourceCollection(tt.originOrder...) + + for _, merge := range tt.mergeSequence { + err := sc.Merge(merge.origin, merge.sources) + if err != nil { + t.Fatalf("Merge(%s) failed: %v", merge.origin, err) + } + } + + result := sc.All() + + if len(result) != len(tt.expectedSources) { + t.Errorf("All() returned %d sources, want %d", len(result), len(tt.expectedSources)) + } + + for id, expected := range tt.expectedSources { + got, ok := result[id] + if !ok { + t.Errorf("source %s not found in result", id) + continue + } + if got.Id != expected.Id { + t.Errorf("source %s: Id = %s, want %s", id, got.Id, expected.Id) + } + if got.Name != expected.Name { + t.Errorf("source %s: Name = %s, want %s", id, got.Name, expected.Name) + } + if *got.Enabled != *expected.Enabled { + t.Errorf("source %s: Enabled = %v, want %v", id, *got.Enabled, *expected.Enabled) + } + if !reflect.DeepEqual(got.Labels, expected.Labels) { + t.Errorf("source %s: Labels = %v, want %v", id, got.Labels, expected.Labels) + } + if !reflect.DeepEqual(got.ExcludedModels, expected.ExcludedModels) { + t.Errorf("source %s: ExcludedModels = %v, want %v", id, got.ExcludedModels, expected.ExcludedModels) + } + } + }) + } +} + +func TestSourceCollection_MergeOverride_Get(t *testing.T) { + sc := NewSourceCollection("default.yaml", "user.yaml") + + // Merge default config + err := sc.Merge("default.yaml", map[string]Source{ + "hf": {CatalogSource: model.CatalogSource{ + Id: "hf", + Name: "Hugging Face Default", + Enabled: apiutils.Of(true), + Labels: []string{}, + }}, + }) + if err != nil { + t.Fatalf("Merge(default.yaml) failed: %v", err) + } + + // Merge user config that overrides + err = sc.Merge("user.yaml", map[string]Source{ + "hf": {CatalogSource: model.CatalogSource{ + Id: "hf", + Name: "Hugging Face User", + Enabled: apiutils.Of(true), + Labels: []string{"user-managed"}, + ExcludedModels: []string{"DeepSeek"}, + }}, + }) + if err != nil { + t.Fatalf("Merge(user.yaml) failed: %v", err) + } + + // Get should return the overridden source + source, ok := sc.Get("hf") + if !ok { + t.Fatal("Get(hf) returned false, want true") + } + if source.Name != "Hugging Face User" { + t.Errorf("Get(hf).Name = %s, want 'Hugging Face User'", source.Name) + } + if len(source.ExcludedModels) != 1 || source.ExcludedModels[0] != "DeepSeek" { + t.Errorf("Get(hf).ExcludedModels = %v, want [DeepSeek]", source.ExcludedModels) + } +} + +func TestSourceCollection_MergeOverride_DynamicOrigin(t *testing.T) { + // Test that origins not in the initial order are appended + sc := NewSourceCollection("default.yaml") + + err := sc.Merge("default.yaml", map[string]Source{ + "hf": {CatalogSource: model.CatalogSource{Id: "hf", Name: "Default", Enabled: apiutils.Of(true), Labels: []string{}}}, + }) + if err != nil { + t.Fatalf("Merge(default.yaml) failed: %v", err) + } + + // Dynamic origin not in initial order + err = sc.Merge("extra.yaml", map[string]Source{ + "hf": {CatalogSource: model.CatalogSource{Id: "hf", Name: "Extra", Enabled: apiutils.Of(true), Labels: []string{}}}, + }) + if err != nil { + t.Fatalf("Merge(extra.yaml) failed: %v", err) + } + + source, _ := sc.Get("hf") + if source.Name != "Extra" { + t.Errorf("dynamically added origin should override earlier origins, got Name = %s", source.Name) + } +} + +func TestSourceCollection_MergeOverride_TypeAndProperties(t *testing.T) { + // Test that Type and Properties are properly inherited with sparse overrides + sc := NewSourceCollection("default.yaml", "user.yaml") + + // Merge default config with full source definition + err := sc.Merge("default.yaml", map[string]Source{ + "models_x": { + CatalogSource: model.CatalogSource{ + Id: "models_x", + Name: "Models X Catalog", + Enabled: apiutils.Of(false), // Disabled + Labels: []string{"enterprise"}, + }, + Type: "yaml", + Properties: map[string]any{ + "yamlCatalogPath": "models-x.yaml", + "otherProp": 123, + }, + }, + }) + if err != nil { + t.Fatalf("Merge(default.yaml) failed: %v", err) + } + + // Merge sparse user config that only enables the source + err = sc.Merge("user.yaml", map[string]Source{ + "models_x": { + CatalogSource: model.CatalogSource{ + Id: "models_x", + Enabled: apiutils.Of(true), // Enable it + }, + // Type and Properties are empty - should be inherited + }, + }) + if err != nil { + t.Fatalf("Merge(user.yaml) failed: %v", err) + } + + // AllSources should return the merged source with Type and Properties + sources := sc.AllSources() + source, ok := sources["models_x"] + if !ok { + t.Fatal("AllSources() should return models_x") + } + + if source.Type != "yaml" { + t.Errorf("Type = %s, want 'yaml' (inherited from default)", source.Type) + } + + if source.Properties == nil { + t.Fatal("Properties should be inherited from default") + } + + if source.Properties["yamlCatalogPath"] != "models-x.yaml" { + t.Errorf("Properties[yamlCatalogPath] = %v, want 'models-x.yaml'", source.Properties["yamlCatalogPath"]) + } + + if source.Properties["otherProp"] != 123 { + t.Errorf("Properties[otherProp] = %v, want 123", source.Properties["otherProp"]) + } + + // CatalogSource fields should also be merged + if source.Name != "Models X Catalog" { + t.Errorf("Name = %s, want 'Models X Catalog'", source.Name) + } + + if *source.Enabled != true { + t.Errorf("Enabled = %v, want true", *source.Enabled) + } +} + +func TestSourceCollection_MergeOverride_Origin(t *testing.T) { + // Test that Origin is correctly tracked through merge operations + // This is important for resolving relative paths in source properties + + tests := []struct { + name string + originOrder []string + mergeSequence []struct { + origin string + sources map[string]Source + } + expectedOrigins map[string]string // sourceId -> expected origin + }{ + { + name: "origin is preserved from first definition", + originOrder: []string{"/config/default.yaml", "/user-config/user.yaml"}, + mergeSequence: []struct { + origin string + sources map[string]Source + }{ + { + origin: "/config/default.yaml", + sources: map[string]Source{ + "hf": { + CatalogSource: model.CatalogSource{ + Id: "hf", + Name: "Hugging Face", + Enabled: apiutils.Of(true), + }, + Type: "yaml", + Properties: map[string]any{ + "yamlCatalogPath": "models.yaml", + }, + Origin: "/config/default.yaml", + }, + }, + }, + { + origin: "/user-config/user.yaml", + sources: map[string]Source{ + // Sparse override: only enable and change name + "hf": { + CatalogSource: model.CatalogSource{ + Id: "hf", + Name: "Hugging Face Custom", + Enabled: apiutils.Of(true), + }, + Origin: "/user-config/user.yaml", + // Properties is nil, so Origin should stay with base + }, + }, + }, + }, + // Origin should be from default.yaml since Properties weren't overridden + expectedOrigins: map[string]string{ + "hf": "/config/default.yaml", + }, + }, + { + name: "origin changes when properties are overridden", + originOrder: []string{"/config/default.yaml", "/user-config/user.yaml"}, + mergeSequence: []struct { + origin string + sources map[string]Source + }{ + { + origin: "/config/default.yaml", + sources: map[string]Source{ + "local": { + CatalogSource: model.CatalogSource{ + Id: "local", + Name: "Local Catalog", + Enabled: apiutils.Of(true), + }, + Type: "yaml", + Properties: map[string]any{ + "yamlCatalogPath": "default-models.yaml", + }, + Origin: "/config/default.yaml", + }, + }, + }, + { + origin: "/user-config/user.yaml", + sources: map[string]Source{ + "local": { + CatalogSource: model.CatalogSource{ + Id: "local", + Enabled: apiutils.Of(true), + }, + // Override Properties - this changes where relative paths resolve from + Properties: map[string]any{ + "yamlCatalogPath": "user-models.yaml", + }, + Origin: "/user-config/user.yaml", + }, + }, + }, + }, + // Origin should be from user.yaml since Properties were overridden + expectedOrigins: map[string]string{ + "local": "/user-config/user.yaml", + }, + }, + { + name: "multiple sources from different origins", + originOrder: []string{"/admin/sources.yaml", "/user/sources.yaml"}, + mergeSequence: []struct { + origin string + sources map[string]Source + }{ + { + origin: "/admin/sources.yaml", + sources: map[string]Source{ + "admin-catalog": { + CatalogSource: model.CatalogSource{ + Id: "admin-catalog", + Name: "Admin Catalog", + Enabled: apiutils.Of(true), + }, + Type: "yaml", + Properties: map[string]any{ + "yamlCatalogPath": "admin-models.yaml", + }, + Origin: "/admin/sources.yaml", + }, + }, + }, + { + origin: "/user/sources.yaml", + sources: map[string]Source{ + "user-catalog": { + CatalogSource: model.CatalogSource{ + Id: "user-catalog", + Name: "User Catalog", + Enabled: apiutils.Of(true), + }, + Type: "yaml", + Properties: map[string]any{ + "yamlCatalogPath": "user-models.yaml", + }, + Origin: "/user/sources.yaml", + }, + }, + }, + }, + // Each source should keep its own origin + expectedOrigins: map[string]string{ + "admin-catalog": "/admin/sources.yaml", + "user-catalog": "/user/sources.yaml", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sc := NewSourceCollection(tt.originOrder...) + + for _, merge := range tt.mergeSequence { + err := sc.Merge(merge.origin, merge.sources) + if err != nil { + t.Fatalf("Merge(%s) failed: %v", merge.origin, err) + } + } + + sources := sc.AllSources() + + for id, expectedOrigin := range tt.expectedOrigins { + source, ok := sources[id] + if !ok { + t.Errorf("source %s not found in AllSources()", id) + continue + } + if source.Origin != expectedOrigin { + t.Errorf("source %s: Origin = %s, want %s", id, source.Origin, expectedOrigin) + } + } + }) + } +} + func TestSourceCollection_ByLabel_NullBehavior(t *testing.T) { // Create test sources with various label configurations to test "null" behavior - sources := map[string]model.CatalogSource{ + // Note: disabled sources are filtered out and not returned + sources := map[string]Source{ "source_with_labels": { - Id: "source_with_labels", - Name: "Source With Labels", - Enabled: apiutils.Of(true), - Labels: []string{"frontend", "production"}, + CatalogSource: model.CatalogSource{ + Id: "source_with_labels", + Name: "Source With Labels", + Enabled: apiutils.Of(true), + Labels: []string{"frontend", "production"}, + }, }, "source_empty_labels": { - Id: "source_empty_labels", - Name: "Source Empty Labels", - Enabled: apiutils.Of(true), - Labels: []string{}, // Empty labels slice + CatalogSource: model.CatalogSource{ + Id: "source_empty_labels", + Name: "Source Empty Labels", + Enabled: apiutils.Of(true), + Labels: []string{}, // Empty labels slice + }, }, "source_nil_labels": { - Id: "source_nil_labels", - Name: "Source Nil Labels", - Enabled: apiutils.Of(true), - Labels: nil, // Nil labels + CatalogSource: model.CatalogSource{ + Id: "source_nil_labels", + Name: "Source Nil Labels", + Enabled: apiutils.Of(true), + Labels: nil, // Nil labels + }, }, "source_another_with_labels": { - Id: "source_another_with_labels", - Name: "Another Source With Labels", - Enabled: apiutils.Of(false), - Labels: []string{"backend", "testing"}, + CatalogSource: model.CatalogSource{ + Id: "source_another_with_labels", + Name: "Another Source With Labels", + Enabled: apiutils.Of(true), // Changed to enabled + Labels: []string{"backend", "testing"}, + }, }, } diff --git a/catalog/internal/catalog/testdata/test-hf-catalog-sources.yaml b/catalog/internal/catalog/testdata/test-hf-catalog-sources.yaml index 164b3482f1..03ca2b6882 100644 --- a/catalog/internal/catalog/testdata/test-hf-catalog-sources.yaml +++ b/catalog/internal/catalog/testdata/test-hf-catalog-sources.yaml @@ -5,13 +5,21 @@ catalogs: enabled: true properties: apiKey: "hf_test_api_key_here" - url: "https://huggingface.co" modelLimit: 50 + includedModels: + - "ibm-granite/granite-4.0-h-small" + - "ibm-granite/granite-4.0-h-tiny" + excludedModels: + - "ibm-granite/granite-4.0-h-small" - name: "HuggingFace Invalid Credentials" id: hf_invalid type: hf enabled: false # disabled so it doesn't cause startup failures in tests properties: apiKey: "invalid_key" - url: "https://huggingface.co" modelLimit: 10 + includedModels: + - "ibm-granite/granite-4.0-h-small" + - "ibm-granite/granite-4.0-h-tiny" + excludedModels: + - "ibm-granite/granite-4.0-h-small" diff --git a/catalog/internal/db/service/property_options_test.go b/catalog/internal/db/service/property_options_test.go index a225ae4f90..4f38482f70 100644 --- a/catalog/internal/db/service/property_options_test.go +++ b/catalog/internal/db/service/property_options_test.go @@ -12,6 +12,64 @@ import ( "github.com/stretchr/testify/require" ) +// TestPropertyOptionsRepository_RefreshOnEmptyDatabase verifies that the materialized +// views can be refreshed and queried even when no models have been loaded. +// This is a regression test for the startup refresh fix - previously, querying +// unpopulated materialized views would fail with "has not been populated" error. +func TestPropertyOptionsRepository_RefreshOnEmptyDatabase(t *testing.T) { + // Create a database with migrations + sharedDB, cleanup := testutils.SetupPostgresWithMigrations(t, service.DatastoreSpec()) + defer cleanup() + + // Clean up all test data to ensure empty database + testutils.CleanupPostgresTestData(t, sharedDB) + + repo := service.NewPropertyOptionsRepository(sharedDB) + catalogModelTypeID := getCatalogModelTypeID(t, sharedDB) + modelArtifactTypeID := getCatalogModelArtifactTypeID(t, sharedDB) + + t.Run("RefreshAndListWithNoModels_ContextPropertyOptions", func(t *testing.T) { + // Refresh should succeed even with no data + err := repo.Refresh(models.ContextPropertyOptionType) + require.NoError(t, err, "Refresh should succeed on empty database") + + // List should succeed after refresh (returning empty results) + options, err := repo.List(models.ContextPropertyOptionType, catalogModelTypeID) + require.NoError(t, err, "List should succeed after refresh even with no models") + assert.NotNil(t, options) + assert.Empty(t, options, "Should return empty list when no models exist") + }) + + t.Run("RefreshAndListWithNoModels_ArtifactPropertyOptions", func(t *testing.T) { + // Refresh should succeed even with no data + err := repo.Refresh(models.ArtifactPropertyOptionType) + require.NoError(t, err, "Refresh should succeed on empty database") + + // List should succeed after refresh (returning empty results) + options, err := repo.List(models.ArtifactPropertyOptionType, modelArtifactTypeID) + require.NoError(t, err, "List should succeed after refresh even with no models") + assert.NotNil(t, options) + assert.Empty(t, options, "Should return empty list when no artifacts exist") + }) + + t.Run("ListAllTypesWithNoModels", func(t *testing.T) { + // First refresh both views + require.NoError(t, repo.Refresh(models.ContextPropertyOptionType)) + require.NoError(t, repo.Refresh(models.ArtifactPropertyOptionType)) + + // List with typeID=0 should return all options (empty in this case) + contextOptions, err := repo.List(models.ContextPropertyOptionType, 0) + require.NoError(t, err) + assert.NotNil(t, contextOptions) + assert.Empty(t, contextOptions, "Should return empty list when no models exist") + + artifactOptions, err := repo.List(models.ArtifactPropertyOptionType, 0) + require.NoError(t, err) + assert.NotNil(t, artifactOptions) + assert.Empty(t, artifactOptions, "Should return empty list when no artifacts exist") + }) +} + func TestPropertyOptionsRepository(t *testing.T) { sharedDB, cleanup := testutils.SetupPostgresWithMigrations(t, service.DatastoreSpec()) defer cleanup() diff --git a/catalog/internal/server/openapi/api_model_catalog_service_service_test.go b/catalog/internal/server/openapi/api_model_catalog_service_service_test.go index f71941b26c..001dba8ad9 100644 --- a/catalog/internal/server/openapi/api_model_catalog_service_service_test.go +++ b/catalog/internal/server/openapi/api_model_catalog_service_service_test.go @@ -291,8 +291,8 @@ func TestFindModels(t *testing.T) { // Create mock source collection sources := catalog.NewSourceCollection() sources.Merge("", - map[string]model.CatalogSource{ - "source1": model.CatalogSource{Id: "source1", Name: "Test Source 1"}, + map[string]catalog.Source{ + "source1": {CatalogSource: model.CatalogSource{Id: "source1", Name: "Test Source 1"}}, }, ) @@ -344,7 +344,7 @@ func TestFindSources(t *testing.T) { trueValue := true testCases := []struct { name string - catalogs map[string]model.CatalogSource + catalogs map[string]catalog.Source nameFilter string pageSize string orderBy model.OrderByField @@ -358,7 +358,7 @@ func TestFindSources(t *testing.T) { }{ { name: "Empty catalog list", - catalogs: map[string]model.CatalogSource{}, + catalogs: map[string]catalog.Source{}, nameFilter: "", pageSize: "10", orderBy: model.ORDERBYFIELD_ID, @@ -369,8 +369,8 @@ func TestFindSources(t *testing.T) { }, { name: "Single catalog", - catalogs: map[string]model.CatalogSource{ - "catalog1": model.CatalogSource{Id: "catalog1", Name: "Test Catalog 1", Enabled: &trueValue}, + catalogs: map[string]catalog.Source{ + "catalog1": {CatalogSource: model.CatalogSource{Id: "catalog1", Name: "Test Catalog 1", Enabled: &trueValue}}, }, nameFilter: "", pageSize: "10", @@ -382,10 +382,10 @@ func TestFindSources(t *testing.T) { }, { name: "Multiple catalogs with no filter", - catalogs: map[string]model.CatalogSource{ - "catalog1": model.CatalogSource{Id: "catalog1", Name: "Test Catalog 1", Enabled: &trueValue}, - "catalog2": model.CatalogSource{Id: "catalog2", Name: "Test Catalog 2", Enabled: &trueValue}, - "catalog3": model.CatalogSource{Id: "catalog3", Name: "Another Catalog", Enabled: &trueValue}, + catalogs: map[string]catalog.Source{ + "catalog1": {CatalogSource: model.CatalogSource{Id: "catalog1", Name: "Test Catalog 1", Enabled: &trueValue}}, + "catalog2": {CatalogSource: model.CatalogSource{Id: "catalog2", Name: "Test Catalog 2", Enabled: &trueValue}}, + "catalog3": {CatalogSource: model.CatalogSource{Id: "catalog3", Name: "Another Catalog", Enabled: &trueValue}}, }, nameFilter: "", pageSize: "10", @@ -397,10 +397,10 @@ func TestFindSources(t *testing.T) { }, { name: "Filter by name", - catalogs: map[string]model.CatalogSource{ - "catalog1": model.CatalogSource{Id: "catalog1", Name: "Test Catalog 1", Enabled: &trueValue}, - "catalog2": model.CatalogSource{Id: "catalog2", Name: "Test Catalog 2", Enabled: &trueValue}, - "catalog3": model.CatalogSource{Id: "catalog3", Name: "Another Catalog", Enabled: &trueValue}, + catalogs: map[string]catalog.Source{ + "catalog1": {CatalogSource: model.CatalogSource{Id: "catalog1", Name: "Test Catalog 1", Enabled: &trueValue}}, + "catalog2": {CatalogSource: model.CatalogSource{Id: "catalog2", Name: "Test Catalog 2", Enabled: &trueValue}}, + "catalog3": {CatalogSource: model.CatalogSource{Id: "catalog3", Name: "Another Catalog", Enabled: &trueValue}}, }, nameFilter: "Test", pageSize: "10", @@ -412,10 +412,10 @@ func TestFindSources(t *testing.T) { }, { name: "Filter by name case insensitive", - catalogs: map[string]model.CatalogSource{ - "catalog1": model.CatalogSource{Id: "catalog1", Name: "Test Catalog 1", Enabled: &trueValue}, - "catalog2": model.CatalogSource{Id: "catalog2", Name: "Test Catalog 2", Enabled: &trueValue}, - "catalog3": model.CatalogSource{Id: "catalog3", Name: "Another Catalog", Enabled: &trueValue}, + catalogs: map[string]catalog.Source{ + "catalog1": {CatalogSource: model.CatalogSource{Id: "catalog1", Name: "Test Catalog 1", Enabled: &trueValue}}, + "catalog2": {CatalogSource: model.CatalogSource{Id: "catalog2", Name: "Test Catalog 2", Enabled: &trueValue}}, + "catalog3": {CatalogSource: model.CatalogSource{Id: "catalog3", Name: "Another Catalog", Enabled: &trueValue}}, }, nameFilter: "test", pageSize: "10", @@ -427,10 +427,10 @@ func TestFindSources(t *testing.T) { }, { name: "Pagination - limit results", - catalogs: map[string]model.CatalogSource{ - "catalog1": model.CatalogSource{Id: "catalog1", Name: "Test Catalog 1", Enabled: &trueValue}, - "catalog2": model.CatalogSource{Id: "catalog2", Name: "Test Catalog 2", Enabled: &trueValue}, - "catalog3": model.CatalogSource{Id: "catalog3", Name: "Another Catalog", Enabled: &trueValue}, + catalogs: map[string]catalog.Source{ + "catalog1": {CatalogSource: model.CatalogSource{Id: "catalog1", Name: "Test Catalog 1", Enabled: &trueValue}}, + "catalog2": {CatalogSource: model.CatalogSource{Id: "catalog2", Name: "Test Catalog 2", Enabled: &trueValue}}, + "catalog3": {CatalogSource: model.CatalogSource{Id: "catalog3", Name: "Another Catalog", Enabled: &trueValue}}, }, nameFilter: "", pageSize: "2", @@ -442,9 +442,9 @@ func TestFindSources(t *testing.T) { }, { name: "Default page size", - catalogs: map[string]model.CatalogSource{ - "catalog1": model.CatalogSource{Id: "catalog1", Name: "Test Catalog 1", Enabled: &trueValue}, - "catalog2": model.CatalogSource{Id: "catalog2", Name: "Test Catalog 2", Enabled: &trueValue}, + catalogs: map[string]catalog.Source{ + "catalog1": {CatalogSource: model.CatalogSource{Id: "catalog1", Name: "Test Catalog 1", Enabled: &trueValue}}, + "catalog2": {CatalogSource: model.CatalogSource{Id: "catalog2", Name: "Test Catalog 2", Enabled: &trueValue}}, }, nameFilter: "", pageSize: "", // Empty to test default @@ -456,8 +456,8 @@ func TestFindSources(t *testing.T) { }, { name: "Invalid page size", - catalogs: map[string]model.CatalogSource{ - "catalog1": model.CatalogSource{Id: "catalog1", Name: "Test Catalog 1", Enabled: &trueValue}, + catalogs: map[string]catalog.Source{ + "catalog1": {CatalogSource: model.CatalogSource{Id: "catalog1", Name: "Test Catalog 1", Enabled: &trueValue}}, }, nameFilter: "", pageSize: "invalid", @@ -467,10 +467,10 @@ func TestFindSources(t *testing.T) { }, { name: "Sort by ID ascending", - catalogs: map[string]model.CatalogSource{ - "catalog2": model.CatalogSource{Id: "catalog2", Name: "B Catalog", Enabled: &trueValue}, - "catalog1": model.CatalogSource{Id: "catalog1", Name: "A Catalog", Enabled: &trueValue}, - "catalog3": model.CatalogSource{Id: "catalog3", Name: "C Catalog", Enabled: &trueValue}, + catalogs: map[string]catalog.Source{ + "catalog2": {CatalogSource: model.CatalogSource{Id: "catalog2", Name: "B Catalog", Enabled: &trueValue}}, + "catalog1": {CatalogSource: model.CatalogSource{Id: "catalog1", Name: "A Catalog", Enabled: &trueValue}}, + "catalog3": {CatalogSource: model.CatalogSource{Id: "catalog3", Name: "C Catalog", Enabled: &trueValue}}, }, nameFilter: "", pageSize: "10", @@ -483,10 +483,10 @@ func TestFindSources(t *testing.T) { }, { name: "Sort by ID descending", - catalogs: map[string]model.CatalogSource{ - "catalog2": model.CatalogSource{Id: "catalog2", Name: "B Catalog", Enabled: &trueValue}, - "catalog1": model.CatalogSource{Id: "catalog1", Name: "A Catalog", Enabled: &trueValue}, - "catalog3": model.CatalogSource{Id: "catalog3", Name: "C Catalog", Enabled: &trueValue}, + catalogs: map[string]catalog.Source{ + "catalog2": {CatalogSource: model.CatalogSource{Id: "catalog2", Name: "B Catalog", Enabled: &trueValue}}, + "catalog1": {CatalogSource: model.CatalogSource{Id: "catalog1", Name: "A Catalog", Enabled: &trueValue}}, + "catalog3": {CatalogSource: model.CatalogSource{Id: "catalog3", Name: "C Catalog", Enabled: &trueValue}}, }, nameFilter: "", pageSize: "10", @@ -499,10 +499,10 @@ func TestFindSources(t *testing.T) { }, { name: "Sort by name ascending", - catalogs: map[string]model.CatalogSource{ - "catalog2": model.CatalogSource{Id: "catalog2", Name: "B Catalog", Enabled: &trueValue}, - "catalog1": model.CatalogSource{Id: "catalog1", Name: "A Catalog", Enabled: &trueValue}, - "catalog3": model.CatalogSource{Id: "catalog3", Name: "C Catalog", Enabled: &trueValue}, + catalogs: map[string]catalog.Source{ + "catalog2": {CatalogSource: model.CatalogSource{Id: "catalog2", Name: "B Catalog", Enabled: &trueValue}}, + "catalog1": {CatalogSource: model.CatalogSource{Id: "catalog1", Name: "A Catalog", Enabled: &trueValue}}, + "catalog3": {CatalogSource: model.CatalogSource{Id: "catalog3", Name: "C Catalog", Enabled: &trueValue}}, }, nameFilter: "", pageSize: "10", @@ -515,10 +515,10 @@ func TestFindSources(t *testing.T) { }, { name: "Sort by name descending", - catalogs: map[string]model.CatalogSource{ - "catalog2": model.CatalogSource{Id: "catalog2", Name: "B Catalog", Enabled: &trueValue}, - "catalog1": model.CatalogSource{Id: "catalog1", Name: "A Catalog", Enabled: &trueValue}, - "catalog3": model.CatalogSource{Id: "catalog3", Name: "C Catalog"}, + catalogs: map[string]catalog.Source{ + "catalog2": {CatalogSource: model.CatalogSource{Id: "catalog2", Name: "B Catalog", Enabled: &trueValue}}, + "catalog1": {CatalogSource: model.CatalogSource{Id: "catalog1", Name: "A Catalog", Enabled: &trueValue}}, + "catalog3": {CatalogSource: model.CatalogSource{Id: "catalog3", Name: "C Catalog"}}, }, nameFilter: "", pageSize: "10", @@ -531,8 +531,8 @@ func TestFindSources(t *testing.T) { }, { name: "Invalid sort order", - catalogs: map[string]model.CatalogSource{ - "catalog1": model.CatalogSource{Id: "catalog1", Name: "Test Catalog 1"}, + catalogs: map[string]catalog.Source{ + "catalog1": {CatalogSource: model.CatalogSource{Id: "catalog1", Name: "Test Catalog 1"}}, }, nameFilter: "", pageSize: "10", @@ -542,8 +542,8 @@ func TestFindSources(t *testing.T) { }, { name: "Invalid order by field", - catalogs: map[string]model.CatalogSource{ - "catalog1": model.CatalogSource{Id: "catalog1", Name: "Test Catalog 1"}, + catalogs: map[string]catalog.Source{ + "catalog1": {CatalogSource: model.CatalogSource{Id: "catalog1", Name: "Test Catalog 1"}}, }, nameFilter: "", pageSize: "10", @@ -553,10 +553,10 @@ func TestFindSources(t *testing.T) { }, { name: "Default sort order (ID ascending)", - catalogs: map[string]model.CatalogSource{ - "catalog2": model.CatalogSource{Id: "catalog2", Name: "B Catalog"}, - "catalog1": model.CatalogSource{Id: "catalog1", Name: "A Catalog"}, - "catalog3": model.CatalogSource{Id: "catalog3", Name: "C Catalog"}, + catalogs: map[string]catalog.Source{ + "catalog2": {CatalogSource: model.CatalogSource{Id: "catalog2", Name: "B Catalog"}}, + "catalog1": {CatalogSource: model.CatalogSource{Id: "catalog1", Name: "A Catalog"}}, + "catalog3": {CatalogSource: model.CatalogSource{Id: "catalog3", Name: "C Catalog"}}, }, nameFilter: "", pageSize: "10", @@ -569,10 +569,10 @@ func TestFindSources(t *testing.T) { }, { name: "Labels should be returned if set", - catalogs: map[string]model.CatalogSource{ - "catalog1": model.CatalogSource{Id: "catalog1", Name: "Test Catalog 1", Labels: []string{"label1", "label2"}}, - "catalog2": model.CatalogSource{Id: "catalog2", Name: "Test Catalog 2", Labels: []string{"label3", "label4"}}, - "catalog3": model.CatalogSource{Id: "catalog3", Name: "Test Catalog 3", Labels: []string{"label5", "label6"}}, + catalogs: map[string]catalog.Source{ + "catalog1": {CatalogSource: model.CatalogSource{Id: "catalog1", Name: "Test Catalog 1", Labels: []string{"label1", "label2"}}}, + "catalog2": {CatalogSource: model.CatalogSource{Id: "catalog2", Name: "Test Catalog 2", Labels: []string{"label3", "label4"}}}, + "catalog3": {CatalogSource: model.CatalogSource{Id: "catalog3", Name: "Test Catalog 3", Labels: []string{"label5", "label6"}}}, }, nameFilter: "", pageSize: "10", @@ -1084,7 +1084,7 @@ func (m *mockModelProvider) GetFilterOptions(ctx context.Context) (*model.Filter func TestGetModel(t *testing.T) { testCases := []struct { name string - sources map[string]model.CatalogSource + sources map[string]catalog.Source sourceID string modelName string expectedStatus int @@ -1093,8 +1093,8 @@ func TestGetModel(t *testing.T) { }{ { name: "Existing model in source", - sources: map[string]model.CatalogSource{ - "source1": model.CatalogSource{Id: "source1", Name: "Test Source"}, + sources: map[string]catalog.Source{ + "source1": {CatalogSource: model.CatalogSource{Id: "source1", Name: "Test Source"}}, }, provider: &mockModelProvider{ models: map[string]*model.CatalogModel{ @@ -1112,8 +1112,8 @@ func TestGetModel(t *testing.T) { }, { name: "Non-existing source", - sources: map[string]model.CatalogSource{ - "source1": model.CatalogSource{Id: "source1", Name: "Test Source"}, + sources: map[string]catalog.Source{ + "source1": {CatalogSource: model.CatalogSource{Id: "source1", Name: "Test Source"}}, }, provider: &mockModelProvider{ models: map[string]*model.CatalogModel{}, @@ -1125,8 +1125,8 @@ func TestGetModel(t *testing.T) { }, { name: "Existing source, non-existing model", - sources: map[string]model.CatalogSource{ - "source1": model.CatalogSource{Id: "source1", Name: "Test Source"}, + sources: map[string]catalog.Source{ + "source1": {CatalogSource: model.CatalogSource{Id: "source1", Name: "Test Source"}}, }, provider: &mockModelProvider{ models: map[string]*model.CatalogModel{}, @@ -1138,8 +1138,8 @@ func TestGetModel(t *testing.T) { }, { name: "Model name with an escaped slash and version", - sources: map[string]model.CatalogSource{ - "source1": model.CatalogSource{Id: "source1", Name: "Test Source"}, + sources: map[string]catalog.Source{ + "source1": {CatalogSource: model.CatalogSource{Id: "source1", Name: "Test Source"}}, }, provider: &mockModelProvider{ models: map[string]*model.CatalogModel{ @@ -1196,7 +1196,7 @@ func TestGetModel(t *testing.T) { func TestGetAllModelArtifacts(t *testing.T) { testCases := []struct { name string - sources map[string]model.CatalogSource + sources map[string]catalog.Source sourceID string modelName string expectedStatus int @@ -1205,8 +1205,8 @@ func TestGetAllModelArtifacts(t *testing.T) { }{ { name: "Existing artifacts for model in source", - sources: map[string]model.CatalogSource{ - "source1": model.CatalogSource{Id: "source1", Name: "Test Source"}, + sources: map[string]catalog.Source{ + "source1": {CatalogSource: model.CatalogSource{Id: "source1", Name: "Test Source"}}, }, provider: &mockModelProvider{ artifacts: map[string][]model.CatalogArtifact{ @@ -1242,8 +1242,8 @@ func TestGetAllModelArtifacts(t *testing.T) { }, { name: "Non-existing source", - sources: map[string]model.CatalogSource{ - "source1": model.CatalogSource{Id: "source1", Name: "Test Source"}, + sources: map[string]catalog.Source{ + "source1": {CatalogSource: model.CatalogSource{Id: "source1", Name: "Test Source"}}, }, provider: &mockModelProvider{ artifacts: map[string][]model.CatalogArtifact{}, @@ -1255,8 +1255,8 @@ func TestGetAllModelArtifacts(t *testing.T) { }, { name: "Existing source, no artifacts for model", - sources: map[string]model.CatalogSource{ - "source1": model.CatalogSource{Id: "source1", Name: "Test Source"}, + sources: map[string]catalog.Source{ + "source1": {CatalogSource: model.CatalogSource{Id: "source1", Name: "Test Source"}}, }, provider: &mockModelProvider{ artifacts: map[string][]model.CatalogArtifact{}, diff --git a/clients/python/Makefile b/clients/python/Makefile index 69c9038ae7..7db5dd70b9 100644 --- a/clients/python/Makefile +++ b/clients/python/Makefile @@ -35,7 +35,7 @@ deploy-test-minio: .PHONY: deploy-local-registry deploy-local-registry: cd ../../ && ./scripts/deploy_local_kind_registry.sh - kubectl port-forward service/distribution-registry-test-service 5001:5001 > /dev/null 2>&1 & echo $$! >> .port-forwards.pid + kubectl port-forward service/distribution-registry-test-service -n local-oci-registry-ns 5001:5001 > /dev/null 2>&1 & echo $$! >> .port-forwards.pid .PHONY: test-e2e test-e2e: deploy-latest-mr deploy-local-registry deploy-test-minio diff --git a/clients/python/poetry.lock b/clients/python/poetry.lock index 1240cef653..de93dea6af 100644 --- a/clients/python/poetry.lock +++ b/clients/python/poetry.lock @@ -1039,15 +1039,15 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.1.5" +version = "1.1.7" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = true python-versions = ">=3.9.0" groups = ["main"] markers = "extra == \"hf\"" files = [ - {file = "huggingface_hub-1.1.5-py3-none-any.whl", hash = "sha256:e88ecc129011f37b868586bbcfae6c56868cae80cd56a79d61575426a3aa0d7d"}, - {file = "huggingface_hub-1.1.5.tar.gz", hash = "sha256:40ba5c9a08792d888fde6088920a0a71ab3cd9d5e6617c81a797c657f1fd9968"}, + {file = "huggingface_hub-1.1.7-py3-none-any.whl", hash = "sha256:f3efa4779f4890e44c957bbbb0f197e6028887ad09f0cf95a21659fa7753605d"}, + {file = "huggingface_hub-1.1.7.tar.gz", hash = "sha256:3c84b6283caca928595f08fd42e9a572f17ec3501dec508c3f2939d94bfbd9d2"}, ] [package.dependencies] @@ -1321,6 +1321,92 @@ files = [ [package.dependencies] six = "*" +[[package]] +name = "librt" +version = "0.6.3" +description = "Mypyc runtime library" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "librt-0.6.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:45660d26569cc22ed30adf583389d8a0d1b468f8b5e518fcf9bfe2cd298f9dd1"}, + {file = "librt-0.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:54f3b2177fb892d47f8016f1087d21654b44f7fc4cf6571c1c6b3ea531ab0fcf"}, + {file = "librt-0.6.3-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:c5b31bed2c2f2fa1fcb4815b75f931121ae210dc89a3d607fb1725f5907f1437"}, + {file = "librt-0.6.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f8ed5053ef9fb08d34f1fd80ff093ccbd1f67f147633a84cf4a7d9b09c0f089"}, + {file = "librt-0.6.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3f0e4bd9bcb0ee34fa3dbedb05570da50b285f49e52c07a241da967840432513"}, + {file = "librt-0.6.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d8f89c8d20dfa648a3f0a56861946eb00e5b00d6b00eea14bc5532b2fcfa8ef1"}, + {file = "librt-0.6.3-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:ecc2c526547eacd20cb9fbba19a5268611dbc70c346499656d6cf30fae328977"}, + {file = "librt-0.6.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fbedeb9b48614d662822ee514567d2d49a8012037fc7b4cd63f282642c2f4b7d"}, + {file = "librt-0.6.3-cp310-cp310-win32.whl", hash = "sha256:0765b0fe0927d189ee14b087cd595ae636bef04992e03fe6dfdaa383866c8a46"}, + {file = "librt-0.6.3-cp310-cp310-win_amd64.whl", hash = "sha256:8c659f9fb8a2f16dc4131b803fa0144c1dadcb3ab24bb7914d01a6da58ae2457"}, + {file = "librt-0.6.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:61348cc488b18d1b1ff9f3e5fcd5ac43ed22d3e13e862489d2267c2337285c08"}, + {file = "librt-0.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:64645b757d617ad5f98c08e07620bc488d4bced9ced91c6279cec418f16056fa"}, + {file = "librt-0.6.3-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:26b8026393920320bb9a811b691d73c5981385d537ffc5b6e22e53f7b65d4122"}, + {file = "librt-0.6.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d998b432ed9ffccc49b820e913c8f327a82026349e9c34fa3690116f6b70770f"}, + {file = "librt-0.6.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e18875e17ef69ba7dfa9623f2f95f3eda6f70b536079ee6d5763ecdfe6cc9040"}, + {file = "librt-0.6.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a218f85081fc3f70cddaed694323a1ad7db5ca028c379c214e3a7c11c0850523"}, + {file = "librt-0.6.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1ef42ff4edd369e84433ce9b188a64df0837f4f69e3d34d3b34d4955c599d03f"}, + {file = "librt-0.6.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0e0f2b79993fec23a685b3e8107ba5f8675eeae286675a216da0b09574fa1e47"}, + {file = "librt-0.6.3-cp311-cp311-win32.whl", hash = "sha256:fd98cacf4e0fabcd4005c452cb8a31750258a85cab9a59fb3559e8078da408d7"}, + {file = "librt-0.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:e17b5b42c8045867ca9d1f54af00cc2275198d38de18545edaa7833d7e9e4ac8"}, + {file = "librt-0.6.3-cp311-cp311-win_arm64.whl", hash = "sha256:87597e3d57ec0120a3e1d857a708f80c02c42ea6b00227c728efbc860f067c45"}, + {file = "librt-0.6.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:74418f718083009108dc9a42c21bf2e4802d49638a1249e13677585fcc9ca176"}, + {file = "librt-0.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:514f3f363d1ebc423357d36222c37e5c8e6674b6eae8d7195ac9a64903722057"}, + {file = "librt-0.6.3-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cf1115207a5049d1f4b7b4b72de0e52f228d6c696803d94843907111cbf80610"}, + {file = "librt-0.6.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ad8ba80cdcea04bea7b78fcd4925bfbf408961e9d8397d2ee5d3ec121e20c08c"}, + {file = "librt-0.6.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4018904c83eab49c814e2494b4e22501a93cdb6c9f9425533fe693c3117126f9"}, + {file = "librt-0.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8983c5c06ac9c990eac5eb97a9f03fe41dc7e9d7993df74d9e8682a1056f596c"}, + {file = "librt-0.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d7769c579663a6f8dbf34878969ac71befa42067ce6bf78e6370bf0d1194997c"}, + {file = "librt-0.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d3c9a07eafdc70556f8c220da4a538e715668c0c63cabcc436a026e4e89950bf"}, + {file = "librt-0.6.3-cp312-cp312-win32.whl", hash = "sha256:38320386a48a15033da295df276aea93a92dfa94a862e06893f75ea1d8bbe89d"}, + {file = "librt-0.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:c0ecf4786ad0404b072196b5df774b1bb23c8aacdcacb6c10b4128bc7b00bd01"}, + {file = "librt-0.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:9f2a6623057989ebc469cd9cc8fe436c40117a0147627568d03f84aef7854c55"}, + {file = "librt-0.6.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e716f9012148a81f02f46a04fc4c663420c6fbfeacfac0b5e128cf43b4413d3"}, + {file = "librt-0.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:669ff2495728009a96339c5ad2612569c6d8be4474e68f3f3ac85d7c3261f5f5"}, + {file = "librt-0.6.3-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:349b6873ebccfc24c9efd244e49da9f8a5c10f60f07575e248921aae2123fc42"}, + {file = "librt-0.6.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c74c26736008481c9f6d0adf1aedb5a52aff7361fea98276d1f965c0256ee70"}, + {file = "librt-0.6.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:408a36ddc75e91918cb15b03460bdc8a015885025d67e68c6f78f08c3a88f522"}, + {file = "librt-0.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e61ab234624c9ffca0248a707feffe6fac2343758a36725d8eb8a6efef0f8c30"}, + {file = "librt-0.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:324462fe7e3896d592b967196512491ec60ca6e49c446fe59f40743d08c97917"}, + {file = "librt-0.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:36b2ec8c15030002c7f688b4863e7be42820d7c62d9c6eece3db54a2400f0530"}, + {file = "librt-0.6.3-cp313-cp313-win32.whl", hash = "sha256:25b1b60cb059471c0c0c803e07d0dfdc79e41a0a122f288b819219ed162672a3"}, + {file = "librt-0.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:10a95ad074e2a98c9e4abc7f5b7d40e5ecbfa84c04c6ab8a70fabf59bd429b88"}, + {file = "librt-0.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:17000df14f552e86877d67e4ab7966912224efc9368e998c96a6974a8d609bf9"}, + {file = "librt-0.6.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8e695f25d1a425ad7a272902af8ab8c8d66c1998b177e4b5f5e7b4e215d0c88a"}, + {file = "librt-0.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3e84a4121a7ae360ca4da436548a9c1ca8ca134a5ced76c893cc5944426164bd"}, + {file = "librt-0.6.3-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:05f385a414de3f950886ea0aad8f109650d4b712cf9cc14cc17f5f62a9ab240b"}, + {file = "librt-0.6.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:36a8e337461150b05ca2c7bdedb9e591dfc262c5230422cea398e89d0c746cdc"}, + {file = "librt-0.6.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dcbe48f6a03979384f27086484dc2a14959be1613cb173458bd58f714f2c48f3"}, + {file = "librt-0.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4bca9e4c260233fba37b15c4ec2f78aa99c1a79fbf902d19dd4a763c5c3fb751"}, + {file = "librt-0.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:760c25ed6ac968e24803eb5f7deb17ce026902d39865e83036bacbf5cf242aa8"}, + {file = "librt-0.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4aa4a93a353ccff20df6e34fa855ae8fd788832c88f40a9070e3ddd3356a9f0e"}, + {file = "librt-0.6.3-cp314-cp314-win32.whl", hash = "sha256:cb92741c2b4ea63c09609b064b26f7f5d9032b61ae222558c55832ec3ad0bcaf"}, + {file = "librt-0.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:fdcd095b1b812d756fa5452aca93b962cf620694c0cadb192cec2bb77dcca9a2"}, + {file = "librt-0.6.3-cp314-cp314-win_arm64.whl", hash = "sha256:822ca79e28720a76a935c228d37da6579edef048a17cd98d406a2484d10eda78"}, + {file = "librt-0.6.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:078cd77064d1640cb7b0650871a772956066174d92c8aeda188a489b58495179"}, + {file = "librt-0.6.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5cc22f7f5c0cc50ed69f4b15b9c51d602aabc4500b433aaa2ddd29e578f452f7"}, + {file = "librt-0.6.3-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:14b345eb7afb61b9fdcdfda6738946bd11b8e0f6be258666b0646af3b9bb5916"}, + {file = "librt-0.6.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d46aa46aa29b067f0b8b84f448fd9719aaf5f4c621cc279164d76a9dc9ab3e8"}, + {file = "librt-0.6.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1b51ba7d9d5d9001494769eca8c0988adce25d0a970c3ba3f2eb9df9d08036fc"}, + {file = "librt-0.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ced0925a18fddcff289ef54386b2fc230c5af3c83b11558571124bfc485b8c07"}, + {file = "librt-0.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:6bac97e51f66da2ca012adddbe9fd656b17f7368d439de30898f24b39512f40f"}, + {file = "librt-0.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b2922a0e8fa97395553c304edc3bd36168d8eeec26b92478e292e5d4445c1ef0"}, + {file = "librt-0.6.3-cp314-cp314t-win32.whl", hash = "sha256:f33462b19503ba68d80dac8a1354402675849259fb3ebf53b67de86421735a3a"}, + {file = "librt-0.6.3-cp314-cp314t-win_amd64.whl", hash = "sha256:04f8ce401d4f6380cfc42af0f4e67342bf34c820dae01343f58f472dbac75dcf"}, + {file = "librt-0.6.3-cp314-cp314t-win_arm64.whl", hash = "sha256:afb39550205cc5e5c935762c6bf6a2bb34f7d21a68eadb25e2db7bf3593fecc0"}, + {file = "librt-0.6.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:09262cb2445b6f15d09141af20b95bb7030c6f13b00e876ad8fdd1a9045d6aa5"}, + {file = "librt-0.6.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:57705e8eec76c5b77130d729c0f70190a9773366c555c5457c51eace80afd873"}, + {file = "librt-0.6.3-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3ac2a7835434b31def8ed5355dd9b895bbf41642d61967522646d1d8b9681106"}, + {file = "librt-0.6.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:71f0a5918aebbea1e7db2179a8fe87e8a8732340d9e8b8107401fb407eda446e"}, + {file = "librt-0.6.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aa346e202e6e1ebc01fe1c69509cffe486425884b96cb9ce155c99da1ecbe0e9"}, + {file = "librt-0.6.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:92267f865c7bbd12327a0d394666948b9bf4b51308b52947c0cc453bfa812f5d"}, + {file = "librt-0.6.3-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:86605d5bac340beb030cbc35859325982a79047ebdfba1e553719c7126a2389d"}, + {file = "librt-0.6.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:98e4bbecbef8d2a60ecf731d735602feee5ac0b32117dbbc765e28b054bac912"}, + {file = "librt-0.6.3-cp39-cp39-win32.whl", hash = "sha256:3caa0634c02d5ff0b2ae4a28052e0d8c5f20d497623dc13f629bd4a9e2a6efad"}, + {file = "librt-0.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:b47395091e7e0ece1e6ebac9b98bf0c9084d1e3d3b2739aa566be7e56e3f7bf2"}, + {file = "librt-0.6.3.tar.gz", hash = "sha256:c724a884e642aa2bbad52bb0203ea40406ad742368a5f90da1b220e970384aae"}, +] + [[package]] name = "linkify-it-py" version = "2.0.2" @@ -1646,53 +1732,54 @@ files = [ [[package]] name = "mypy" -version = "1.18.2" +version = "1.19.0" description = "Optional static typing for Python" optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "mypy-1.18.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c1eab0cf6294dafe397c261a75f96dc2c31bffe3b944faa24db5def4e2b0f77c"}, - {file = "mypy-1.18.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7a780ca61fc239e4865968ebc5240bb3bf610ef59ac398de9a7421b54e4a207e"}, - {file = "mypy-1.18.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:448acd386266989ef11662ce3c8011fd2a7b632e0ec7d61a98edd8e27472225b"}, - {file = "mypy-1.18.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f9e171c465ad3901dc652643ee4bffa8e9fef4d7d0eece23b428908c77a76a66"}, - {file = "mypy-1.18.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:592ec214750bc00741af1f80cbf96b5013d81486b7bb24cb052382c19e40b428"}, - {file = "mypy-1.18.2-cp310-cp310-win_amd64.whl", hash = "sha256:7fb95f97199ea11769ebe3638c29b550b5221e997c63b14ef93d2e971606ebed"}, - {file = "mypy-1.18.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:807d9315ab9d464125aa9fcf6d84fde6e1dc67da0b6f80e7405506b8ac72bc7f"}, - {file = "mypy-1.18.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:776bb00de1778caf4db739c6e83919c1d85a448f71979b6a0edd774ea8399341"}, - {file = "mypy-1.18.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1379451880512ffce14505493bd9fe469e0697543717298242574882cf8cdb8d"}, - {file = "mypy-1.18.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1331eb7fd110d60c24999893320967594ff84c38ac6d19e0a76c5fd809a84c86"}, - {file = "mypy-1.18.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3ca30b50a51e7ba93b00422e486cbb124f1c56a535e20eff7b2d6ab72b3b2e37"}, - {file = "mypy-1.18.2-cp311-cp311-win_amd64.whl", hash = "sha256:664dc726e67fa54e14536f6e1224bcfce1d9e5ac02426d2326e2bb4e081d1ce8"}, - {file = "mypy-1.18.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:33eca32dd124b29400c31d7cf784e795b050ace0e1f91b8dc035672725617e34"}, - {file = "mypy-1.18.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a3c47adf30d65e89b2dcd2fa32f3aeb5e94ca970d2c15fcb25e297871c8e4764"}, - {file = "mypy-1.18.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d6c838e831a062f5f29d11c9057c6009f60cb294fea33a98422688181fe2893"}, - {file = "mypy-1.18.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01199871b6110a2ce984bde85acd481232d17413868c9807e95c1b0739a58914"}, - {file = "mypy-1.18.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a2afc0fa0b0e91b4599ddfe0f91e2c26c2b5a5ab263737e998d6817874c5f7c8"}, - {file = "mypy-1.18.2-cp312-cp312-win_amd64.whl", hash = "sha256:d8068d0afe682c7c4897c0f7ce84ea77f6de953262b12d07038f4d296d547074"}, - {file = "mypy-1.18.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:07b8b0f580ca6d289e69209ec9d3911b4a26e5abfde32228a288eb79df129fcc"}, - {file = "mypy-1.18.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ed4482847168439651d3feee5833ccedbf6657e964572706a2adb1f7fa4dfe2e"}, - {file = "mypy-1.18.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3ad2afadd1e9fea5cf99a45a822346971ede8685cc581ed9cd4d42eaf940986"}, - {file = "mypy-1.18.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a431a6f1ef14cf8c144c6b14793a23ec4eae3db28277c358136e79d7d062f62d"}, - {file = "mypy-1.18.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7ab28cc197f1dd77a67e1c6f35cd1f8e8b73ed2217e4fc005f9e6a504e46e7ba"}, - {file = "mypy-1.18.2-cp313-cp313-win_amd64.whl", hash = "sha256:0e2785a84b34a72ba55fb5daf079a1003a34c05b22238da94fcae2bbe46f3544"}, - {file = "mypy-1.18.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:62f0e1e988ad41c2a110edde6c398383a889d95b36b3e60bcf155f5164c4fdce"}, - {file = "mypy-1.18.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8795a039bab805ff0c1dfdb8cd3344642c2b99b8e439d057aba30850b8d3423d"}, - {file = "mypy-1.18.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6ca1e64b24a700ab5ce10133f7ccd956a04715463d30498e64ea8715236f9c9c"}, - {file = "mypy-1.18.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d924eef3795cc89fecf6bedc6ed32b33ac13e8321344f6ddbf8ee89f706c05cb"}, - {file = "mypy-1.18.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20c02215a080e3a2be3aa50506c67242df1c151eaba0dcbc1e4e557922a26075"}, - {file = "mypy-1.18.2-cp314-cp314-win_amd64.whl", hash = "sha256:749b5f83198f1ca64345603118a6f01a4e99ad4bf9d103ddc5a3200cc4614adf"}, - {file = "mypy-1.18.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:25a9c8fb67b00599f839cf472713f54249a62efd53a54b565eb61956a7e3296b"}, - {file = "mypy-1.18.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c2b9c7e284ee20e7598d6f42e13ca40b4928e6957ed6813d1ab6348aa3f47133"}, - {file = "mypy-1.18.2-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d6985ed057513e344e43a26cc1cd815c7a94602fb6a3130a34798625bc2f07b6"}, - {file = "mypy-1.18.2-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22f27105f1525ec024b5c630c0b9f36d5c1cc4d447d61fe51ff4bd60633f47ac"}, - {file = "mypy-1.18.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:030c52d0ea8144e721e49b1f68391e39553d7451f0c3f8a7565b59e19fcb608b"}, - {file = "mypy-1.18.2-cp39-cp39-win_amd64.whl", hash = "sha256:aa5e07ac1a60a253445797e42b8b2963c9675563a94f11291ab40718b016a7a0"}, - {file = "mypy-1.18.2-py3-none-any.whl", hash = "sha256:22a1748707dd62b58d2ae53562ffc4d7f8bcc727e8ac7cbc69c053ddc874d47e"}, - {file = "mypy-1.18.2.tar.gz", hash = "sha256:06a398102a5f203d7477b2923dda3634c36727fa5c237d8f859ef90c42a9924b"}, -] - -[package.dependencies] + {file = "mypy-1.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6148ede033982a8c5ca1143de34c71836a09f105068aaa8b7d5edab2b053e6c8"}, + {file = "mypy-1.19.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a9ac09e52bb0f7fb912f5d2a783345c72441a08ef56ce3e17c1752af36340a39"}, + {file = "mypy-1.19.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:11f7254c15ab3f8ed68f8e8f5cbe88757848df793e31c36aaa4d4f9783fd08ab"}, + {file = "mypy-1.19.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318ba74f75899b0e78b847d8c50821e4c9637c79d9a59680fc1259f29338cb3e"}, + {file = "mypy-1.19.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cf7d84f497f78b682edd407f14a7b6e1a2212b433eedb054e2081380b7395aa3"}, + {file = "mypy-1.19.0-cp310-cp310-win_amd64.whl", hash = "sha256:c3385246593ac2b97f155a0e9639be906e73534630f663747c71908dfbf26134"}, + {file = "mypy-1.19.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a31e4c28e8ddb042c84c5e977e28a21195d086aaffaf08b016b78e19c9ef8106"}, + {file = "mypy-1.19.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:34ec1ac66d31644f194b7c163d7f8b8434f1b49719d403a5d26c87fff7e913f7"}, + {file = "mypy-1.19.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb64b0ba5980466a0f3f9990d1c582bcab8db12e29815ecb57f1408d99b4bff7"}, + {file = "mypy-1.19.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:120cffe120cca5c23c03c77f84abc0c14c5d2e03736f6c312480020082f1994b"}, + {file = "mypy-1.19.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7a500ab5c444268a70565e374fc803972bfd1f09545b13418a5174e29883dab7"}, + {file = "mypy-1.19.0-cp311-cp311-win_amd64.whl", hash = "sha256:c14a98bc63fd867530e8ec82f217dae29d0550c86e70debc9667fff1ec83284e"}, + {file = "mypy-1.19.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0fb3115cb8fa7c5f887c8a8d81ccdcb94cff334684980d847e5a62e926910e1d"}, + {file = "mypy-1.19.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3e19e3b897562276bb331074d64c076dbdd3e79213f36eed4e592272dabd760"}, + {file = "mypy-1.19.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b9d491295825182fba01b6ffe2c6fe4e5a49dbf4e2bb4d1217b6ced3b4797bc6"}, + {file = "mypy-1.19.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6016c52ab209919b46169651b362068f632efcd5eb8ef9d1735f6f86da7853b2"}, + {file = "mypy-1.19.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f188dcf16483b3e59f9278c4ed939ec0254aa8a60e8fc100648d9ab5ee95a431"}, + {file = "mypy-1.19.0-cp312-cp312-win_amd64.whl", hash = "sha256:0e3c3d1e1d62e678c339e7ade72746a9e0325de42cd2cccc51616c7b2ed1a018"}, + {file = "mypy-1.19.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7686ed65dbabd24d20066f3115018d2dce030d8fa9db01aa9f0a59b6813e9f9e"}, + {file = "mypy-1.19.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fd4a985b2e32f23bead72e2fb4bbe5d6aceee176be471243bd831d5b2644672d"}, + {file = "mypy-1.19.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fc51a5b864f73a3a182584b1ac75c404396a17eced54341629d8bdcb644a5bba"}, + {file = "mypy-1.19.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:37af5166f9475872034b56c5efdcf65ee25394e9e1d172907b84577120714364"}, + {file = "mypy-1.19.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:510c014b722308c9bd377993bcbf9a07d7e0692e5fa8fc70e639c1eb19fc6bee"}, + {file = "mypy-1.19.0-cp313-cp313-win_amd64.whl", hash = "sha256:cabbee74f29aa9cd3b444ec2f1e4fa5a9d0d746ce7567a6a609e224429781f53"}, + {file = "mypy-1.19.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:f2e36bed3c6d9b5f35d28b63ca4b727cb0228e480826ffc8953d1892ddc8999d"}, + {file = "mypy-1.19.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a18d8abdda14035c5718acb748faec09571432811af129bf0d9e7b2d6699bf18"}, + {file = "mypy-1.19.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f75e60aca3723a23511948539b0d7ed514dda194bc3755eae0bfc7a6b4887aa7"}, + {file = "mypy-1.19.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f44f2ae3c58421ee05fe609160343c25f70e3967f6e32792b5a78006a9d850f"}, + {file = "mypy-1.19.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:63ea6a00e4bd6822adbfc75b02ab3653a17c02c4347f5bb0cf1d5b9df3a05835"}, + {file = "mypy-1.19.0-cp314-cp314-win_amd64.whl", hash = "sha256:3ad925b14a0bb99821ff6f734553294aa6a3440a8cb082fe1f5b84dfb662afb1"}, + {file = "mypy-1.19.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0dde5cb375cb94deff0d4b548b993bec52859d1651e073d63a1386d392a95495"}, + {file = "mypy-1.19.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1cf9c59398db1c68a134b0b5354a09a1e124523f00bacd68e553b8bd16ff3299"}, + {file = "mypy-1.19.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3210d87b30e6af9c8faed61be2642fcbe60ef77cec64fa1ef810a630a4cf671c"}, + {file = "mypy-1.19.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2c1101ab41d01303103ab6ef82cbbfedb81c1a060c868fa7cc013d573d37ab5"}, + {file = "mypy-1.19.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:0ea4fd21bb48f0da49e6d3b37ef6bd7e8228b9fe41bbf4d80d9364d11adbd43c"}, + {file = "mypy-1.19.0-cp39-cp39-win_amd64.whl", hash = "sha256:16f76ff3f3fd8137aadf593cb4607d82634fca675e8211ad75c43d86033ee6c6"}, + {file = "mypy-1.19.0-py3-none-any.whl", hash = "sha256:0c01c99d626380752e527d5ce8e69ffbba2046eb8a060db0329690849cf9b6f9"}, + {file = "mypy-1.19.0.tar.gz", hash = "sha256:f6b874ca77f733222641e5c46e4711648c4037ea13646fd0cdc814c2eaec2528"}, +] + +[package.dependencies] +librt = ">=0.6.2" mypy_extensions = ">=1.0.0" pathspec = ">=0.9.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} @@ -1759,21 +1846,24 @@ files = [ [[package]] name = "olot" -version = "0.1.11" +version = "0.1.13" description = "oci layers on top" optional = true python-versions = "<4.0,>=3.9" groups = ["main"] markers = "extra == \"olot\"" files = [ - {file = "olot-0.1.11-py3-none-any.whl", hash = "sha256:a1e6949bb3c559aa4829f854ad20ebe3d72c73815369403c62e2529e5c5ca3bd"}, - {file = "olot-0.1.11.tar.gz", hash = "sha256:2e3be9fa871e5e53ef551ab9906197f7a9f795c6e21739d163041bef40571416"}, + {file = "olot-0.1.13-py3-none-any.whl", hash = "sha256:fa5dff7f5766c62e49fc49983546b6e8050268b4733f9dd2fb305a402f6e939a"}, + {file = "olot-0.1.13.tar.gz", hash = "sha256:a553c0e1798156ec13257689d483993363303de8573cc91465c48edb1c66b70f"}, ] [package.dependencies] click = ">=8.1.7,<9.0.0" pydantic = ">=2.10.3,<3.0.0" +[package.extras] +modelcar-base-image = ["modelcar-base-image (>=0.1.0,<0.2.0)"] + [[package]] name = "packaging" version = "23.2" @@ -2664,31 +2754,31 @@ files = [ [[package]] name = "ruff" -version = "0.14.6" +version = "0.14.7" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "ruff-0.14.6-py3-none-linux_armv6l.whl", hash = "sha256:d724ac2f1c240dbd01a2ae98db5d1d9a5e1d9e96eba999d1c48e30062df578a3"}, - {file = "ruff-0.14.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9f7539ea257aa4d07b7ce87aed580e485c40143f2473ff2f2b75aee003186004"}, - {file = "ruff-0.14.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7f6007e55b90a2a7e93083ba48a9f23c3158c433591c33ee2e99a49b889c6332"}, - {file = "ruff-0.14.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a8e7b9d73d8728b68f632aa8e824ef041d068d231d8dbc7808532d3629a6bef"}, - {file = "ruff-0.14.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d50d45d4553a3ebcbd33e7c5e0fe6ca4aafd9a9122492de357205c2c48f00775"}, - {file = "ruff-0.14.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:118548dd121f8a21bfa8ab2c5b80e5b4aed67ead4b7567790962554f38e598ce"}, - {file = "ruff-0.14.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:57256efafbfefcb8748df9d1d766062f62b20150691021f8ab79e2d919f7c11f"}, - {file = "ruff-0.14.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ff18134841e5c68f8e5df1999a64429a02d5549036b394fafbe410f886e1989d"}, - {file = "ruff-0.14.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:29c4b7ec1e66a105d5c27bd57fa93203637d66a26d10ca9809dc7fc18ec58440"}, - {file = "ruff-0.14.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:167843a6f78680746d7e226f255d920aeed5e4ad9c03258094a2d49d3028b105"}, - {file = "ruff-0.14.6-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:16a33af621c9c523b1ae006b1b99b159bf5ac7e4b1f20b85b2572455018e0821"}, - {file = "ruff-0.14.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:1432ab6e1ae2dc565a7eea707d3b03a0c234ef401482a6f1621bc1f427c2ff55"}, - {file = "ruff-0.14.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:4c55cfbbe7abb61eb914bfd20683d14cdfb38a6d56c6c66efa55ec6570ee4e71"}, - {file = "ruff-0.14.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:efea3c0f21901a685fff4befda6d61a1bf4cb43de16da87e8226a281d614350b"}, - {file = "ruff-0.14.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:344d97172576d75dc6afc0e9243376dbe1668559c72de1864439c4fc95f78185"}, - {file = "ruff-0.14.6-py3-none-win32.whl", hash = "sha256:00169c0c8b85396516fdd9ce3446c7ca20c2a8f90a77aa945ba6b8f2bfe99e85"}, - {file = "ruff-0.14.6-py3-none-win_amd64.whl", hash = "sha256:390e6480c5e3659f8a4c8d6a0373027820419ac14fa0d2713bd8e6c3e125b8b9"}, - {file = "ruff-0.14.6-py3-none-win_arm64.whl", hash = "sha256:d43c81fbeae52cfa8728d8766bbf46ee4298c888072105815b392da70ca836b2"}, - {file = "ruff-0.14.6.tar.gz", hash = "sha256:6f0c742ca6a7783a736b867a263b9a7a80a45ce9bee391eeda296895f1b4e1cc"}, + {file = "ruff-0.14.7-py3-none-linux_armv6l.whl", hash = "sha256:b9d5cb5a176c7236892ad7224bc1e63902e4842c460a0b5210701b13e3de4fca"}, + {file = "ruff-0.14.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:3f64fe375aefaf36ca7d7250292141e39b4cea8250427482ae779a2aa5d90015"}, + {file = "ruff-0.14.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:93e83bd3a9e1a3bda64cb771c0d47cda0e0d148165013ae2d3554d718632d554"}, + {file = "ruff-0.14.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3838948e3facc59a6070795de2ae16e5786861850f78d5914a03f12659e88f94"}, + {file = "ruff-0.14.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:24c8487194d38b6d71cd0fd17a5b6715cda29f59baca1defe1e3a03240f851d1"}, + {file = "ruff-0.14.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:79c73db6833f058a4be8ffe4a0913b6d4ad41f6324745179bd2aa09275b01d0b"}, + {file = "ruff-0.14.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:12eb7014fccff10fc62d15c79d8a6be4d0c2d60fe3f8e4d169a0d2def75f5dad"}, + {file = "ruff-0.14.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c623bbdc902de7ff715a93fa3bb377a4e42dd696937bf95669118773dbf0c50"}, + {file = "ruff-0.14.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f53accc02ed2d200fa621593cdb3c1ae06aa9b2c3cae70bc96f72f0000ae97a9"}, + {file = "ruff-0.14.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:281f0e61a23fcdcffca210591f0f53aafaa15f9025b5b3f9706879aaa8683bc4"}, + {file = "ruff-0.14.7-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:dbbaa5e14148965b91cb090236931182ee522a5fac9bc5575bafc5c07b9f9682"}, + {file = "ruff-0.14.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:1464b6e54880c0fe2f2d6eaefb6db15373331414eddf89d6b903767ae2458143"}, + {file = "ruff-0.14.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:f217ed871e4621ea6128460df57b19ce0580606c23aeab50f5de425d05226784"}, + {file = "ruff-0.14.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:6be02e849440ed3602d2eb478ff7ff07d53e3758f7948a2a598829660988619e"}, + {file = "ruff-0.14.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:19a0f116ee5e2b468dfe80c41c84e2bbd6b74f7b719bee86c2ecde0a34563bcc"}, + {file = "ruff-0.14.7-py3-none-win32.whl", hash = "sha256:e33052c9199b347c8937937163b9b149ef6ab2e4bb37b042e593da2e6f6cccfa"}, + {file = "ruff-0.14.7-py3-none-win_amd64.whl", hash = "sha256:e17a20ad0d3fad47a326d773a042b924d3ac31c6ca6deb6c72e9e6b5f661a7c6"}, + {file = "ruff-0.14.7-py3-none-win_arm64.whl", hash = "sha256:be4d653d3bea1b19742fcc6502354e32f65cd61ff2fbdb365803ef2c2aec6228"}, + {file = "ruff-0.14.7.tar.gz", hash = "sha256:3417deb75d23bd14a722b57b0a1435561db65f0ad97435b4cf9f85ffcef34ae5"}, ] [[package]] @@ -3157,14 +3247,14 @@ standard = ["rich (>=10.11.0)", "shellingham (>=1.3.0)"] [[package]] name = "types-python-dateutil" -version = "2.9.0.20251108" +version = "2.9.0.20251115" description = "Typing stubs for python-dateutil" optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "types_python_dateutil-2.9.0.20251108-py3-none-any.whl", hash = "sha256:a4a537f0ea7126f8ccc2763eec9aa31ac8609e3c8e530eb2ddc5ee234b3cd764"}, - {file = "types_python_dateutil-2.9.0.20251108.tar.gz", hash = "sha256:d8a6687e197f2fa71779ce36176c666841f811368710ab8d274b876424ebfcaa"}, + {file = "types_python_dateutil-2.9.0.20251115-py3-none-any.whl", hash = "sha256:9cf9c1c582019753b8639a081deefd7e044b9fa36bd8217f565c6c4e36ee0624"}, + {file = "types_python_dateutil-2.9.0.20251115.tar.gz", hash = "sha256:8a47f2c3920f52a994056b8786309b43143faa5a64d4cbb2722d6addabdf1a58"}, ] [[package]] @@ -3510,18 +3600,18 @@ files = [ [[package]] name = "werkzeug" -version = "3.1.3" +version = "3.1.4" description = "The comprehensive WSGI web application library." optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e"}, - {file = "werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746"}, + {file = "werkzeug-3.1.4-py3-none-any.whl", hash = "sha256:2ad50fb9ed09cc3af22c54698351027ace879a0b60a3b5edf5730b2f7d876905"}, + {file = "werkzeug-3.1.4.tar.gz", hash = "sha256:cd3cd98b1b92dc3b7b3995038826c68097dcb16f9baa63abe35f20eafeb9fe5e"}, ] [package.dependencies] -MarkupSafe = ">=2.1.1" +markupsafe = ">=2.1.1" [package.extras] watchdog = ["watchdog (>=2.3)"] diff --git a/clients/ui/api/openapi/mod-arch.yaml b/clients/ui/api/openapi/mod-arch.yaml index dec29d1d92..541cad5dd8 100644 --- a/clients/ui/api/openapi/mod-arch.yaml +++ b/clients/ui/api/openapi/mod-arch.yaml @@ -975,6 +975,7 @@ paths: tags: - ModelCatalogService parameters: + - $ref: "#/components/parameters/kubeflowUserId" - name: statusFilter description: |- Filter the response to show specific model statuses. diff --git a/clients/ui/bff/internal/api/catalog_source_preview_handler.go b/clients/ui/bff/internal/api/catalog_source_preview_handler.go index 3d2e5c6c1b..82f6576d4f 100644 --- a/clients/ui/bff/internal/api/catalog_source_preview_handler.go +++ b/clients/ui/bff/internal/api/catalog_source_preview_handler.go @@ -31,6 +31,7 @@ func (app *App) CreateCatalogSourcePreviewHandler(w http.ResponseWriter, r *http if err != nil { app.serverErrorResponse(w, r, err) + return } catalogSourcePreview := CatalogSourcePreviewEnvelope{ diff --git a/clients/ui/bff/internal/api/model_catalog_settings_handler.go b/clients/ui/bff/internal/api/model_catalog_settings_handler.go index f4ac218bf9..90a07da99d 100644 --- a/clients/ui/bff/internal/api/model_catalog_settings_handler.go +++ b/clients/ui/bff/internal/api/model_catalog_settings_handler.go @@ -4,9 +4,10 @@ import ( "encoding/json" "errors" "fmt" - "github.com/kubeflow/model-registry/ui/bff/internal/mocks" "net/http" + "github.com/kubeflow/model-registry/ui/bff/internal/mocks" + "github.com/julienschmidt/httprouter" "github.com/kubeflow/model-registry/ui/bff/internal/constants" "github.com/kubeflow/model-registry/ui/bff/internal/models" @@ -59,7 +60,7 @@ func (app *App) GetCatalogSourceConfigHandler(w http.ResponseWriter, r *http.Req catalogSourceId := ps.ByName(CatalogSourceId) // TODO ppadti write the real implementation here - catalogSourceConfig := mocks.CreateSampleCatalogSource(catalogSourceId, "catalog-source-1", "yaml") + catalogSourceConfig := mocks.CreateSampleCatalogSource(catalogSourceId, "catalog-source-1", "yaml", true) modelCatalogSource := ModelCatalogSettingsSourceConfigEnvelope{ Data: &catalogSourceConfig, @@ -92,7 +93,7 @@ func (app *App) CreateCatalogSourceConfigHandler(w http.ResponseWriter, r *http. var sourceId = envelope.Data.Id var sourceType = envelope.Data.Type // TODO ppadti write the real implementation here - newCatalogSource := mocks.CreateSampleCatalogSource(sourceId, sourceName, sourceType) + newCatalogSource := mocks.CreateSampleCatalogSource(sourceId, sourceName, sourceType, true) modelCatalogSource := ModelCatalogSettingsSourceConfigEnvelope{ Data: &newCatalogSource, @@ -124,7 +125,7 @@ func (app *App) UpdateCatalogSourceConfigHandler(w http.ResponseWriter, r *http. catalogSourceId := envelope.Data.Id // TODO ppadti write the real implementation here - newCatalogSource := mocks.CreateSampleCatalogSource(catalogSourceId, "Updated Catalog", "yaml") + newCatalogSource := mocks.CreateSampleCatalogSource(catalogSourceId, "Updated Catalog", "yaml", true) modelCatalogSource := ModelCatalogSettingsSourceConfigEnvelope{ Data: &newCatalogSource, @@ -149,7 +150,7 @@ func (app *App) DeleteCatalogSourceConfigHandler(w http.ResponseWriter, r *http. // TODO ppadti write the real implementation here catalogSourceId := ps.ByName(CatalogSourceId) - deletedCatalogSource := mocks.CreateSampleCatalogSource(catalogSourceId, "Updated Catalog", "yaml") + deletedCatalogSource := mocks.CreateSampleCatalogSource(catalogSourceId, "Updated Catalog", "yaml", true) modelCatalogSource := ModelCatalogSettingsSourceConfigEnvelope{ Data: &deletedCatalogSource, diff --git a/clients/ui/bff/internal/integrations/kubernetes/k8mocks/base_testenv.go b/clients/ui/bff/internal/integrations/kubernetes/k8mocks/base_testenv.go index d7af1e514b..890a504e63 100644 --- a/clients/ui/bff/internal/integrations/kubernetes/k8mocks/base_testenv.go +++ b/clients/ui/bff/internal/integrations/kubernetes/k8mocks/base_testenv.go @@ -163,7 +163,16 @@ func setupMock(mockK8sClient kubernetes.Interface, ctx context.Context) error { return fmt.Errorf("failed to set up group access to namespace: %w", err) } - //TODO ppadti: Add more mock setup as needed for other namespaces + err = createModelCatalogDefaultSourcesConfigMap(mockK8sClient, ctx, "bella-namespace") + if err != nil { + return err + } + + err = createModelCatalogSourcesConfigMap(mockK8sClient, ctx, "bella-namespace") + if err != nil { + return err + } + err = createModelCatalogDefaultSourcesConfigMap(mockK8sClient, ctx, "kubeflow") if err != nil { return err diff --git a/clients/ui/bff/internal/mocks/static_data_mock.go b/clients/ui/bff/internal/mocks/static_data_mock.go index 8ea722f266..caf0cdf18e 100644 --- a/clients/ui/bff/internal/mocks/static_data_mock.go +++ b/clients/ui/bff/internal/mocks/static_data_mock.go @@ -810,7 +810,7 @@ func GetCatalogSourceMocks() []models.CatalogSource { disabledStatus := "disabled" invalidCredentialError := "The provided API key is invalid or has expired. Please update your credentials." - invalidOrgError := "The specified organization 'invalid-org' does not exist or you don't have access to it." + invalidOrgError := "The specified organization 'invalid-org' does not exist or you don't have access to it. Please verify the organization name and ensure you have the necessary permissions to access models from this organization." return []models.CatalogSource{ { @@ -1320,14 +1320,6 @@ func GetFilterOptionMocks() map[string]models.FilterOption { }, } - // String type filter for use cases - filterOptions["use_case"] = models.FilterOption{ - Type: FilterOptionTypeString, - Values: []interface{}{ - "chatbot", "code_fixing", "long_rag", "rag", - }, - } - filterOptions["ttft_mean"] = models.FilterOption{ Type: FilterOptionTypeNumber, Range: &models.FilterRange{ @@ -1347,14 +1339,14 @@ func GetFilterOptionsListMock() models.FilterOptionsList { } } -func CreateSampleCatalogSource(id string, name string, catalogType string) models.CatalogSourceConfig { - defaultCatalog := id == "catalog-1" +func CreateSampleCatalogSource(id string, name string, catalogType string, enabled bool) models.CatalogSourceConfig { + defaultCatalog := id == "sample-source" sourceConfig := models.CatalogSourceConfig{ Name: name, Id: id, Type: catalogType, - Enabled: BoolPtr(true), + Enabled: &enabled, Labels: []string{"source-1"}, IsDefault: &defaultCatalog, } @@ -1368,7 +1360,12 @@ func CreateSampleCatalogSource(id string, name string, catalogType string) model case "yaml": sourceConfig.Yaml = stringToPointer("models:\n - name: model1") case "huggingface": - sourceConfig.AllowedOrganization = stringToPointer("org1") + // Use different organizations for the failed sources + if id == "adminModel2" { + sourceConfig.AllowedOrganization = stringToPointer("invalid-org") + } else { + sourceConfig.AllowedOrganization = stringToPointer("org1") + } sourceConfig.ApiKey = stringToPointer("apikey") } @@ -1380,10 +1377,14 @@ func BoolPtr(b bool) *bool { } func GetCatalogSourceConfigsMocks() []models.CatalogSourceConfig { + // Match IDs with catalog sources to show proper statuses return []models.CatalogSourceConfig{ - CreateSampleCatalogSource("catalog-1", "Default Catalog", "yaml"), - CreateSampleCatalogSource("catalog-2", "HuggingFace Catalog", "huggingface"), - CreateSampleCatalogSource("catalog-3", "Custom Catalog", "yaml"), + CreateSampleCatalogSource("sample-source", "Sample mocked source", "yaml", true), + CreateSampleCatalogSource("huggingface", "Hugging Face", "huggingface", false), + CreateSampleCatalogSource("adminModel1", "Admin model 1", "huggingface", true), + CreateSampleCatalogSource("adminModel2", "Admin model 2", "huggingface", true), + CreateSampleCatalogSource("dora", "Dora source", "yaml", true), + CreateSampleCatalogSource("catalog-4", "Custom Catalog 2", "yaml", false), } } diff --git a/clients/ui/frontend/package-lock.json b/clients/ui/frontend/package-lock.json index 3e5b84cf44..e5b20b1c2f 100644 --- a/clients/ui/frontend/package-lock.json +++ b/clients/ui/frontend/package-lock.json @@ -20,9 +20,9 @@ "classnames": "^2.2.6", "dompurify": "^3.2.4", "lodash-es": "^4.17.15", - "mod-arch-core": "~1.1.5", - "mod-arch-kubeflow": "~1.1.5", - "mod-arch-shared": "~1.1.5", + "mod-arch-core": "~1.2.2", + "mod-arch-kubeflow": "~1.2.2", + "mod-arch-shared": "~1.2.2", "react": "^18", "react-dom": "^18", "react-markdown": "^10.1.0", @@ -20152,9 +20152,9 @@ } }, "node_modules/mod-arch-core": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/mod-arch-core/-/mod-arch-core-1.1.5.tgz", - "integrity": "sha512-63O2yFN7EmEFpGNr9p+ZNNz/l4vNRpt3jmy78ltzicGaexl5n4c0VZNAUNOvCqQZR11VeQg/quLsTq336C1m6Q==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/mod-arch-core/-/mod-arch-core-1.2.2.tgz", + "integrity": "sha512-i9Y669kDcvCeFHAH7W1DENldi9TeGyFuWdJAXUvy1kFcSJFzatA4sP+aKxH1TAAcnKVsxuHm4SqFPSj0eToIwA==", "license": "Apache-2.0", "dependencies": { "lodash-es": "^4.17.15", @@ -20185,9 +20185,9 @@ } }, "node_modules/mod-arch-kubeflow": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/mod-arch-kubeflow/-/mod-arch-kubeflow-1.1.5.tgz", - "integrity": "sha512-JdTM11Mx9QcqY67YlzyYlF/wmYR4K5jv3qwOvwoCoqAf20hgE2CyHkLyUe0xkR/JzroPy8fvFdVqQnAsaFooMw==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/mod-arch-kubeflow/-/mod-arch-kubeflow-1.2.2.tgz", + "integrity": "sha512-S70TQvRAWXZbs8JW2CBuV1m0dhzl5WyjPThWpv8WlxBFlAXRs3ABuuC+gDIyBaKOZYNO5AjatJsrMIvqH1J1Wg==", "license": "Apache-2.0", "engines": { "node": ">=20.0.0" @@ -20215,9 +20215,9 @@ } }, "node_modules/mod-arch-shared": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/mod-arch-shared/-/mod-arch-shared-1.1.5.tgz", - "integrity": "sha512-VP9V6DXa8+VJ3UawbrqE11VCBBTNI8B3MfPB/+GyBa+Ko9bfDbnsBfZ+dPqfY9ydT/R0+Q6ZMGnqvMEXqwwEew==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/mod-arch-shared/-/mod-arch-shared-1.2.2.tgz", + "integrity": "sha512-12bTo8atwWrJwApNGDYi2/dlUeX2Jw1lmL8oQWkcRuDt3W2ZAOOydupNrLaJJHNxLc+FVFFljdAOg+C6VPBSRA==", "license": "Apache-2.0", "dependencies": { "@patternfly/patternfly": "^6.2.0", @@ -20247,7 +20247,8 @@ }, "peerDependencies": { "react": ">=16.8.0", - "react-dom": ">=16.8.0" + "react-dom": ">=16.8.0", + "react-router-dom": ">=6.4.0" } }, "node_modules/mrmime": { diff --git a/clients/ui/frontend/package.json b/clients/ui/frontend/package.json index 9f1927ce26..3c058ff5f2 100644 --- a/clients/ui/frontend/package.json +++ b/clients/ui/frontend/package.json @@ -99,9 +99,9 @@ "classnames": "^2.2.6", "dompurify": "^3.2.4", "lodash-es": "^4.17.15", - "mod-arch-core": "~1.1.5", - "mod-arch-kubeflow": "~1.1.5", - "mod-arch-shared": "~1.1.5", + "mod-arch-core": "~1.2.2", + "mod-arch-kubeflow": "~1.2.2", + "mod-arch-shared": "~1.2.2", "react": "^18", "react-dom": "^18", "react-router": "^7.5.2", diff --git a/clients/ui/frontend/src/__mocks__/mockCatalogSourceConfigList.ts b/clients/ui/frontend/src/__mocks__/mockCatalogSourceConfigList.ts index 593240a60b..bffaadd14f 100644 --- a/clients/ui/frontend/src/__mocks__/mockCatalogSourceConfigList.ts +++ b/clients/ui/frontend/src/__mocks__/mockCatalogSourceConfigList.ts @@ -1,70 +1,67 @@ import { - CatalogSourceConfig, CatalogSourceConfigList, - YamlCatalogSourceConfig, - HuggingFaceCatalogSourceConfig, CatalogSourceType, + HuggingFaceCatalogSourceConfig, + YamlCatalogSourceConfig, } from '~/app/modelCatalogTypes'; export const mockYamlCatalogSourceConfig = ( partial?: Partial, ): YamlCatalogSourceConfig => ({ - id: 'yaml-source-1', - name: 'Red Hat AI', + id: 'sample-source-1', + name: 'Source 1', type: CatalogSourceType.YAML, enabled: true, - labels: ['Red Hat AI'], includedModels: [], excludedModels: [], isDefault: true, - yaml: 'version: 1.0\nmodels:\n - name: example-model', + yaml: 'models:\n - name: model1', ...partial, }); export const mockHuggingFaceCatalogSourceConfig = ( partial?: Partial, ): HuggingFaceCatalogSourceConfig => ({ - id: 'huggingface-source-1', - name: 'Huggingface_Admin_1', + id: 'source-2', + name: 'Huggingface source 2', type: CatalogSourceType.HUGGING_FACE, enabled: true, - labels: ['Hugging Face'], includedModels: [], excludedModels: [], isDefault: false, - allowedOrganization: 'Google', - apiKey: undefined, + allowedOrganization: 'org1', + apiKey: 'apikey', ...partial, }); -export const mockCatalogSourceConfig = ( - partial?: Partial, -): CatalogSourceConfig => { - if (partial?.type === CatalogSourceType.HUGGING_FACE) { - return mockHuggingFaceCatalogSourceConfig(partial as Partial); - } - return mockYamlCatalogSourceConfig(partial as Partial); -}; - export const mockCatalogSourceConfigList = ( partial?: Partial, ): CatalogSourceConfigList => ({ catalogs: [ - mockYamlCatalogSourceConfig({ id: 'red-hat-ai', name: 'Red Hat AI', isDefault: true }), mockYamlCatalogSourceConfig({ - id: 'red-hat-ai-validated', - name: 'Red Hat AI validated', + id: 'sample_source_1', + name: 'Sample source 1', isDefault: true, + includedModels: [], + excludedModels: [], + }), + mockYamlCatalogSourceConfig({ + id: 'source_2', + name: 'Source 2', + isDefault: false, + includedModels: ['model1', 'model2'], + excludedModels: ['model3'], + enabled: false, }), mockHuggingFaceCatalogSourceConfig({ - id: 'huggingface-admin-1', - name: 'Huggingface_Admin_1', - allowedOrganization: 'Google', + id: 'huggingface_source_3', + name: 'Huggingface source 3', + allowedOrganization: 'org1', isDefault: false, }), mockYamlCatalogSourceConfig({ - id: 'yaml-amdimport-1', - name: 'YAMLAmdImport_1', + id: 'sample_source_4', + name: 'Sample source 4', isDefault: false, includedModels: ['model1', 'model2'], excludedModels: ['model3'], diff --git a/clients/ui/frontend/src/__tests__/cypress/cypress/pages/modelCatalogSettings.ts b/clients/ui/frontend/src/__tests__/cypress/cypress/pages/modelCatalogSettings.ts index 9a40b73a12..b733964a2b 100644 --- a/clients/ui/frontend/src/__tests__/cypress/cypress/pages/modelCatalogSettings.ts +++ b/clients/ui/frontend/src/__tests__/cypress/cypress/pages/modelCatalogSettings.ts @@ -76,15 +76,6 @@ class CatalogSourceConfigRow extends TableRow { return this; } - shouldHaveEnableToggle(shouldExist: boolean) { - if (shouldExist) { - this.findEnableToggle().should('exist'); - } else { - this.find().find('[data-label="Enable"]').should('be.empty'); - } - return this; - } - shouldHaveEnableState(enabled: boolean) { if (enabled) { this.findEnableToggle().should('be.checked'); @@ -104,6 +95,20 @@ class CatalogSourceConfigRow extends TableRow { } return this; } + + shouldHaveValidationStatus(status: 'Connected' | 'Failed' | 'Starting' | 'Unknown' | '-') { + this.findValidationStatus().contains(status); + return this; + } + + findValidationStatusErrorLink() { + return this.findValidationStatus().find('[data-testid*="source-status-error-link"]'); + } + + clickValidationStatusErrorLink() { + this.findValidationStatusErrorLink().click(); + return this; + } } class ModelCatalogSettings { @@ -145,6 +150,10 @@ class ModelCatalogSettings { return cy.findByTestId('add-source-button'); } + findToggleAlert() { + return cy.findByTestId('toggle-alert'); + } + findTable() { return cy.findByTestId('catalog-source-configs-table'); } @@ -173,6 +182,15 @@ class ModelCatalogSettings { this.findEmptyState().should('exist'); return this; } + + findSourceStatusErrorAlert() { + return cy.findByTestId('source-status-error-alert'); + } + + shouldHaveSourceStatusErrorAlert() { + this.findSourceStatusErrorAlert().should('exist'); + return this; + } } class ManageSourcePage { diff --git a/clients/ui/frontend/src/__tests__/cypress/cypress/tests/mocked/modelCatalog/modelCatalogCard.cy.ts b/clients/ui/frontend/src/__tests__/cypress/cypress/tests/mocked/modelCatalog/modelCatalogCard.cy.ts index 51e3bfb7eb..e18c020381 100644 --- a/clients/ui/frontend/src/__tests__/cypress/cypress/tests/mocked/modelCatalog/modelCatalogCard.cy.ts +++ b/clients/ui/frontend/src/__tests__/cypress/cypress/tests/mocked/modelCatalog/modelCatalogCard.cy.ts @@ -182,9 +182,7 @@ describe('ModelCatalogCard Component', () => { it('should display correct source labels', () => { modelCatalog.findFirstModelCatalogCard().within(() => { - modelCatalog - .findSourceLabel() - .should('contain.text', 'source 2text-generationprovider1apache-2.0'); + modelCatalog.findSourceLabel().should('contain.text', 'source 2text-generationprovider1'); }); }); diff --git a/clients/ui/frontend/src/__tests__/cypress/cypress/tests/mocked/modelCatalogSettings/modelCatalogSettings.cy.ts b/clients/ui/frontend/src/__tests__/cypress/cypress/tests/mocked/modelCatalogSettings/modelCatalogSettings.cy.ts index d262ca4e46..bab0df6a2a 100644 --- a/clients/ui/frontend/src/__tests__/cypress/cypress/tests/mocked/modelCatalogSettings/modelCatalogSettings.cy.ts +++ b/clients/ui/frontend/src/__tests__/cypress/cypress/tests/mocked/modelCatalogSettings/modelCatalogSettings.cy.ts @@ -5,13 +5,13 @@ import { } from '~/__tests__/cypress/cypress/pages/modelCatalogSettings'; import { MODEL_CATALOG_API_VERSION } from '~/__tests__/cypress/cypress/support/commands/api'; import { - mockCatalogSource, - mockCatalogSourceList, mockCatalogSourceConfigList, mockYamlCatalogSourceConfig, mockHuggingFaceCatalogSourceConfig, + mockCatalogSource, + mockCatalogSourceList, } from '~/__mocks__'; -import type { CatalogSource, CatalogSourceConfig } from '~/app/modelCatalogTypes'; +import type { CatalogSource, CatalogSourceConfigList } from '~/app/modelCatalogTypes'; const NAMESPACE = 'kubeflow'; const userMock = { @@ -21,11 +21,19 @@ const userMock = { }, }; -const setupMocks = (sources: CatalogSource[] = [], sourceConfigs: CatalogSourceConfig[] = []) => { +const setupMocks = ( + sources: CatalogSource[] = [], + sourceConfigs: CatalogSourceConfigList = { catalogs: [] }, +) => { cy.intercept('GET', '/model-registry/api/v1/namespaces', { data: [{ metadata: { name: NAMESPACE } }], }); cy.intercept('GET', '/model-registry/api/v1/user', userMock); + + cy.intercept('GET', '/model-registry/api/v1/settings/model_catalog/source_configs', { + data: sourceConfigs, + }); + cy.interceptApi( `GET /api/:apiVersion/model_catalog/sources`, { @@ -35,18 +43,6 @@ const setupMocks = (sources: CatalogSource[] = [], sourceConfigs: CatalogSourceC items: sources, }), ); - cy.intercept( - 'GET', - `/model-registry/api/${MODEL_CATALOG_API_VERSION}/settings/model_catalog/source_configs*`, - { - statusCode: 200, - body: { - data: mockCatalogSourceConfigList({ - catalogs: sourceConfigs, - }), - }, - }, - ).as('getCatalogSourceConfigs'); }; function selectNamespaceIfPresent() { @@ -59,14 +55,8 @@ function selectNamespaceIfPresent() { } describe('Model Catalog Settings', () => { - const defaultYamlSource = mockYamlCatalogSourceConfig({ - id: 'default-yaml', - name: 'Default Catalog', - isDefault: true, - }); - beforeEach(() => { - setupMocks([], [defaultYamlSource]); + setupMocks([], mockCatalogSourceConfigList({})); }); it('should display the settings page', () => { @@ -129,13 +119,12 @@ describe('Catalog Source Configs Table', () => { enabled: false, excludedModels: ['excluded-model'], }); - beforeEach(() => { - setupMocks([], [defaultYamlSource, huggingFaceSource, customYamlSource]); + setupMocks([], { catalogs: [defaultYamlSource, huggingFaceSource, customYamlSource] }); }); it('should display empty state when no source configs exist', () => { - setupMocks([], []); + setupMocks([], { catalogs: [] }); modelCatalogSettings.visit(); modelCatalogSettings.shouldBeEmpty(); modelCatalogSettings.findEmptyState().should('contain', 'No catalog sources'); @@ -166,7 +155,6 @@ describe('Catalog Source Configs Table', () => { row.shouldHaveOrganization('-'); row.shouldHaveModelVisibility('Unfiltered'); row.shouldHaveSourceType('YAML file'); - row.shouldHaveEnableToggle(false); // Default sources don't have toggle }); it('should render Hugging Face source correctly', () => { @@ -176,7 +164,6 @@ describe('Catalog Source Configs Table', () => { row.shouldHaveOrganization('Google'); row.shouldHaveModelVisibility('Filtered'); row.shouldHaveSourceType('Hugging Face'); - row.shouldHaveEnableToggle(true); row.shouldHaveEnableState(true); }); @@ -187,35 +174,95 @@ describe('Catalog Source Configs Table', () => { row.shouldHaveOrganization('-'); row.shouldHaveModelVisibility('Filtered'); row.shouldHaveSourceType('YAML file'); - row.shouldHaveEnableToggle(true); row.shouldHaveEnableState(false); }); }); describe('Enable toggle functionality', () => { - it('should show alert when enable toggle is clicked', () => { + it('should disable the source when toggle is clicked', () => { + cy.intercept('PATCH', '/model-registry/api/v1/settings/model_catalog/source_configs/*', { + statusCode: 200, + body: { + data: mockYamlCatalogSourceConfig({ id: 'source_2', isDefault: false }), + }, + }).as('manageToggle'); modelCatalogSettings.visit(); const row = modelCatalogSettings.getRow('HuggingFace Google'); row.findName().should('be.visible'); row.findEnableToggle().should('exist').and('be.checked'); - cy.window().then((win) => { - cy.stub(win, 'alert').as('windowAlert'); + row.toggleEnable(); + cy.wait('@manageToggle').then((interception) => { + expect(interception.request.body).to.eql({ + data: { + enabled: false, + }, + }); }); + }); + + it('should enable the source when toggle is clicked', () => { + cy.intercept('PATCH', '/model-registry/api/v1/settings/model_catalog/source_configs/*', { + statusCode: 200, + body: { + data: mockYamlCatalogSourceConfig({ id: 'source_2', isDefault: false }), + }, + }).as('manageToggle'); + modelCatalogSettings.visit(); + const row = modelCatalogSettings.getRow('Custom YAML'); + row.findName().should('be.visible'); + row.findEnableToggle().should('exist').and('not.be.checked'); row.toggleEnable(); + cy.wait('@manageToggle').then((interception) => { + expect(interception.request.body).to.eql({ + data: { + enabled: true, + }, + }); + }); + }); - cy.get('@windowAlert').should( - 'have.been.calledWith', - 'Toggle clicked! "HuggingFace Google" will be disabled when functionality is implemented.', - ); + it('should show error, if the patch call to toggle fails', () => { + cy.intercept( + 'PATCH', + '/model-registry/api/v1/settings/model_catalog/source_configs/*', + (req) => { + req.reply({ + statusCode: 404, + }); + }, + ).as('manageToggle'); + modelCatalogSettings.visit(); + const row = modelCatalogSettings.getRow('Custom YAML'); + row.findName().should('be.visible'); + row.findEnableToggle().should('exist').and('not.be.checked'); + + row.toggleEnable(); + modelCatalogSettings.findToggleAlert().should('exist'); + modelCatalogSettings + .findToggleAlert() + .should('have.text', 'Danger alert:Error enabling/disabling source Custom YAML'); }); - it('should not show toggle for default sources', () => { + it('should disable the toggle, when the request is processing', () => { + cy.intercept( + 'PATCH', + '/model-registry/api/v1/settings/model_catalog/source_configs/*', + (req) => { + req.reply({ + statusCode: 200, + delay: 1000, + }); + }, + ).as('manageToggle'); modelCatalogSettings.visit(); - const row = modelCatalogSettings.getRow('Default Catalog'); + const row = modelCatalogSettings.getRow('Custom YAML'); row.findName().should('be.visible'); - row.shouldHaveEnableToggle(false); + row.findEnableToggle().should('exist').and('not.be.checked'); + + row.toggleEnable(); + row.findEnableToggle().should('be.disabled'); }); }); @@ -363,11 +410,146 @@ describe('Catalog Source Configs Table', () => { .should('be.visible'); }); }); + + describe('Validation status column', () => { + it('should show "-" for default sources', () => { + setupMocks([], { catalogs: [defaultYamlSource, huggingFaceSource] }); + modelCatalogSettings.visit(); + const row = modelCatalogSettings.getRow('Default Catalog'); + row.findName().should('be.visible'); + row.shouldHaveValidationStatus('-'); + }); + + it('should show "-" for disabled sources', () => { + setupMocks([], { catalogs: [defaultYamlSource, huggingFaceSource] }); + modelCatalogSettings.visit(); + const row = modelCatalogSettings.getRow('Default Catalog'); + row.findName().should('be.visible'); + row.shouldHaveValidationStatus('-'); + }); + + it('should show "Connected" status for available sources', () => { + const availableSource = mockCatalogSource({ + id: 'hf-google', + name: 'HuggingFace Google', + status: 'available', + }); + setupMocks([availableSource], { catalogs: [huggingFaceSource] }); + modelCatalogSettings.visit(); + const row = modelCatalogSettings.getRow('HuggingFace Google'); + row.findName().should('be.visible'); + row.shouldHaveValidationStatus('Connected'); + row.findValidationStatus().findByTestId('source-status-connected-hf-google').should('exist'); + }); + + it('should show "Starting" status when no matching source found', () => { + setupMocks([], { catalogs: [huggingFaceSource] }); + modelCatalogSettings.visit(); + const row = modelCatalogSettings.getRow('HuggingFace Google'); + row.findName().should('be.visible'); + row.shouldHaveValidationStatus('Starting'); + row.findValidationStatus().findByTestId('source-status-starting-hf-google').should('exist'); + }); + + it('should show "Starting" status when source has no status field', () => { + const startingSource = mockCatalogSource({ + id: 'hf-google', + name: 'HuggingFace Google', + status: undefined, + }); + setupMocks([startingSource], { catalogs: [huggingFaceSource] }); + modelCatalogSettings.visit(); + const row = modelCatalogSettings.getRow('HuggingFace Google'); + row.findName().should('be.visible'); + row.shouldHaveValidationStatus('Starting'); + }); + + it('should show "Failed" status with error message for error sources', () => { + const errorSource = mockCatalogSource({ + id: 'hf-google', + name: 'HuggingFace Google', + status: 'error', + error: 'The provided API key is invalid or has expired. Please update your credentials.', + }); + setupMocks([errorSource], { catalogs: [huggingFaceSource] }); + modelCatalogSettings.visit(); + const row = modelCatalogSettings.getRow('HuggingFace Google'); + row.findName().should('be.visible'); + row.shouldHaveValidationStatus('Failed'); + row.findValidationStatus().findByTestId('source-status-failed-hf-google').should('exist'); + row.findValidationStatusErrorLink().should('exist'); + }); + + it('should show truncated error message for long errors', () => { + const longErrorMessage = + 'The specified organization "invalid-org" does not exist or you don\'t have access to it. Please verify the organization name and ensure you have the necessary permissions to access models from this organization.'; + const errorSource = mockCatalogSource({ + id: 'hf-google', + name: 'HuggingFace Google', + status: 'error', + error: longErrorMessage, + }); + setupMocks([errorSource], { catalogs: [huggingFaceSource] }); + modelCatalogSettings.visit(); + const row = modelCatalogSettings.getRow('HuggingFace Google'); + row.findName().should('be.visible'); + row.findValidationStatusErrorLink().find('.pf-v6-c-truncate').should('exist'); + row.findValidationStatusErrorLink().should('contain', longErrorMessage); + }); + + it('should open error modal when clicking error message', () => { + const errorSource = mockCatalogSource({ + id: 'hf-google', + name: 'HuggingFace Google', + status: 'error', + error: 'The provided API key is invalid or has expired.', + }); + setupMocks([errorSource], { catalogs: [huggingFaceSource] }); + modelCatalogSettings.visit(); + const row = modelCatalogSettings.getRow('HuggingFace Google'); + row.findName().should('be.visible'); + row.clickValidationStatusErrorLink(); + + // Check modal is displayed + cy.findByTestId('catalog-source-status-error-modal').should('exist'); + cy.findByTestId('catalog-source-status-error-modal') + .contains('Source status') + .should('exist'); + cy.findByTestId('catalog-source-status-error-modal').contains('Failed').should('exist'); + cy.findByTestId('catalog-source-status-error-alert').should('exist'); + cy.findByTestId('catalog-source-status-error-alert') + .contains('Validation failed') + .should('exist'); + cy.findByTestId('catalog-source-status-error-message').should( + 'contain', + 'The provided API key is invalid or has expired.', + ); + }); + + it('should close error modal when clicking close button', () => { + const errorSource = mockCatalogSource({ + id: 'hf-google', + name: 'HuggingFace Google', + status: 'error', + error: 'The provided API key is invalid.', + }); + setupMocks([errorSource], { catalogs: [huggingFaceSource] }); + modelCatalogSettings.visit(); + const row = modelCatalogSettings.getRow('HuggingFace Google'); + row.clickValidationStatusErrorLink(); + + cy.findByTestId('catalog-source-status-error-modal').should('exist'); + cy.findByTestId('catalog-source-status-error-modal') + .findByRole('button', { name: 'Close' }) + .click(); + cy.findByTestId('catalog-source-status-error-modal').should('not.exist'); + }); + }); }); describe('Manage Source Page', () => { beforeEach(() => { - setupMocks(); + setupMocks([], mockCatalogSourceConfigList({})); }); describe('Add Source Mode', () => { @@ -643,17 +825,82 @@ describe('Manage Source Page', () => { manageSourcePage.findPreviewButtonHeader().should('not.be.disabled'); manageSourcePage.findPreviewButtonPanel().should('not.be.disabled'); }); + + it('submit add source form with yaml source type', () => { + cy.intercept('POST', '/model-registry/api/v1/settings/model_catalog/source_configs', { + data: mockYamlCatalogSourceConfig({}), + }).as('addSourcewithYamlType'); + manageSourcePage.visitAddSource(); + manageSourcePage.findNameInput().type('sample source'); + manageSourcePage.selectSourceType('yaml'); + manageSourcePage.findSourceTypeYaml().should('be.checked'); + + manageSourcePage.findYamlSection().should('exist'); + manageSourcePage.findYamlContentInput().should('exist'); + manageSourcePage.findYamlContentInput().type('models:\n - name: model1'); + + manageSourcePage.toggleModelVisibility(); + manageSourcePage.findAllowedModelsInput().should('exist'); + manageSourcePage.findAllowedModelsInput().type('model-1-*, model-2-*'); + manageSourcePage.findExcludedModelsInput().should('exist'); + manageSourcePage.findExcludedModelsInput().type('model-3-*, model-4-*'); + + manageSourcePage.findSubmitButton().should('be.enabled'); + manageSourcePage.findSubmitButton().click(); + cy.wait('@addSourcewithYamlType').then((interception) => { + expect(interception.request.body).to.eql({ + data: mockYamlCatalogSourceConfig({ + name: 'sample source', + id: 'sample_source', + includedModels: ['model-1-*', 'model-2-*'], + excludedModels: ['model-3-*', 'model-4-*'], + enabled: false, + isDefault: false, + }), + }); + }); + }); + + it('submit the add source form with hugging face source type', () => { + cy.intercept('POST', '/model-registry/api/v1/settings/model_catalog/source_configs', { + data: mockHuggingFaceCatalogSourceConfig({}), + }).as('addSourcewithHuggingFaceType'); + manageSourcePage.visitAddSource(); + manageSourcePage.findNameInput().type('sample source'); + manageSourcePage.selectSourceType('huggingface'); + manageSourcePage.findSourceTypeHuggingFace().should('be.checked'); + + manageSourcePage.findAccessTokenInput().type('apikey'); + manageSourcePage.findOrganizationInput().type('org1'); + + manageSourcePage.toggleModelVisibility(); + manageSourcePage.findAllowedModelsInput().should('exist'); + manageSourcePage.findAllowedModelsInput().type('model-1-*, model-2-*'); + manageSourcePage.findExcludedModelsInput().should('exist'); + manageSourcePage.findExcludedModelsInput().type('model-3-*, model-4-*'); + + manageSourcePage.findSubmitButton().should('be.enabled'); + manageSourcePage.findSubmitButton().click(); + cy.wait('@addSourcewithHuggingFaceType').then((interception) => { + expect(interception.request.body).to.eql({ + data: mockHuggingFaceCatalogSourceConfig({ + name: 'sample source', + id: 'sample_source', + includedModels: ['model-1-*', 'model-2-*'], + excludedModels: ['model-3-*', 'model-4-*'], + enabled: false, + isDefault: false, + }), + }); + }); + }); }); describe('Manage Source Mode', () => { const catalogSourceId = 'test-source-id'; - const catalogSource = mockCatalogSource({ - id: catalogSourceId, - name: 'Test Source', - }); beforeEach(() => { - setupMocks([catalogSource]); + setupMocks([], mockCatalogSourceConfigList({})); }); it('should display manage source page', () => { @@ -680,4 +927,125 @@ describe('Manage Source Page', () => { manageSourcePage.findSubmitButton().should('contain', 'Save'); }); }); + + it('should succesfully update the source with yaml type', () => { + cy.intercept('PATCH', '/model-registry/api/v1/settings/model_catalog/source_configs/*', { + statusCode: 200, + body: { + data: mockYamlCatalogSourceConfig({ id: 'source_2', isDefault: false }), + }, + }).as('manageSourcewithYamlType'); + + manageSourcePage.visitManageSource('source_2'); + cy.url().should('include', '/model-catalog-settings/manage-source/source_2'); + manageSourcePage.findNameInput().should('have.value', 'Source 2'); + manageSourcePage.findSourceTypeHuggingFace().should('not.exist'); + manageSourcePage.findSourceTypeYaml().should('not.exist'); + + manageSourcePage.findAllowedModelsInput().should('exist'); + manageSourcePage.findExcludedModelsInput().should('exist'); + manageSourcePage.findAllowedModelsInput().type(', model-1-*, model-2-*'); + manageSourcePage.findExcludedModelsInput().type(', model-3-*, model-4-*'); + manageSourcePage.findEnableSourceCheckbox().should('not.be.checked'); + manageSourcePage.findEnableSourceCheckbox().check(); + + manageSourcePage.findSubmitButton().should('be.enabled'); + manageSourcePage.findSubmitButton().should('have.text', 'Save'); + manageSourcePage.findSubmitButton().click(); + + cy.wait('@manageSourcewithYamlType').then((interception) => { + expect(interception.request.body).to.eql({ + data: mockYamlCatalogSourceConfig({ + name: 'Source 2', + id: 'source_2', + includedModels: ['model1', 'model2', 'model-1-*', 'model-2-*'], + excludedModels: ['model3', 'model-3-*', 'model-4-*'], + enabled: true, + isDefault: false, + }), + }); + }); + }); + + it('should succesfully update the source with yaml type and default one', () => { + cy.intercept('PATCH', '/model-registry/api/v1/settings/model_catalog/source_configs/*', { + statusCode: 200, + body: { + data: mockYamlCatalogSourceConfig({ id: 'sample_source_1' }), + }, + }).as('manageSourcewithYamlType'); + + manageSourcePage.visitManageSource('sample_source_1'); + cy.url().should('include', '/model-catalog-settings/manage-source/sample_source_1'); + manageSourcePage.findNameInput().should('have.value', 'Sample source 1'); + manageSourcePage.findSourceTypeHuggingFace().should('not.exist'); + manageSourcePage.findSourceTypeYaml().should('not.exist'); + + manageSourcePage.findAllowedModelsInput().should('exist'); + manageSourcePage.findExcludedModelsInput().should('exist'); + manageSourcePage.findAllowedModelsInput().type('model-1-*, model-2-*'); + manageSourcePage.findExcludedModelsInput().type('model-3-*, model-4-*'); + manageSourcePage.findEnableSourceCheckbox().should('be.checked'); + manageSourcePage.findEnableSourceCheckbox().uncheck(); + + manageSourcePage.findSubmitButton().should('be.enabled'); + manageSourcePage.findSubmitButton().should('have.text', 'Save'); + manageSourcePage.findSubmitButton().click(); + + cy.wait('@manageSourcewithYamlType').then((interception) => { + expect(interception.request.body).to.eql({ + data: mockYamlCatalogSourceConfig({ + name: 'Sample source 1', + id: 'sample_source_1', + includedModels: ['model-1-*', 'model-2-*'], + excludedModels: ['model-3-*', 'model-4-*'], + enabled: false, + isDefault: true, + }), + }); + }); + }); + + it('should successfully update the source with huggingface type type', () => { + cy.intercept( + 'PATCH', + `/model-registry/api/${MODEL_CATALOG_API_VERSION}/settings/model_catalog/source_configs/*`, + { + statusCode: 200, + body: { + data: mockYamlCatalogSourceConfig({ id: 'sample-source-1' }), + }, + }, + ).as('manageSourcewithHuggingFaceType'); + + manageSourcePage.visitManageSource('huggingface_source_3'); + manageSourcePage.findNameInput().should('have.value', 'Huggingface source 3'); + + manageSourcePage.findAccessTokenInput().should('have.value', 'apikey'); + manageSourcePage.findOrganizationInput().should('have.value', 'org1'); + + manageSourcePage.toggleModelVisibility(); + manageSourcePage.findAllowedModelsInput().should('exist'); + manageSourcePage.findAllowedModelsInput().type('model-1-*, model-2-*'); + manageSourcePage.findExcludedModelsInput().should('exist'); + manageSourcePage.findExcludedModelsInput().type('model-3-*, model-4-*'); + manageSourcePage.findEnableSourceCheckbox().should('be.checked'); + manageSourcePage.findEnableSourceCheckbox().uncheck(); + + manageSourcePage.findSubmitButton().should('be.enabled'); + manageSourcePage.findSubmitButton().should('have.text', 'Save'); + manageSourcePage.findSubmitButton().click(); + cy.wait('@manageSourcewithHuggingFaceType').then((interception) => { + expect(interception.request.body).to.eql({ + data: mockHuggingFaceCatalogSourceConfig({ + name: 'Huggingface source 3', + id: 'huggingface_source_3', + includedModels: ['model-1-*', 'model-2-*'], + excludedModels: ['model-3-*', 'model-4-*'], + enabled: false, + isDefault: false, + }), + }); + }); + }); }); diff --git a/clients/ui/frontend/src/app/api/modelCatalogSettings/service.ts b/clients/ui/frontend/src/app/api/modelCatalogSettings/service.ts index 4a8ae57b98..ce94eedb21 100644 --- a/clients/ui/frontend/src/app/api/modelCatalogSettings/service.ts +++ b/clients/ui/frontend/src/app/api/modelCatalogSettings/service.ts @@ -53,7 +53,7 @@ export const updateCatalogSourceConfig = ( opts: APIOptions, sourceId: string, - data: CatalogSourceConfigPayload, + data: Partial, ): Promise => handleRestFailures( restPATCH( diff --git a/clients/ui/frontend/src/app/context/modelCatalogSettings/ModelCatalogSettingsContext.tsx b/clients/ui/frontend/src/app/context/modelCatalogSettings/ModelCatalogSettingsContext.tsx index 0f79224b8b..33c6637a16 100644 --- a/clients/ui/frontend/src/app/context/modelCatalogSettings/ModelCatalogSettingsContext.tsx +++ b/clients/ui/frontend/src/app/context/modelCatalogSettings/ModelCatalogSettingsContext.tsx @@ -4,8 +4,10 @@ import useModelCatalogSettingsAPIState, { ModelCatalogSettingsAPIState, } from '~/app/hooks/modelCatalogSettings/useModelCatalogSettingsAPIState'; import { useCatalogSourceConfigs } from '~/app/hooks/modelCatalogSettings/useCatalogSourceConfigs'; -import { CatalogSourceConfigList } from '~/app/modelCatalogTypes'; +import { CatalogSourceConfigList, CatalogSourceList } from '~/app/modelCatalogTypes'; import { BFF_API_VERSION, URL_PREFIX } from '~/app/utilities/const'; +import useModelCatalogAPIState from '~/app/hooks/modelCatalog/useModelCatalogAPIState'; +import { useCatalogSourcesWithPolling } from '~/app/hooks/modelCatalogSettings/useCatalogSourcesWithPolling'; export type ModelCatalogSettingsContextType = { apiState: ModelCatalogSettingsAPIState; @@ -14,6 +16,10 @@ export type ModelCatalogSettingsContextType = { catalogSourceConfigsLoaded: boolean; catalogSourceConfigsLoadError?: Error; refreshCatalogSourceConfigs: () => void; + catalogSources: CatalogSourceList | null; + catalogSourcesLoaded: boolean; + catalogSourcesLoadError?: Error; + refreshCatalogSources: () => void; }; type ModelCatalogSettingsContextProviderProps = { @@ -28,14 +34,20 @@ export const ModelCatalogSettingsContext = React.createContext undefined, + catalogSources: null, + catalogSourcesLoaded: false, + catalogSourcesLoadError: undefined, + refreshCatalogSources: () => undefined, }); export const ModelCatalogSettingsContextProvider: React.FC< ModelCatalogSettingsContextProviderProps > = ({ children }) => { const hostPath = `${URL_PREFIX}/api/${BFF_API_VERSION}/settings/model_catalog`; + const catalogHostPath = `${URL_PREFIX}/api/${BFF_API_VERSION}/model_catalog`; const queryParams = useQueryParamNamespaces(); const [apiState, refreshAPIState] = useModelCatalogSettingsAPIState(hostPath, queryParams); + const [catalogAPIState] = useModelCatalogAPIState(catalogHostPath, queryParams); const [ catalogSourceConfigs, catalogSourceConfigsLoaded, @@ -43,6 +55,10 @@ export const ModelCatalogSettingsContextProvider: React.FC< refreshCatalogSourceConfigs, ] = useCatalogSourceConfigs(apiState); + // Fetch catalog sources with polling for status updates + const [catalogSources, catalogSourcesLoaded, catalogSourcesLoadError, refreshCatalogSources] = + useCatalogSourcesWithPolling(catalogAPIState); + const contextValue = React.useMemo( () => ({ apiState, @@ -51,6 +67,10 @@ export const ModelCatalogSettingsContextProvider: React.FC< catalogSourceConfigsLoaded, catalogSourceConfigsLoadError, refreshCatalogSourceConfigs, + catalogSources, + catalogSourcesLoaded, + catalogSourcesLoadError, + refreshCatalogSources, }), [ apiState, @@ -59,6 +79,10 @@ export const ModelCatalogSettingsContextProvider: React.FC< catalogSourceConfigsLoaded, catalogSourceConfigsLoadError, refreshCatalogSourceConfigs, + catalogSources, + catalogSourcesLoaded, + catalogSourcesLoadError, + refreshCatalogSources, ], ); diff --git a/clients/ui/frontend/src/app/hooks/modelCatalog/useCatalogSources.ts b/clients/ui/frontend/src/app/hooks/modelCatalog/useCatalogSources.ts index 86d6b2bf28..46613aa024 100644 --- a/clients/ui/frontend/src/app/hooks/modelCatalog/useCatalogSources.ts +++ b/clients/ui/frontend/src/app/hooks/modelCatalog/useCatalogSources.ts @@ -11,8 +11,10 @@ export const useCatalogSources = ( if (!apiState.apiAvailable) { return Promise.reject(new Error('API not yet available')); } - - return apiState.api.getListSources(opts); + return apiState.api.getListSources(opts).then((data) => ({ + ...data, + items: data.items ?? [], + })); }, [apiState], ); diff --git a/clients/ui/frontend/src/app/hooks/modelCatalogSettings/useCatalogSourcesWithPolling.ts b/clients/ui/frontend/src/app/hooks/modelCatalogSettings/useCatalogSourcesWithPolling.ts new file mode 100644 index 0000000000..487d947be2 --- /dev/null +++ b/clients/ui/frontend/src/app/hooks/modelCatalogSettings/useCatalogSourcesWithPolling.ts @@ -0,0 +1,24 @@ +import { FetchState, FetchStateCallbackPromise, useFetchState, POLL_INTERVAL } from 'mod-arch-core'; +import React from 'react'; +import { CatalogSourceList } from '~/app/modelCatalogTypes'; +import { ModelCatalogAPIState } from '~/app/hooks/modelCatalog/useModelCatalogAPIState'; + +export const useCatalogSourcesWithPolling = ( + apiState: ModelCatalogAPIState, +): FetchState => { + const call = React.useCallback>( + (opts) => { + if (!apiState.apiAvailable) { + return Promise.reject(new Error('API not yet available')); + } + + return apiState.api.getListSources(opts); + }, + [apiState], + ); + return useFetchState( + call, + { items: [], size: 0, pageSize: 0, nextPageToken: '' }, + { initialPromisePurity: true, refreshRate: POLL_INTERVAL }, + ); +}; diff --git a/clients/ui/frontend/src/app/modelCatalogTypes.ts b/clients/ui/frontend/src/app/modelCatalogTypes.ts index 8eff766bf5..2585406ce0 100644 --- a/clients/ui/frontend/src/app/modelCatalogTypes.ts +++ b/clients/ui/frontend/src/app/modelCatalogTypes.ts @@ -25,7 +25,7 @@ export type CatalogSource = { error?: string; }; -export type CatalogSourceList = ModelCatalogListParams & { items: CatalogSource[] }; +export type CatalogSourceList = ModelCatalogListParams & { items?: CatalogSource[] }; export type CatalogModel = { source_id?: string; @@ -280,7 +280,7 @@ export type GetCatalogSourceConfig = ( export type UpdateCatalogSourceConfig = ( opts: APIOptions, sourceId: string, - data: CatalogSourceConfigPayload, + data: Partial, ) => Promise; export type DeleteCatalogSourceConfig = (opts: APIOptions, sourceId: string) => Promise; diff --git a/clients/ui/frontend/src/app/pages/modelCatalog/ModelCatalogCoreLoader.tsx b/clients/ui/frontend/src/app/pages/modelCatalog/ModelCatalogCoreLoader.tsx index 26b26bf700..69359cdab3 100644 --- a/clients/ui/frontend/src/app/pages/modelCatalog/ModelCatalogCoreLoader.tsx +++ b/clients/ui/frontend/src/app/pages/modelCatalog/ModelCatalogCoreLoader.tsx @@ -51,7 +51,7 @@ const ModelCatalogCoreLoader: React.FC = () => { ); } - if (catalogSources?.items.length === 0) { + if (catalogSources?.items?.length === 0) { return ( } diff --git a/clients/ui/frontend/src/app/pages/modelCatalog/components/HardwareConfigurationFilterToolbar.tsx b/clients/ui/frontend/src/app/pages/modelCatalog/components/HardwareConfigurationFilterToolbar.tsx index 99213e2e5a..ee6a1d41db 100644 --- a/clients/ui/frontend/src/app/pages/modelCatalog/components/HardwareConfigurationFilterToolbar.tsx +++ b/clients/ui/frontend/src/app/pages/modelCatalog/components/HardwareConfigurationFilterToolbar.tsx @@ -1,5 +1,13 @@ import * as React from 'react'; -import { Toolbar, ToolbarContent, ToolbarGroup, ToolbarItem } from '@patternfly/react-core'; +import { + Button, + Popover, + Toolbar, + ToolbarContent, + ToolbarGroup, + ToolbarItem, +} from '@patternfly/react-core'; +import { HelpIcon } from '@patternfly/react-icons'; import { ModelCatalogContext } from '~/app/context/modelCatalog/ModelCatalogContext'; import { CatalogPerformanceMetricsArtifact } from '~/app/modelCatalogTypes'; import { clearAllFilters } from '~/app/pages/modelCatalog/utils/hardwareConfigurationFilterUtils'; @@ -32,6 +40,17 @@ const HardwareConfigurationFilterToolbar: React.FC + document.body} + > + + + + setIsErrorModalOpen(false)} + errorMessage={errorMessage} + /> + + ); + } + + case CatalogSourceStatusEnum.DISABLED: + return <>-; + + default: + return startingOrUnknownLabel; + } +}; + +export default CatalogSourceStatus; diff --git a/clients/ui/frontend/src/app/pages/modelCatalogSettings/components/CatalogSourceStatusErrorModal.tsx b/clients/ui/frontend/src/app/pages/modelCatalogSettings/components/CatalogSourceStatusErrorModal.tsx new file mode 100644 index 0000000000..298f302b5f --- /dev/null +++ b/clients/ui/frontend/src/app/pages/modelCatalogSettings/components/CatalogSourceStatusErrorModal.tsx @@ -0,0 +1,62 @@ +import * as React from 'react'; +import { + Alert, + Flex, + FlexItem, + Label, + Modal, + ModalBody, + ModalHeader, + ModalVariant, +} from '@patternfly/react-core'; +import { ExclamationCircleIcon } from '@patternfly/react-icons'; + +type CatalogSourceStatusErrorModalProps = { + isOpen: boolean; + onClose: () => void; + errorMessage: string; +}; + +const CatalogSourceStatusErrorModal: React.FC = ({ + isOpen, + onClose, + errorMessage, +}) => { + const titleWithLabel = ( + + Source status + + + + + ); + + return ( + + + + +

+ The system cannot establish a connection to the source. Ensure that the organization and + access token are accurate, then try again. +

+ {errorMessage &&

{errorMessage}

} + + + + ); +}; + +export default CatalogSourceStatusErrorModal; diff --git a/clients/ui/frontend/src/app/pages/modelCatalogSettings/components/CredentialsSection.tsx b/clients/ui/frontend/src/app/pages/modelCatalogSettings/components/CredentialsSection.tsx index f4b0d3c95a..eae6685180 100644 --- a/clients/ui/frontend/src/app/pages/modelCatalogSettings/components/CredentialsSection.tsx +++ b/clients/ui/frontend/src/app/pages/modelCatalogSettings/components/CredentialsSection.tsx @@ -5,6 +5,10 @@ import { FormHelperText, HelperText, HelperTextItem, + Button, + ActionList, + Alert, + AlertActionCloseButton, } from '@patternfly/react-core'; import { UpdateObjectAtPropAndValue } from 'mod-arch-shared'; import PasswordInput from '~/app/shared/components/PasswordInput'; @@ -33,6 +37,26 @@ const CredentialsSection: React.FC = ({ formData, setDa const isOrganizationValid = validateOrganization(formData.organization); const isAccessTokenValid = validateAccessToken(formData.accessToken); + const [validationError, setValidationError] = React.useState(undefined); + const [isValidating, setIsValidating] = React.useState(false); + const [isValidationSuccess, setIsValidationSuccess] = React.useState(false); + + const handleValidate = async () => { + // setIsValidating(true); + // setValidationError(undefined); + + // TODO: Implement validation logic + // setShowAlert(true); + + // if success + setValidationError(undefined); + setIsValidationSuccess(true); + setIsValidating(false); + + //if fails + // setValidationError(new Error('error')); + // setIsValidationSuccess(false); + }; const organizationInput = ( = ({ formData, setDa return ( - {HELP_TEXT.ORGANIZATION} + {isOrganizationTouched && !isOrganizationValid && ( @@ -85,12 +109,12 @@ const CredentialsSection: React.FC = ({ formData, setDa - {HELP_TEXT.ACCESS_TOKEN} + {isAccessTokenTouched && !isAccessTokenValid && ( @@ -101,6 +125,34 @@ const CredentialsSection: React.FC = ({ formData, setDa )} + {validationError && ( + + The system cannot establish a connection to the source. Ensure that the organization and + access token are accurate, then try again. + + )} + {isValidationSuccess && ( + setIsValidationSuccess(false)} />} + > + The organization and accessToken are valid for connection. + + )} + + + + ); }; diff --git a/clients/ui/frontend/src/app/pages/modelCatalogSettings/components/ManageSourceForm.tsx b/clients/ui/frontend/src/app/pages/modelCatalogSettings/components/ManageSourceForm.tsx index 26ca7e9a54..ac4087ce36 100644 --- a/clients/ui/frontend/src/app/pages/modelCatalogSettings/components/ManageSourceForm.tsx +++ b/clients/ui/frontend/src/app/pages/modelCatalogSettings/components/ManageSourceForm.tsx @@ -15,10 +15,12 @@ import { catalogSettingsUrl } from '~/app/routes/modelCatalogSettings/modelCatal import { isFormValid, isPreviewReady } from '~/app/pages/modelCatalogSettings/utils/validation'; import { ManageSourceFormData, - SourceType, useManageSourceData, } from '~/app/pages/modelCatalogSettings/useManageSourceData'; import { FORM_LABELS, DESCRIPTIONS } from '~/app/pages/modelCatalogSettings/constants'; +import { ModelCatalogSettingsContext } from '~/app/context/modelCatalogSettings/ModelCatalogSettingsContext'; +import { transformFormDataToPayload } from '~/app/pages/modelCatalogSettings/utils/modelCatalogSettingsUtils'; +import { CatalogSourceType } from '~/app/modelCatalogTypes'; import SourceDetailsSection from './SourceDetailsSection'; import CredentialsSection from './CredentialsSection'; import YamlSection from './YamlSection'; @@ -36,20 +38,33 @@ const ManageSourceForm: React.FC = ({ existingData, isEdi const [formData, setData] = useManageSourceData(existingData); const [isSubmitting, setIsSubmitting] = React.useState(false); const [submitError, setSubmitError] = React.useState(undefined); + const { apiState, refreshCatalogSourceConfigs } = React.useContext(ModelCatalogSettingsContext); - const isHuggingFaceMode = formData.sourceType === SourceType.HuggingFace; + const isHuggingFaceMode = formData.sourceType === CatalogSourceType.HUGGING_FACE; const isFormComplete = isFormValid(formData); const canPreview = isPreviewReady(formData); const handleSubmit = async () => { + if (!apiState.apiAvailable) { + setSubmitError(new Error('API is not available')); + return; + } setIsSubmitting(true); setSubmitError(undefined); try { - // TODO: Implement submit logic (will be part of API integration) - // navigate(catalogSettingsUrl()); + const payload = transformFormDataToPayload(formData); + + if (isEditMode) { + await apiState.api.updateCatalogSourceConfig({}, payload.id, payload); + } else { + await apiState.api.createCatalogSourceConfig({}, payload); + } + + refreshCatalogSourceConfigs(); + navigate(catalogSettingsUrl()); } catch (error) { - setSubmitError(error instanceof Error ? error : new Error('Failed to save source')); + setSubmitError(error instanceof Error ? error : new Error(`Failed to save source`)); } finally { setIsSubmitting(false); } @@ -65,12 +80,16 @@ const ManageSourceForm: React.FC = ({ existingData, isEdi return ( <> - - + +
- + {isHuggingFaceMode && ( @@ -79,14 +98,22 @@ const ManageSourceForm: React.FC = ({ existingData, isEdi )} - {!isHuggingFaceMode && ( + {!formData.isDefault && !isHuggingFaceMode && ( )} - + @@ -111,7 +138,7 @@ const ManageSourceForm: React.FC = ({ existingData, isEdi
- + diff --git a/clients/ui/frontend/src/app/pages/modelCatalogSettings/components/ModelVisibilitySection.tsx b/clients/ui/frontend/src/app/pages/modelCatalogSettings/components/ModelVisibilitySection.tsx index 73b69637d4..c87c9ff57d 100644 --- a/clients/ui/frontend/src/app/pages/modelCatalogSettings/components/ModelVisibilitySection.tsx +++ b/clients/ui/frontend/src/app/pages/modelCatalogSettings/components/ModelVisibilitySection.tsx @@ -11,10 +11,7 @@ import { import { UpdateObjectAtPropAndValue } from 'mod-arch-shared'; import FormFieldset from '~/app/pages/modelRegistry/screens/components/FormFieldset'; import FormSection from '~/app/pages/modelRegistry/components/pf-overrides/FormSection'; -import { - ManageSourceFormData, - SourceType, -} from '~/app/pages/modelCatalogSettings/useManageSourceData'; +import { ManageSourceFormData } from '~/app/pages/modelCatalogSettings/useManageSourceData'; import { FORM_LABELS, PLACEHOLDERS, @@ -24,14 +21,20 @@ import { getAllowedModelsHelp, getExcludedModelsHelp, } from '~/app/pages/modelCatalogSettings/constants'; +import { CatalogSourceType } from '~/app/modelCatalogTypes'; type ModelVisibilitySectionProps = { formData: ManageSourceFormData; setData: UpdateObjectAtPropAndValue; + isDefaultExpanded?: boolean; }; -const ModelVisibilitySection: React.FC = ({ formData, setData }) => { - const isHuggingFaceMode = formData.sourceType === SourceType.HuggingFace; +const ModelVisibilitySection: React.FC = ({ + formData, + setData, + isDefaultExpanded = false, +}) => { + const isHuggingFaceMode = formData.sourceType === CatalogSourceType.HUGGING_FACE; const organization = isHuggingFaceMode ? formData.organization : undefined; const sectionDescription = @@ -86,6 +89,7 @@ const ModelVisibilitySection: React.FC = ({ formDat titleDescription={sectionDescription} /> } + isExpanded={isDefaultExpanded} data-testid="model-visibility-section" > diff --git a/clients/ui/frontend/src/app/pages/modelCatalogSettings/components/SourceDetailsSection.tsx b/clients/ui/frontend/src/app/pages/modelCatalogSettings/components/SourceDetailsSection.tsx index 94e40d272b..f9d6729f5d 100644 --- a/clients/ui/frontend/src/app/pages/modelCatalogSettings/components/SourceDetailsSection.tsx +++ b/clients/ui/frontend/src/app/pages/modelCatalogSettings/components/SourceDetailsSection.tsx @@ -12,29 +12,33 @@ import { import { UpdateObjectAtPropAndValue } from 'mod-arch-shared'; import FormFieldset from '~/app/pages/modelRegistry/screens/components/FormFieldset'; import FormSection from '~/app/pages/modelRegistry/components/pf-overrides/FormSection'; -import { - ManageSourceFormData, - SourceType, -} from '~/app/pages/modelCatalogSettings/useManageSourceData'; +import { ManageSourceFormData } from '~/app/pages/modelCatalogSettings/useManageSourceData'; import { validateSourceName } from '~/app/pages/modelCatalogSettings/utils/validation'; import { FORM_LABELS, SOURCE_TYPE_LABELS, VALIDATION_MESSAGES, } from '~/app/pages/modelCatalogSettings/constants'; +import { CatalogSourceType } from '~/app/modelCatalogTypes'; type SourceDetailsSectionProps = { formData: ManageSourceFormData; setData: UpdateObjectAtPropAndValue; + isEditMode: boolean; }; -const SourceDetailsSection: React.FC = ({ formData, setData }) => { +const SourceDetailsSection: React.FC = ({ + formData, + setData, + isEditMode, +}) => { const [isNameTouched, setIsNameTouched] = React.useState(false); const isNameValid = validateSourceName(formData.name); const nameInput = ( = ({ formData, s return ( - + {isNameTouched && !isNameValid && ( @@ -68,28 +72,36 @@ const SourceDetailsSection: React.FC = ({ formData, s role="radiogroup" aria-labelledby="source-type-label" > - - - setData('sourceType', SourceType.HuggingFace)} - label={SOURCE_TYPE_LABELS.HUGGING_FACE} - id="source-type-huggingface" - data-testid="source-type-huggingface" - /> - - - setData('sourceType', SourceType.YAML)} - label={SOURCE_TYPE_LABELS.YAML} - id="source-type-yaml" - data-testid="source-type-yaml" - /> - - + {isEditMode ? ( + + {formData.sourceType === CatalogSourceType.HUGGING_FACE + ? SOURCE_TYPE_LABELS.HUGGING_FACE + : SOURCE_TYPE_LABELS.YAML} + + ) : ( + + + setData('sourceType', CatalogSourceType.HUGGING_FACE)} + label={SOURCE_TYPE_LABELS.HUGGING_FACE} + id="source-type-huggingface" + data-testid="source-type-huggingface" + /> + + + setData('sourceType', CatalogSourceType.YAML)} + label={SOURCE_TYPE_LABELS.YAML} + id="source-type-yaml" + data-testid="source-type-yaml" + /> + + + )} ); diff --git a/clients/ui/frontend/src/app/pages/modelCatalogSettings/screens/CatalogSourceConfigsTable.tsx b/clients/ui/frontend/src/app/pages/modelCatalogSettings/screens/CatalogSourceConfigsTable.tsx index 67b4c4504b..7a8248698c 100644 --- a/clients/ui/frontend/src/app/pages/modelCatalogSettings/screens/CatalogSourceConfigsTable.tsx +++ b/clients/ui/frontend/src/app/pages/modelCatalogSettings/screens/CatalogSourceConfigsTable.tsx @@ -1,45 +1,128 @@ import * as React from 'react'; -import { Button, Toolbar, ToolbarContent, ToolbarItem } from '@patternfly/react-core'; +import { + Alert, + AlertActionCloseButton, + Button, + Flex, + FlexItem, + Stack, + StackItem, + Toolbar, + ToolbarContent, + ToolbarItem, +} from '@patternfly/react-core'; import { PlusCircleIcon } from '@patternfly/react-icons'; import { Table } from 'mod-arch-shared'; import { CatalogSourceConfig } from '~/app/modelCatalogTypes'; +import { ModelCatalogSettingsContext } from '~/app/context/modelCatalogSettings/ModelCatalogSettingsContext'; import { catalogSourceConfigsColumns } from './CatalogSourceConfigsTableColumns'; import CatalogSourceConfigsTableRow from './CatalogSourceConfigsTableRow'; type CatalogSourceConfigsTableProps = { catalogSourceConfigs: CatalogSourceConfig[]; onAddSource: () => void; + onDeleteSource: (sourceId: string) => Promise; }; const CatalogSourceConfigsTable: React.FC = ({ catalogSourceConfigs, onAddSource, -}) => ( - - - - - - - + onDeleteSource, +}) => { + const [toggleError, setToggleError] = React.useState(undefined); + const [isUpdatingToggle, setIsUpdatingToggle] = React.useState(false); + const { apiState, refreshCatalogSourceConfigs, catalogSourcesLoadError } = React.useContext( + ModelCatalogSettingsContext, + ); + + const handleEnableToggle = async (checked: boolean, catalogSourceConfig: CatalogSourceConfig) => { + if (!apiState.apiAvailable) { + setToggleError(new Error('API is not available')); + return; + } + setIsUpdatingToggle(true); + setToggleError(undefined); + + try { + await apiState.api.updateCatalogSourceConfig({}, catalogSourceConfig.id, { + enabled: checked, + }); + setToggleError(undefined); + refreshCatalogSourceConfigs(); + } catch (e) { + if (e instanceof Error) { + setToggleError(new Error(`Error enabling/disabling source ${catalogSourceConfig.name}`)); + } + } finally { + setIsUpdatingToggle(false); } - rowRenderer={(config) => ( - - )} - variant="compact" - /> -); + }; + + return ( + + {catalogSourcesLoadError && ( + + + {catalogSourcesLoadError.message} + + + )} + +
+ + + + + + + + + + {toggleError && ( + + setToggleError(undefined)} /> + } + /> + + )} + + } + rowRenderer={(config) => ( + + )} + variant="compact" + /> + + + ); +}; export default CatalogSourceConfigsTable; diff --git a/clients/ui/frontend/src/app/pages/modelCatalogSettings/screens/CatalogSourceConfigsTableRow.tsx b/clients/ui/frontend/src/app/pages/modelCatalogSettings/screens/CatalogSourceConfigsTableRow.tsx index 32f225ca84..9994de0b04 100644 --- a/clients/ui/frontend/src/app/pages/modelCatalogSettings/screens/CatalogSourceConfigsTableRow.tsx +++ b/clients/ui/frontend/src/app/pages/modelCatalogSettings/screens/CatalogSourceConfigsTableRow.tsx @@ -9,19 +9,24 @@ import { ModelVisibilityBadgeColor, } from '~/concepts/modelCatalogSettings/const'; import { hasSourceFilters, getOrganizationDisplay } from '~/concepts/modelCatalogSettings/utils'; -import { ModelCatalogSettingsContext } from '~/app/context/modelCatalogSettings/ModelCatalogSettingsContext'; import DeleteModal from '~/app/shared/components/DeleteModal'; import { useNotification } from '~/app/hooks/useNotification'; +import CatalogSourceStatus from '~/app/pages/modelCatalogSettings/components/CatalogSourceStatus'; type CatalogSourceConfigsTableRowProps = { catalogSourceConfig: CatalogSourceConfig; + onDeleteSource: (sourceId: string) => Promise; + isUpdatingToggle: boolean; + onToggleUpdate: (checked: boolean, sourceConfig: CatalogSourceConfig) => void; }; const CatalogSourceConfigsTableRow: React.FC = ({ catalogSourceConfig, + onDeleteSource, + isUpdatingToggle, + onToggleUpdate, }) => { const navigate = useNavigate(); - const { apiState, refreshCatalogSourceConfigs } = React.useContext(ModelCatalogSettingsContext); const notification = useNotification(); const [isDeleteModalOpen, setIsDeleteModalOpen] = React.useState(false); const [isDeleting, setIsDeleting] = React.useState(false); @@ -36,17 +41,14 @@ const CatalogSourceConfigsTableRow: React.FC ); const handleEnableToggle = (checked: boolean) => { - // TODO: Implement actual enable/disable functionality - window.alert( - `Toggle clicked! "${catalogSourceConfig.name}" will be ${checked ? 'enabled' : 'disabled'} when functionality is implemented.`, - ); + onToggleUpdate(checked, catalogSourceConfig); }; const handleManageSource = () => { navigate(manageSourceUrl(catalogSourceConfig.id)); }; - const handleDeleteSource = () => { + const handleDeleteClick = () => { setDeleteError(undefined); setIsDeleteModalOpen(true); }; @@ -56,9 +58,8 @@ const CatalogSourceConfigsTableRow: React.FC setDeleteError(undefined); try { - await apiState.api.deleteCatalogSourceConfig({}, catalogSourceConfig.id); + await onDeleteSource(catalogSourceConfig.id); setIsDeleteModalOpen(false); - refreshCatalogSourceConfigs(); notification.success(`${catalogSourceConfig.name} deleted successfully`); } catch (error) { setDeleteError(error instanceof Error ? error : new Error('Failed to delete source')); @@ -111,17 +112,18 @@ const CatalogSourceConfigsTableRow: React.FC + -
- {!isDefault && ( - handleEnableToggle(checked)} - /> - )} + handleEnableToggle(checked)} + /> + + {/* TODO: Status implementation */}