feat: add support for catalog dynamic catalog filtering

Signed-off-by: Sidney Glinton <[email protected]>

Author: syntaxsdev

catalog/internal/catalog/catalog.go

@@ -40,4 +40,8 @@ type APIProvider interface {
 	// model is found with that name, it returns nil. If the model is
 	// found, but has no artifacts, an empty list is returned.
 	GetArtifacts(ctx context.Context, modelName string, sourceID string, params ListArtifactsParams) (model.CatalogArtifactList, error)
+
+	// GetFilterOptions returns all available filter options for models.
+	// This includes field names, data types, and available values or ranges.
+	GetFilterOptions(ctx context.Context) (*model.FilterOptionsList, error)
 }

catalog/internal/catalog/db_catalog.go

@@ -167,6 +167,61 @@ func (d *dbCatalogImpl) GetArtifacts(ctx context.Context, modelName string, sour
 	return *artifactList, nil
 }
 
+func (d *dbCatalogImpl) GetFilterOptions(ctx context.Context) (*apimodels.FilterOptionsList, error) {
+	// Max length threshold for filter values (excludes verbose fields like readme, description)
+	const maxFilterValueLength = 100
+
+	// Query database for filterable properties
+	filterableProps, err := d.catalogModelRepository.GetFilterableProperties(maxFilterValueLength)
+	if err != nil {
+		return nil, err
+	}
+
+	// Build FilterOptionsList
+	options := make(map[string]apimodels.FilterOption)
+
+	// Process each property and its values
+	for fieldName, values := range filterableProps {
+		// Skip internal/technical fields that shouldn't be exposed as filters
+		if fieldName == "source_id" || fieldName == "logo" || fieldName == "license_link" {
+			continue
+		}
+
+		// Deduplicate values
+		uniqueValues := make(map[string]bool)
+
+		// Parse JSON arrays for fields like language and tasks
+		for _, value := range values {
+			var arrayValues []string
+			if err := json.Unmarshal([]byte(value), &arrayValues); err == nil {
+				// Successfully parsed as array, add individual values
+				for _, v := range arrayValues {
+					uniqueValues[v] = true
+				}
+			} else {
+				// Not a JSON array
+				uniqueValues[value] = true
+			}
+		}
+
+		if len(uniqueValues) > 0 {
+			expandedValues := make([]interface{}, 0, len(uniqueValues))
+			for v := range uniqueValues {
+				expandedValues = append(expandedValues, v)
+			}
+
+			options[fieldName] = apimodels.FilterOption{
+				Type:   "string",
+				Values: expandedValues,
+			}
+		}
+	}
+
+	return &apimodels.FilterOptionsList{
+		Filters: &options,
+	}, nil
+}
+
 func mapDBModelToAPIModel(m dbmodels.CatalogModel) apimodels.CatalogModel {
 	res := apimodels.CatalogModel{}
 

catalog/internal/catalog/hf_catalog.go

@@ -49,6 +49,15 @@ func (h *hfCatalogImpl) GetArtifacts(ctx context.Context, modelName string, sour
 	}, nil
 }
 
+func (h *hfCatalogImpl) GetFilterOptions(ctx context.Context) (*openapi.FilterOptionsList, error) {
+	// TODO: Implement HuggingFace filter options retrieval
+	// For now, return empty options to satisfy interface
+	emptyFilters := make(map[string]openapi.FilterOption)
+	return &openapi.FilterOptionsList{
+		Filters: &emptyFilters,
+	}, nil
+}
+
 // validateCredentials checks if the HuggingFace API credentials are valid
 func (h *hfCatalogImpl) validateCredentials(ctx context.Context) error {
 	glog.Infof("Validating HuggingFace API credentials")

catalog/internal/db/models/catalog_model.go

@@ -37,4 +37,7 @@ type CatalogModelRepository interface {
 	GetByName(name string) (CatalogModel, error)
 	List(listOptions CatalogModelListOptions) (*models.ListWrapper[CatalogModel], error)
 	Save(model CatalogModel) (CatalogModel, error)
+	// GetFilterableProperties returns a map of property names to their unique values
+	// Only includes string properties where all values are shorter than maxLength
+	GetFilterableProperties(maxLength int) (map[string][]string, error)
 }

catalog/internal/db/service/catalog_model.go

@@ -231,3 +231,53 @@ func mapDataLayerToCatalogModel(modelCtx schema.Context, propertiesCtx []schema.
 
 	return catalogModel
 }
+
+// GetFilterableProperties returns property names and their unique values
+// Only includes properties where ALL values are shorter than maxLength
+func (r *CatalogModelRepositoryImpl) GetFilterableProperties(maxLength int) (map[string][]string, error) {
+	config := r.GetConfig()
+
+	// Simplified query: get distinct property name/value pairs
+	query := `
+		SELECT DISTINCT cp.name, cp.string_value
+		FROM "ContextProperty" cp
+		WHERE cp.context_id IN (
+			SELECT id FROM "Context" WHERE type_id = ?
+		)
+		AND cp.name IN (
+			-- Only include property names where max length is within threshold
+			SELECT name FROM (
+				SELECT name, MAX(CHAR_LENGTH(string_value)) as max_len
+				FROM "ContextProperty"
+				WHERE context_id IN (
+					SELECT id FROM "Context" WHERE type_id = ?
+				)
+				AND string_value IS NOT NULL
+				AND string_value != ''
+				GROUP BY name
+			) AS field_lengths
+			WHERE max_len <= ?
+		)
+		AND cp.string_value IS NOT NULL
+		AND cp.string_value != ''
+		ORDER BY cp.name, cp.string_value
+	`
+
+	type propertyRow struct {
+		Name        string
+		StringValue string
+	}
+
+	var rows []propertyRow
+	if err := config.DB.Raw(query, config.TypeID, config.TypeID, maxLength).Scan(&rows).Error; err != nil {
+		return nil, fmt.Errorf("error querying filterable properties: %w", err)
+	}
+
+	// Aggregate values by property name in Go
+	result := make(map[string][]string)
+	for _, row := range rows {
+		result[row.Name] = append(result[row.Name], row.StringValue)
+	}
+
+	return result, nil
+}

catalog/internal/server/openapi/type_asserts.go

@@ -283,8 +283,10 @@ func AssertFilterOptionRequired(obj model.FilterOption) error {
 		}
 	}
 
-	if err := AssertFilterOptionRangeRequired(obj.Range); err != nil {
-		return err
+	if obj.Range != nil {
+		if err := AssertFilterOptionRangeRequired(*obj.Range); err != nil {
+			return err
+		}
 	}
 	return nil
 }