Merge pull request #47 from planetf1/chunkfixcli

Author: planetf1

cli/README.md

@@ -344,6 +344,26 @@ Semantic chunking uses sentence transformers to identify natural break points in
 
 **Note**: Semantic chunking uses sentence-transformers for chunking decisions, but the resulting chunks are embedded using your collection's embedding model (e.g., nomic-embed-text) for search operations.
 
+Additional notes:
+
+- Advanced semantic parameters are fully supported via flags in the CLI in addition to the common ones:
+  - `--semantic-model` (model identifier, e.g., all-MiniLM-L6-v2)
+  - `--semantic-window-size` (integer context window)
+  - `--semantic-threshold-percentile` (0–100 split sensitivity)
+  - Plus common: `--chunk-size`, `--chunk-overlap`
+- Completion: the CLI provides completion for `--chunking-strategy` (includes `Semantic`). The `--semantic-model` value is free-form (no static suggestions); numeric flags disable file completion.
+
+Example with semantic-specific flags:
+
+```bash
+./maestro-k collection create --vdb=my-database --name=my-collection \
+   --chunking-strategy=Semantic \
+   --chunk-size=768 \
+   --semantic-model=all-MiniLM-L6-v2 \
+   --semantic-window-size=1 \
+   --semantic-threshold-percentile=95
+```
+
 ### Environment Variable Substitution in YAML Files
 
 The CLI supports environment variable substitution in YAML files using the `{{ENV_VAR_NAME}}` syntax. This allows you to use environment variables directly in your configuration files:
@@ -368,6 +388,7 @@ When you run `./maestro-k create vector-db config.yaml`, the CLI will:
 3. Process the YAML file with the substituted values
 
 **Features**:
+
 - **Automatic substitution**: All `{{ENV_VAR_NAME}}` placeholders are replaced before YAML parsing
 - **Error handling**: Clear error messages if required environment variables are missing
 - **Verbose logging**: Shows which environment variables are being substituted (when using `--verbose`)
@@ -541,6 +562,7 @@ The CLI provides resource-based create commands for vector databases, collection
 ```
 
 **Supported Override Flags**:
+
 - `--type`: Override database type (milvus, weaviate)
 - `--uri`: Override connection URI
 - `--collection-name`: Override collection name
@@ -558,6 +580,13 @@ The CLI provides resource-based create commands for vector databases, collection
 
 # Create collection with dry-run mode
 ./maestro-k collection create --name=my-collection --vdb=my-database --dry-run
+
+# Create collection with chunking configuration
+./maestro-k collection create --name=my-collection --vdb=my-database \
+   --embedding=text-embedding-3-small \
+   --chunking-strategy=Sentence \
+   --chunk-size=512 \
+   --chunk-overlap=32
 ```
 
 #### Create Document Command

cli/src/commands.go

@@ -380,4 +380,12 @@ func init() {
 
 	// Add flags to collection create command
 	collectionCreateCmd.Flags().StringVar(&collectionEmbedding, "embedding", "default", "Embedding model to use for the collection")
+	// Expose chunking flags here as well (kept in sync with create_collection.go)
+	collectionCreateCmd.Flags().StringVar(&collectionChunkStrategy, "chunking-strategy", "None", "Chunking strategy to use for the collection (None, Fixed, Sentence, Semantic)")
+	collectionCreateCmd.Flags().IntVar(&collectionChunkSize, "chunk-size", 0, "Chunk size in characters (optional; defaults may vary by strategy)")
+	collectionCreateCmd.Flags().IntVar(&collectionChunkOverlap, "chunk-overlap", 0, "Chunk overlap in characters (optional)")
+	// Semantic-specific parameters
+	collectionCreateCmd.Flags().StringVar(&semanticModel, "semantic-model", "", "Semantic chunking model identifier (e.g., all-MiniLM-L6-v2)")
+	collectionCreateCmd.Flags().IntVar(&semanticWindowSize, "semantic-window-size", 0, "Semantic chunking window size (optional)")
+	collectionCreateCmd.Flags().Float64Var(&semanticThresholdPercentile, "semantic-threshold-percentile", 0, "Semantic chunking threshold percentile (0-100, optional)")
 }

cli/src/completion.go

@@ -177,7 +177,7 @@ func (cp *CompletionProvider) CompleteEmbeddings(partial string) ([]CompletionIt
 
 // CompleteChunkingStrategies provides completion for chunking strategy values
 func (cp *CompletionProvider) CompleteChunkingStrategies(partial string) ([]CompletionItem, error) {
-	strategies := []string{"None", "Fixed", "Sentence"}
+	strategies := []string{"None", "Fixed", "Sentence", "Semantic"}
 
 	var completions []CompletionItem
 	for _, s := range strategies {
@@ -259,6 +259,7 @@ func (cp *CompletionProvider) CompleteFlags(partial string) ([]CompletionItem, e
 		"--verbose", "--silent", "--dry-run", "--force",
 		"--mcp-server-uri", "--doc-limit",
 		"--chunking-strategy", "--chunk-size", "--chunk-overlap",
+		"--semantic-model", "--semantic-window-size", "--semantic-threshold-percentile",
 		"-h", "--help", "--version",
 	}
 

cli/src/completion_cmd.go

@@ -154,6 +154,17 @@ func SetupCustomCompletions() {
 		collectionCreateCmd.RegisterFlagCompletionFunc("chunk-overlap", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return nil, cobra.ShellCompDirectiveNoFileComp
 		})
+
+		// Semantic-specific flags
+		collectionCreateCmd.RegisterFlagCompletionFunc("semantic-model", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return nil, cobra.ShellCompDirectiveNoFileComp
+		})
+		collectionCreateCmd.RegisterFlagCompletionFunc("semantic-window-size", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return nil, cobra.ShellCompDirectiveNoFileComp
+		})
+		collectionCreateCmd.RegisterFlagCompletionFunc("semantic-threshold-percentile", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return nil, cobra.ShellCompDirectiveNoFileComp
+		})
 	}
 
 	// Status --vdb completion using vector database names

cli/src/create_collection.go

@@ -78,15 +78,19 @@ var (
 	collectionChunkStrategy string
 	collectionChunkSize     int
 	collectionChunkOverlap  int
+	// Semantic-specific params
+	semanticModel               string
+	semanticWindowSize          int
+	semanticThresholdPercentile float64
 )
 
 func init() {
 	// Add flags for collection creation to all collection creation commands
 	commands := []*cobra.Command{createCollectionCmd, createVdbColCmd, createColCmd}
 	for _, cmd := range commands {
 		cmd.Flags().StringVar(&collectionEmbedding, "embedding", "default", "Embedding model to use for the collection")
-		cmd.Flags().StringVar(&collectionChunkStrategy, "chunking-strategy", "None", "Chunking strategy to use for the collection (None, Fixed, Sentence)")
-		cmd.Flags().IntVar(&collectionChunkSize, "chunk-size", 0, "Chunk size in characters (optional, defaults to 512 when strategy != None)")
+		cmd.Flags().StringVar(&collectionChunkStrategy, "chunking-strategy", "None", "Chunking strategy to use for the collection (None, Fixed, Sentence, Semantic)")
+		cmd.Flags().IntVar(&collectionChunkSize, "chunk-size", 0, "Chunk size in characters (optional; defaults may vary by strategy)")
 		cmd.Flags().IntVar(&collectionChunkOverlap, "chunk-overlap", 0, "Chunk overlap in characters (optional)")
 	}
 }
@@ -149,6 +153,18 @@ func createCollection(vdbName, collectionName string) error {
 			if collectionChunkOverlap > 0 {
 				params["overlap"] = collectionChunkOverlap
 			}
+			if collectionChunkStrategy == "Semantic" {
+				if semanticWindowSize > 0 {
+					params["window_size"] = semanticWindowSize
+				}
+				if semanticThresholdPercentile > 0 {
+					params["threshold_percentile"] = semanticThresholdPercentile
+				}
+				if semanticModel != "" {
+					// Use 'model_name' to align with server and semantic chunking API
+					params["model_name"] = semanticModel
+				}
+			}
 			chunkCfg = map[string]interface{}{
 				"strategy":   collectionChunkStrategy,
 				"parameters": params,

cli/tests/create_collection_test.go

@@ -133,4 +133,56 @@ func TestCreateCollectionHelp(t *testing.T) {
 	if !contains(outputStr, "--name string") {
 		t.Errorf("Should show name flag in help, got: %s", outputStr)
 	}
+
+	// Ensure chunking flags are shown in help
+	if !contains(outputStr, "--chunking-strategy") {
+		t.Errorf("Help should include --chunking-strategy flag, got: %s", outputStr)
+	}
+	if !contains(outputStr, "--chunk-size") {
+		t.Errorf("Help should include --chunk-size flag, got: %s", outputStr)
+	}
+	if !contains(outputStr, "--chunk-overlap") {
+		t.Errorf("Help should include --chunk-overlap flag, got: %s", outputStr)
+	}
+
+	// Ensure semantic-specific flags are shown in help
+	if !contains(outputStr, "--semantic-model") {
+		t.Errorf("Help should include --semantic-model flag, got: %s", outputStr)
+	}
+	if !contains(outputStr, "--semantic-window-size") {
+		t.Errorf("Help should include --semantic-window-size flag, got: %s", outputStr)
+	}
+	if !contains(outputStr, "--semantic-threshold-percentile") {
+		t.Errorf("Help should include --semantic-threshold-percentile flag, got: %s", outputStr)
+	}
+}
+
+// TestCreateCollectionWithChunkingFlagsDryRun ensures chunking flags are accepted (dry-run)
+func TestCreateCollectionWithChunkingFlagsDryRun(t *testing.T) {
+	cmd := exec.Command("../maestro-k", "collection", "create",
+		"--name=test-collection", "--vdb=test-db",
+		"--chunking-strategy=Sentence", "--chunk-size=512", "--chunk-overlap=64",
+		"--dry-run")
+	output, err := cmd.CombinedOutput()
+
+	if err != nil {
+		t.Fatalf("Create collection with chunking flags failed: %v, output: %s", err, string(output))
+	}
+}
+
+// TestCreateCollectionWithSemanticFlagsDryRun ensures semantic-specific flags are accepted (dry-run)
+func TestCreateCollectionWithSemanticFlagsDryRun(t *testing.T) {
+	cmd := exec.Command("../maestro-k", "collection", "create",
+		"--name=test-collection", "--vdb=test-db",
+		"--chunking-strategy=Semantic",
+		"--chunk-size=768",
+		"--semantic-model=all-MiniLM-L6-v2",
+		"--semantic-window-size=1",
+		"--semantic-threshold-percentile=95",
+		"--dry-run")
+	output, err := cmd.CombinedOutput()
+
+	if err != nil {
+		t.Fatalf("Create collection with semantic flags failed: %v, output: %s", err, string(output))
+	}
 }