refactor(artifact): flatten protobuf packages and decouple files from knowledge bases (#321)

**Because**

- The protobuf package structure was nested
(`artifact/artifact/v1alpha`) and needed flattening to
`artifact/v1alpha` for consistency across the codebase
- Files were tightly coupled to a single knowledge base, limiting
reusability
- Resource IDs needed to follow AIP standard with unique constraints and
proper scoping
- The System resource used RFC-1034 IDs instead of hash-based IDs like
other resources

**This commit**

- Flattens protobuf imports from `protogen-go/artifact/artifact/v1alpha`
to `protogen-go/artifact/v1alpha`
- Decouples files from knowledge bases - files are now namespace-scoped
and can be associated with multiple KBs via a junction table
(`file_knowledge_base`)
- Adds 7 database migrations:
  - `000048`: Add slug columns to knowledge_base and file tables
  - `000049`: Add unique constraints for resource IDs (AIP standard)
- `000050`: Unify System schema to use hash-based IDs with `sys-` prefix
  - `000051`: Unify storage paths
  - `000052`: Rename chunk content destination
  - `000053`: Add ID column to chunk table with `chk-` prefix
  - `000054`: Create junction table and migrate file-KB relationships
- Renames `KnowledgeBaseFile*` to `File*` throughout repository and
service layers
- Updates all integration tests to use the flattened proto paths and new
file API structure
- Removes deprecated code: `pkg/usage/usage.go`,
`pkg/client/http/registry.go`
- Updates handler parsing to support new file resource pattern:
`namespaces/{namespace}/files/{file}`

Author: pinglin

.env

@@ -1,5 +1,5 @@
 # version
-GOLANG_VERSION=1.24.6
+GOLANG_VERSION=1.25.6
 K6_VERSION=1.0.0
 XK6_VERSION=0.19.3
 XK6_SQL_VERSION=1.0.4

.github/workflows/coverage.yml

@@ -31,7 +31,7 @@ jobs:
           go test -race ./... -coverprofile=coverage.txt -covermode=atomic
 
       - name: Upload coverage report
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v5
         with:
           file: ./coverage.txt
           flags: unittests

.github/workflows/golangci-lint.yml

@@ -24,5 +24,5 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v8
         with:
-          version: v2.1.6
+          version: v2.8.0
           args: --timeout=10m

Dockerfile

@@ -1,4 +1,4 @@
-ARG GOLANG_VERSION=1.24.6
+ARG GOLANG_VERSION=1.25.6
 FROM golang:${GOLANG_VERSION} AS build
 
 WORKDIR /build

Dockerfile.dev

@@ -1,4 +1,4 @@
-ARG GOLANG_VERSION=1.24.6
+ARG GOLANG_VERSION=1.25.6
 FROM golang:${GOLANG_VERSION}-bullseye
 
 ARG SERVICE_NAME SERVICE_VERSION

Makefile

@@ -6,7 +6,9 @@
 include .env
 export
 
-# Integration test defaults (for running from host)
+# Integration test configuration
+# - From host: make integration-test (uses localhost:8080)
+# - In container: make integration-test API_GATEWAY_URL=api-gateway:8080 DB_HOST=pg_sql
 API_GATEWAY_PROTOCOL ?= http
 API_GATEWAY_URL ?= localhost:8080
 DB_HOST ?= localhost
@@ -94,11 +96,7 @@ unit-test:       				## Run unit test
 
 .PHONY: integration-test
 integration-test:		## Run integration tests (parallel by default, sequential if CI=true)
-	@if [ -n "${API_GATEWAY_URL}" ]; then \
-		echo "✓ Running tests through API Gateway: ${API_GATEWAY_URL}"; \
-	else \
-		echo "⚠ WARNING: No API_GATEWAY_URL set - using default localhost:8080"; \
-	fi
+	@echo "✓ Running tests via API Gateway: ${API_GATEWAY_URL}"
 	@echo "  DB_HOST: ${DB_HOST}"
 ifeq ($(CI),true)
 	@echo "Running integration tests sequentially (CI mode)..."
@@ -108,18 +106,21 @@ ifeq ($(CI),true)
 		integration-test/rest.js \
 		integration-test/rest-object-storage.js \
 		integration-test/rest-hash-based-ids.js \
+		integration-test/rest-invariants.js \
 		integration-test/rest-file-type.js \
 		integration-test/rest-db.js \
 		integration-test/rest-ai-client.js \
 		integration-test/rest-kb-e2e-file-process.js \
 		integration-test/rest-file-reprocess.js \
 		integration-test/rest-kb-delete.js \
+		integration-test/rest-namespace-permission.js \
 		integration-test/grpc.js \
 		integration-test/grpc-kb-update.js \
 		integration-test/grpc-system-config-update.js \
 		integration-test/grpc-system-admin.js; do \
 		echo "Running $$test..." | tee -a /tmp/artifact-integration-test.log; \
 		TEST_FOLDER_ABS_PATH=$(PWD) k6 run --address="" \
+			-e CI=true \
 			-e API_GATEWAY_PROTOCOL=$(API_GATEWAY_PROTOCOL) \
 			-e API_GATEWAY_URL=$(API_GATEWAY_URL) \
 			-e DB_HOST=$(DB_HOST) \
@@ -138,13 +139,15 @@ else
 		{} --no-usage-report" ::: \
 		integration-test/rest.js \
 		integration-test/rest-object-storage.js \
-		integration-test/rest-hash-based-ids.js 2>&1 | tee -a /tmp/artifact-integration-test.log
+		integration-test/rest-hash-based-ids.js \
+		integration-test/rest-invariants.js 2>&1 | tee -a /tmp/artifact-integration-test.log
 	# Batch 2: File processing tests (heavy AI workload)
 	@parallel --halt now,fail=1 --tag --line-buffer \
 		"TEST_FOLDER_ABS_PATH=${PWD} k6 run --address=\"\" \
 		-e API_GATEWAY_PROTOCOL=${API_GATEWAY_PROTOCOL} -e API_GATEWAY_URL=${API_GATEWAY_URL} \
 		-e DB_HOST=${DB_HOST} \
 		{} --no-usage-report" ::: \
+		integration-test/rest-namespace-permission.js \
 		integration-test/rest-file-type.js \
 		integration-test/rest-db.js \
 		integration-test/rest-ai-client.js 2>&1 | tee -a /tmp/artifact-integration-test.log

cmd/init/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"log"
 	"time"
 
@@ -11,14 +12,39 @@ import (
 	grpczap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap"
 
 	"github.com/instill-ai/artifact-backend/config"
+	"github.com/instill-ai/artifact-backend/pkg/ai"
+	"github.com/instill-ai/artifact-backend/pkg/ai/gemini"
+	"github.com/instill-ai/artifact-backend/pkg/ai/openai"
 	"github.com/instill-ai/artifact-backend/pkg/pipeline"
+	"github.com/instill-ai/artifact-backend/pkg/repository"
 
-	pipelinepb "github.com/instill-ai/protogen-go/pipeline/pipeline/v1beta"
+	database "github.com/instill-ai/artifact-backend/pkg/db"
+	pipelinepb "github.com/instill-ai/protogen-go/pipeline/v1beta"
 	clientx "github.com/instill-ai/x/client"
 	clientgrpcx "github.com/instill-ai/x/client/grpc"
 	logx "github.com/instill-ai/x/log"
 )
 
+// DefaultSystemPresets defines the default system configurations to seed
+var DefaultSystemPresets = []repository.PresetSystem{
+	{
+		DisplayName:    "OpenAI",
+		Slug:           "openai",
+		ModelFamily:    ai.ModelFamilyOpenAI,
+		Dimensionality: openai.DefaultEmbeddingDimension,
+		Description:    "OpenAI embedding configuration (text-embedding-3-small, 1536 dimensions)",
+		IsDefault:      false,
+	},
+	{
+		DisplayName:    "Gemini",
+		Slug:           "gemini",
+		ModelFamily:    ai.ModelFamilyGemini,
+		Dimensionality: gemini.DefaultEmbeddingDimension,
+		Description:    "Gemini embedding configuration (text-embedding-004, 3072 dimensions)",
+		IsDefault:      true, // Gemini is the default
+	},
+}
+
 func main() {
 	ctx := context.Background()
 
@@ -40,6 +66,19 @@ func main() {
 		grpczap.ReplaceGrpcLoggerV2WithVerbosity(logger, 3) // verbosity 3 will avoid [transport] from emitting
 	}
 
+	// Initialize database connection for system seeding
+	db := database.GetSharedConnection()
+	defer database.Close(db)
+
+	// Seed default systems
+	logger.Info("Seeding default system configurations...")
+	repo := repository.NewDBOnlyRepository(db)
+	if err := repo.SeedDefaultSystems(ctx, DefaultSystemPresets); err != nil {
+		logger.Fatal(fmt.Sprintf("failed to seed default systems: %v", err))
+	}
+	logger.Info("Default system configurations seeded successfully")
+
+	// Initialize pipeline client for preset pipelines
 	pipelinePublicServiceClient, pipelinePublicClose, err := clientgrpcx.NewClient[pipelinepb.PipelinePublicServiceClient](
 		clientgrpcx.WithServiceConfig(clientx.ServiceConfig{
 			Host:       config.Config.PipelineBackend.Host,
@@ -66,7 +105,7 @@ func main() {
 	}
 
 	for _, pr := range pipeline.PresetPipelinesList {
-		logger := logger.With(zap.String("id", pr.ID), zap.String("version", pr.Version))
+		logger := logger.With(zap.String("slug", pr.Slug()), zap.String("displayName", pr.DisplayName), zap.String("version", pr.Version))
 		if err := upserter.Upsert(ctx, pr); err != nil {
 			logger.Error("Failed to add pipeline", zap.Error(err))
 			continue

cmd/main/main.go

@@ -39,15 +39,12 @@ import (
 	"github.com/instill-ai/artifact-backend/pkg/repository"
 	"github.com/instill-ai/artifact-backend/pkg/repository/object"
 	"github.com/instill-ai/artifact-backend/pkg/service"
-	"github.com/instill-ai/artifact-backend/pkg/usage"
-	"github.com/instill-ai/artifact-backend/pkg/utils"
 	"github.com/instill-ai/artifact-backend/pkg/worker"
 
 	database "github.com/instill-ai/artifact-backend/pkg/db"
-	artifactpb "github.com/instill-ai/protogen-go/artifact/artifact/v1alpha"
-	mgmtpb "github.com/instill-ai/protogen-go/core/mgmt/v1beta"
-	usagepb "github.com/instill-ai/protogen-go/core/usage/v1beta"
-	pipelinepb "github.com/instill-ai/protogen-go/pipeline/pipeline/v1beta"
+	artifactpb "github.com/instill-ai/protogen-go/artifact/v1alpha"
+	mgmtpb "github.com/instill-ai/protogen-go/mgmt/v1beta"
+	pipelinepb "github.com/instill-ai/protogen-go/pipeline/v1beta"
 	clientx "github.com/instill-ai/x/client"
 	clientgrpcx "github.com/instill-ai/x/client/grpc"
 	logx "github.com/instill-ai/x/log"
@@ -253,39 +250,6 @@ func main() {
 		}),
 	)
 
-	// Start usage reporter
-	var usg usage.Usage
-	if config.Config.Server.Usage.Enabled {
-		usageServiceClient, usageServiceClientClose, err := clientgrpcx.NewClient[usagepb.UsageServiceClient](
-			clientgrpcx.WithServiceConfig(clientx.ServiceConfig{
-				Host:       config.Config.Server.Usage.Host,
-				PublicPort: config.Config.Server.Usage.Port,
-			}),
-			clientgrpcx.WithSetOTELClientHandler(config.Config.OTELCollector.Enable),
-		)
-		if err != nil {
-			logger.Error("failed to create usage service client", zap.Error(err))
-		}
-		defer func() {
-			if err := usageServiceClientClose(); err != nil {
-				logger.Error("failed to close usage service client", zap.Error(err))
-			}
-		}()
-		logger.Info("try to start usage reporter")
-		go utils.GoRecover(func() {
-			for {
-				usg = usage.NewUsage(ctx, mgmtPrivateServiceClient, redisClient, usageServiceClient, serviceVersion)
-				if usg != nil {
-					usg.StartReporter(ctx)
-					logger.Info("usage reporter started")
-					break
-				}
-				logger.Warn("retry to start usage reporter after 5 minutes")
-				time.Sleep(5 * time.Minute)
-			}
-		}, "Usage Reporter")
-	}
-
 	dialOpts, err := clientgrpcx.NewClientOptionsAndCreds(
 		clientgrpcx.WithServiceConfig(clientx.ServiceConfig{
 			HTTPS: clientx.HTTPSConfig{
@@ -363,9 +327,6 @@ func main() {
 		logger.Error(fmt.Sprintf("Fatal error: %v\n", err))
 		os.Exit(1)
 	case <-quitSig:
-		if config.Config.Server.Usage.Enabled && usg != nil {
-			usg.TriggerSingleReporter(ctx)
-		}
 		logger.Info("Shutting down server...")
 		publicGrpcS.GracefulStop()
 		logger.Info("server shutdown due to signal")

cmd/worker/main.go

@@ -37,8 +37,8 @@ import (
 
 	database "github.com/instill-ai/artifact-backend/pkg/db"
 	artifactworker "github.com/instill-ai/artifact-backend/pkg/worker"
-	mgmtpb "github.com/instill-ai/protogen-go/core/mgmt/v1beta"
-	pipelinepb "github.com/instill-ai/protogen-go/pipeline/pipeline/v1beta"
+	mgmtpb "github.com/instill-ai/protogen-go/mgmt/v1beta"
+	pipelinepb "github.com/instill-ai/protogen-go/pipeline/v1beta"
 	clientgrpcx "github.com/instill-ai/x/client/grpc"
 	logx "github.com/instill-ai/x/log"
 	miniox "github.com/instill-ai/x/minio"
@@ -207,16 +207,17 @@ func main() {
 
 	// ===== CleanupKnowledgeBaseWorkflow Activities =====
 	// Activities for cleaning up entire knowledge base resources
-	w.RegisterActivity(cw.GetInProgressFileCountActivity)       // Check for in-progress files before cleanup
-	w.RegisterActivity(cw.DeleteKBFilesFromMinIOActivity)       // Delete all KB files from MinIO
-	w.RegisterActivity(cw.DropVectorDBCollectionActivity)       // Drop Milvus collection for KB
-	w.RegisterActivity(cw.DeleteKBFileRecordsActivity)          // Delete all file records from database
-	w.RegisterActivity(cw.DeleteKBConvertedFileRecordsActivity) // Delete all converted file records
-	w.RegisterActivity(cw.DeleteKBTextChunkRecordsActivity)     // Delete all chunk records
-	w.RegisterActivity(cw.DeleteKBEmbeddingRecordsActivity)     // Delete all embedding records
-	w.RegisterActivity(cw.SoftDeleteKBRecordActivity)           // Soft-delete the KB record itself
-	w.RegisterActivity(cw.PurgeKBACLActivity)                   // Remove all ACL permissions for KB
-	w.RegisterActivity(cw.ClearProductionKBRetentionActivity)   // Clear retention field on production KB after rollback cleanup
+	w.RegisterActivity(cw.GetInProgressFileCountActivity)        // Check for in-progress files before cleanup
+	w.RegisterActivity(cw.DeleteKBFilesFromMinIOActivity)        // Delete all KB files from MinIO
+	w.RegisterActivity(cw.DropVectorDBCollectionActivity)        // Drop Milvus collection for KB
+	w.RegisterActivity(cw.DeleteKBFileRecordsActivity)           // Delete all file records from database
+	w.RegisterActivity(cw.DeleteKBConvertedFileRecordsActivity)  // Delete all converted file records
+	w.RegisterActivity(cw.DeleteKBTextChunkRecordsActivity)      // Delete all chunk records
+	w.RegisterActivity(cw.DeleteKBEmbeddingRecordsActivity)      // Delete all embedding records
+	w.RegisterActivity(cw.SoftDeleteKBRecordActivity)            // Soft-delete the KB record itself
+	w.RegisterActivity(cw.PurgeKBACLActivity)                    // Remove all ACL permissions for KB
+	w.RegisterActivity(cw.ClearProductionKBRetentionActivity)    // Clear retention field on production KB after rollback cleanup
+	w.RegisterActivity(cw.CheckRollbackRetentionExpiredActivity) // Check if rollback KB retention period has expired
 
 	// ===== RAG update activities =====
 	// Activities for knowledge base update workflow (6-phase)
@@ -264,7 +265,7 @@ func main() {
 	// Chunking Phase - Combined content and summary chunking (sequential after parallel AI operations)
 	w.RegisterActivity(cw.DeleteOldTextChunksActivity) // Delete old text chunk records before creating new ones
 	w.RegisterActivity(cw.ChunkContentActivity)        // Split markdown content into semantic chunks
-	w.RegisterActivity(cw.SaveTextChunksActivity)      // Persist chunks to database and MinIO storage
+	w.RegisterActivity(cw.SaveChunksActivity)          // Persist chunks to database and MinIO storage
 
 	// Embedding Phase - Vector embedding generation and storage
 	w.RegisterActivity(cw.EmbedAndSaveChunksActivity)      // Combined: query chunks, generate embeddings, save to DB/Milvus