[POC] CSGHub server integration testing framework

.github/workflows/test.yaml

@@ -10,6 +10,10 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   lint:
     name: lint
@@ -44,8 +48,24 @@ jobs:
         run: |
           go install github.com/swaggo/swag/cmd/swag@latest
           make swag
-  test:
-    name: test
+  unit-tests:
+    name: unit-tests
+    strategy:
+      matrix:
+        go: ["1.23"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Go
+        with:
+          go-version: ${{ matrix.go }}
+        uses: actions/setup-go@v2
+
+      - uses: actions/checkout@v2
+
+      - name: Test
+        run: go test ./... -short
+  integration-tests:
+    name: integration-tests
     strategy:
       matrix:
         go: ["1.23"]
@@ -59,4 +79,4 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: Test
-        run: go test ./...
+        run: GITHUB=true go test opencsg.com/csghub-server/tests/...

Makefile

@@ -1,4 +1,4 @@
-.PHONY: test lint cover mock_wire mock_gen swag migrate_local
+.PHONY: test lint cover mock_wire mock_gen swag migrate_local run_test_server
 
 test:
 	go test ./...
@@ -31,3 +31,6 @@ swag:
 
 migrate_local:
 	go run cmd/csghub-server/main.go migration migrate --config local.toml
+
+run_test_server:
+	go run tests/main.go

api/httpbase/graceful_server.go

@@ -65,3 +65,7 @@ func (s *GracefulServer) Run() {
 
 	slog.Info("Server stopped")
 }
+
+func (s *GracefulServer) Shutdown(ctx context.Context) error {
+	return s.server.Shutdown(ctx)
+}

api/router/api.go

@@ -24,14 +24,10 @@ import (
 	"opencsg.com/csghub-server/mirror"
 )
 
-func RunServer(config *config.Config, enableSwagger bool) {
-	stopOtel, err := instrumentation.SetupOTelSDK(context.Background(), config, "csghub-api")
-	if err != nil {
-		panic(err)
-	}
+func NewServer(config *config.Config, enableSwagger bool) (*httpbase.GracefulServer, error) {
 	r, err := NewRouter(config, enableSwagger)
 	if err != nil {
-		panic(err)
+		return nil, err
 	}
 	slog.Info("csghub service is running", slog.Any("port", config.APIServer.Port))
 	server := httpbase.NewGracefulServer(
@@ -49,11 +45,21 @@ func RunServer(config *config.Config, enableSwagger bool) {
 	if config.MirrorServer.Enable && config.GitServer.Type == types.GitServerTypeGitaly {
 		mirrorService.EnqueueMirrorTasks()
 	}
+	return server, nil
+}
 
+func RunServer(config *config.Config, enableSwagger bool) {
+	stopOtel, err := instrumentation.SetupOTelSDK(context.Background(), config, "csghub-api")
+	if err != nil {
+		panic(err)
+	}
+	server, err := NewServer(config, enableSwagger)
+	if err != nil {
+		panic(err)
+	}
 	server.Run()
 	_ = stopOtel(context.Background())
 	temporal.Stop()
-
 }
 
 func NewRouter(config *config.Config, enableSwagger bool) (*gin.Engine, error) {

api/workflow/activity/sync_as_client.go

@@ -14,6 +14,9 @@ func (a *Activities) SyncAsClient(ctx context.Context) error {
 		return err
 	}
 	apiDomain := a.config.MultiSync.SaasAPIDomain
+	if apiDomain == "" {
+		return nil
+	}
 	sc := multisync.FromOpenCSG(apiDomain, setting.Token)
 	return a.multisync.SyncAsClient(ctx, sc)
 }

api/workflow/schedule_ce_test.go

@@ -29,7 +29,7 @@ func TestSchedule_SyncAsClient(t *testing.T) {
 		&database.SyncClientSetting{Token: "tk"}, nil,
 	)
 	tester.mocks.multisync.EXPECT().SyncAsClient(
-		mock.Anything, multisync.FromOpenCSG("", "tk"),
+		mock.Anything, multisync.FromOpenCSG("http://foo.com", "tk"),
 	).Return(nil)
 
 	tester.scheduler.Execute("sync-as-client-schedule", tester.cronEnv)

api/workflow/workflow_ce_test.go

@@ -42,6 +42,7 @@ func newWorkflowTester(t *testing.T) (*workflowTester, error) {
 	scanner := mock_component.NewMockRuntimeArchitectureComponent(t)
 
 	cfg := &config.Config{}
+	cfg.MultiSync.SaasAPIDomain = "http://foo.com"
 	mtc := mock_temporal.NewMockClient(t)
 	mtc.EXPECT().NewWorker(workflow.HandlePushQueueName, mock.Anything).Return(tester.env)
 	mtc.EXPECT().NewWorker(workflow.CronJobQueueName, mock.Anything).Return(tester.cronEnv)

builder/store/database/db.go

@@ -51,6 +51,10 @@ type Operator struct {
 	Core bun.IDB
 }
 
+func GetDB() *DB {
+	return defaultDB
+}
+
 func InitDB(config DBConfig) {
 	bg, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()

common/config/config.go

@@ -2,8 +2,10 @@ package config
 
 import (
 	"context"
+	"log/slog"
 	"os"
 	"reflect"
+	"sync"
 
 	"github.com/naoina/toml"
 	"github.com/sethvargo/go-envconfig"
@@ -261,9 +263,30 @@ type Config struct {
 
 func SetConfigFile(file string) {
 	configFile = file
+	// The config file is provided via the command line, which should be the only entry point for running
+	// all CSGHub server commands. However, if some code accidentally calls LoadConfig in the init function
+	// before the config file is loaded from the command line, the config system will be broken entirely.
+	// To address this, we reset the `once` variable when SetConfigFile is called, ensuring that even if
+	// the config is incorrect during initialization, it will be corrected when the config is loaded again
+	// from the command line.
+	// This is a temporary fix. We should avoid relying on a global config and instead use DI
+	// to pass the config wherever it's needed.
+	once = sync.OnceValues(func() (*Config, error) {
+		return loadConfig()
+	})
 }
 
+var once = sync.OnceValues(func() (*Config, error) {
+	return loadConfig()
+})
+
 func LoadConfig() (*Config, error) {
+	return once()
+}
+
+func loadConfig() (*Config, error) {
+	defer slog.Debug("end load config")
+	slog.Debug("start load config")
 	cfg := &Config{}
 
 	toml.DefaultConfig.MissingField = func(typ reflect.Type, key string) error {

common/config/config_test.go

@@ -8,6 +8,8 @@ import (
 
 func TestConfig_LoadConfig(t *testing.T) {
 	t.Run("config env", func(t *testing.T) {
+		// force reload, because config is a global var and may affected by other tests
+		SetConfigFile("")
 		t.Setenv("STARHUB_SERVER_INSTANCE_ID", "foo")
 		t.Setenv("STARHUB_SERVER_SERVER_PORT", "6789")
 		cfg, err := LoadConfig()
@@ -23,9 +25,8 @@ func TestConfig_LoadConfig(t *testing.T) {
 		cfg, err := LoadConfig()
 		require.Nil(t, err)
 
-		require.Equal(t, "bar", cfg.InstanceID)
-		require.Equal(t, 4321, cfg.APIServer.Port)
-		require.Equal(t, "git@localhost:2222", cfg.APIServer.SSHDomain)
+		require.Equal(t, 9091, cfg.APIServer.Port)
+		require.Equal(t, "ssh://git@localhost:2222", cfg.APIServer.SSHDomain)
 	})
 
 	t.Run("file and env", func(t *testing.T) {
@@ -35,7 +36,38 @@ func TestConfig_LoadConfig(t *testing.T) {
 		require.Nil(t, err)
 
 		require.Equal(t, "foobar", cfg.InstanceID)
-		require.Equal(t, 4321, cfg.APIServer.Port)
-		require.Equal(t, "git@localhost:2222", cfg.APIServer.SSHDomain)
+		require.Equal(t, 9091, cfg.APIServer.Port)
+		require.Equal(t, "ssh://git@localhost:2222", cfg.APIServer.SSHDomain)
 	})
 }
+
+func TestConfig_LoadConfigOnce(t *testing.T) {
+	// force reload, because config is a global var and may affected by other tests
+	SetConfigFile("")
+	cfg, err := LoadConfig()
+	require.NoError(t, err)
+	require.Equal(t, 8080, cfg.APIServer.Port)
+
+	SetConfigFile("test.toml")
+	cfg, err = LoadConfig()
+	require.NoError(t, err)
+	require.Equal(t, 9091, cfg.APIServer.Port)
+
+}
+
+func TestConfig_Update(t *testing.T) {
+	// force reload, because config is a global var and may affected by other tests
+	SetConfigFile("")
+	cfg, err := LoadConfig()
+	require.NoError(t, err)
+	require.Equal(t, "", cfg.InstanceID)
+
+	cfg, err = LoadConfig()
+	require.NoError(t, err)
+	cfg.InstanceID = "abc"
+
+	cfg, err = LoadConfig()
+	require.NoError(t, err)
+	require.Equal(t, "abc", cfg.InstanceID)
+
+}

common/config/test.toml

@@ -1,4 +1,105 @@
-instance_id = "bar"
+saas = false
+instance_id = ""
+enable_swagger = false
+enable_https = false
+api_token = "f3a7b9c1d6e5f8e2a1b5d4f9e6a2b8d7c3a4e2b1d9f6e7a8d2c5a7b4c1e3f5b8a1d4f9b7d6e2f8a5d3b1e7f9c6a8b2d1e4f7d5b6e9f2a4b3c8e1d7f995hd82hf"
 
 [api_server]
-port = 4321
+port = 9091
+public_domain = "http://localhost:9091"
+ssh_domain = "ssh://git@localhost:2222"
+
+[database]
+driver = "pg"
+dsn = ""
+timezone = "Asia/Shanghai"
+
+[redis]
+endpoint = "localhost:6379"
+max_retries = 3
+min_idle_connections = 0
+user = ""
+password = ""
+sentinel_mode = false
+sentinel_master = ""
+sentinel_endpoint = ""
+
+[git_server]
+type = "gitaly"
+
+[gitaly_server]
+address = "tcp://localhost:9876"
+storge = "default"
+token = "abc123secret"
+jwt_secret = "signing-key"
+
+[jwt]
+signing_key = "signing-key"
+valid_hour = 24
+
+[model]
+deploy_timeout_in_min = 60
+download_endpoint = "https://hub.opencsg.com"
+docker_reg_base = "opencsg-registry.cn-beijing.cr.aliyuncs.com/public/"
+nim_docker_secret_name = "ngc-secret"
+nim_ngc_secret_name = "nvidia-nim-secrets"
+
+[event]
+sync_interval = 1
+
+[nats]
+url = "nats://natsadmin:[email protected]:4222"
+msg_fetch_timeout_in_sec = 5
+fee_notify_no_balance_subject = "accounting.notify.nobalance"
+fee_request_subject = "accounting.fee.>"
+fee_send_subject = "accounting.fee.credit"
+token_send_subject = "accounting.fee.token"
+quota_send_subject = "accounting.fee.quota"
+meter_request_subject = "accounting.metering.>"
+meter_duration_send_subject = "accounting.metering.duration"
+meter_token_send_subject = "accounting.metering.token"
+meter_quota_send_subject = "accounting.metering.quota"
+order_expired_subject = "accounting.order.expired"
+
+[user]
+host = "http://localhost"
+port = 9092
+signin_success_redirect_url = "http://localhost:3000/server/callback"
+codesouler_vscode_redirect_url = "http://127.0.0.1:37678/callback"
+codesouler_jetbrains_redirect_url = "http://127.0.0.1:37679/callback"
+
+[multi_sync]
+saasApiDomain = ""
+saasSyncDomain = ""
+
+[telemetry]
+enable = false
+
+[dataset]
+promptMaxJsonlFileSize = 1048576  # 1MB
+
+[workflow]
+endpoint = "localhost:7233"
+
+[cron_job]
+sync_as_client_cron_expression = "0 * * * *"
+calc_recom_score_cron_expression = "0 1 * * *"
+
+[dataviewer]
+host = "http://localhost"
+port = 9093
+
+[proxy]
+hosts = ["opencsg.com", "sync.opencsg.com"]
+
+[casdoor]
+certificate = "go.mod"
+
+[s3]
+access_key_id = "abc"
+access_key_secret = "def"
+endpoint = ""
+bucket = "testcsg"
+enable_ssl = false
+url_upload_max_file_size = 5153960755
+bucket_lookup = "path"

common/tests/testutils.go

@@ -69,21 +69,26 @@ func chProjectRoot() {
 
 // Init a test db, must call `defer db.Close()` in the test
 func InitTestDB() *database.DB {
+	db, _ := CreateTestDB("csghub_test")
+	return db
+}
+
+func CreateTestDB(name string) (*database.DB, string) {
 	ctx := context.TODO()
 	// reuse the container, so we don't need to recreate the db for each test
 	// https://github.com/testcontainers/testcontainers-go/issues/2726
 	reuse := testcontainers.CustomizeRequestOption(
 		func(req *testcontainers.GenericContainerRequest) error {
 			req.Reuse = true
-			req.Name = "csghub_test"
+			req.Name = name
 			return nil
 		},
 	)
 
 	pc, err := postgres.Run(ctx,
 		"postgres:15.7",
 		reuse,
-		postgres.WithDatabase("csghub_test"),
+		postgres.WithDatabase(name),
 		testcontainers.WithWaitStrategy(
 			wait.ForLog("database system is ready to accept connections").
 				WithOccurrence(2).
@@ -141,7 +146,7 @@ func InitTestDB() *database.DB {
 	return &database.DB{
 		Operator: database.Operator{Core: bdb},
 		BunDB:    bdb,
-	}
+	}, dsn
 }
 
 // Create a random test postgres Database without txdb,