diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 868364f3..6d64bca0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -10,6 +10,10 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: lint: name: lint @@ -44,8 +48,24 @@ jobs: run: | go install github.com/swaggo/swag/cmd/swag@latest make swag - test: - name: test + unit-tests: + name: unit-tests + strategy: + matrix: + go: ["1.23"] + runs-on: ubuntu-latest + steps: + - name: Setup Go + with: + go-version: ${{ matrix.go }} + uses: actions/setup-go@v2 + + - uses: actions/checkout@v2 + + - name: Test + run: go test ./... -short + integration-tests: + name: integration-tests strategy: matrix: go: ["1.23"] @@ -59,4 +79,4 @@ jobs: - uses: actions/checkout@v2 - name: Test - run: go test ./... + run: GITHUB=true go test opencsg.com/csghub-server/tests/... diff --git a/Makefile b/Makefile index c2e66855..3404f2c3 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: test lint cover mock_wire mock_gen swag migrate_local +.PHONY: test lint cover mock_wire mock_gen swag migrate_local run_test_server test: go test ./... @@ -31,3 +31,6 @@ swag: migrate_local: go run cmd/csghub-server/main.go migration migrate --config local.toml + +run_test_server: + go run tests/main.go diff --git a/api/httpbase/graceful_server.go b/api/httpbase/graceful_server.go index ad8064ed..470796cd 100644 --- a/api/httpbase/graceful_server.go +++ b/api/httpbase/graceful_server.go @@ -65,3 +65,7 @@ func (s *GracefulServer) Run() { slog.Info("Server stopped") } + +func (s *GracefulServer) Shutdown(ctx context.Context) error { + return s.server.Shutdown(ctx) +} diff --git a/api/router/api.go b/api/router/api.go index aef3e7cd..1d5eb168 100644 --- a/api/router/api.go +++ b/api/router/api.go @@ -24,14 +24,10 @@ import ( "opencsg.com/csghub-server/mirror" ) -func RunServer(config *config.Config, enableSwagger bool) { - stopOtel, err := instrumentation.SetupOTelSDK(context.Background(), config, "csghub-api") - if err != nil { - panic(err) - } +func NewServer(config *config.Config, enableSwagger bool) (*httpbase.GracefulServer, error) { r, err := NewRouter(config, enableSwagger) if err != nil { - panic(err) + return nil, err } slog.Info("csghub service is running", slog.Any("port", config.APIServer.Port)) server := httpbase.NewGracefulServer( @@ -49,11 +45,21 @@ func RunServer(config *config.Config, enableSwagger bool) { if config.MirrorServer.Enable && config.GitServer.Type == types.GitServerTypeGitaly { mirrorService.EnqueueMirrorTasks() } + return server, nil +} +func RunServer(config *config.Config, enableSwagger bool) { + stopOtel, err := instrumentation.SetupOTelSDK(context.Background(), config, "csghub-api") + if err != nil { + panic(err) + } + server, err := NewServer(config, enableSwagger) + if err != nil { + panic(err) + } server.Run() _ = stopOtel(context.Background()) temporal.Stop() - } func NewRouter(config *config.Config, enableSwagger bool) (*gin.Engine, error) { diff --git a/api/workflow/activity/sync_as_client.go b/api/workflow/activity/sync_as_client.go index 92652ae2..562bb641 100644 --- a/api/workflow/activity/sync_as_client.go +++ b/api/workflow/activity/sync_as_client.go @@ -14,6 +14,9 @@ func (a *Activities) SyncAsClient(ctx context.Context) error { return err } apiDomain := a.config.MultiSync.SaasAPIDomain + if apiDomain == "" { + return nil + } sc := multisync.FromOpenCSG(apiDomain, setting.Token) return a.multisync.SyncAsClient(ctx, sc) } diff --git a/api/workflow/schedule_ce_test.go b/api/workflow/schedule_ce_test.go index ae1d9d4f..a5584f06 100644 --- a/api/workflow/schedule_ce_test.go +++ b/api/workflow/schedule_ce_test.go @@ -29,7 +29,7 @@ func TestSchedule_SyncAsClient(t *testing.T) { &database.SyncClientSetting{Token: "tk"}, nil, ) tester.mocks.multisync.EXPECT().SyncAsClient( - mock.Anything, multisync.FromOpenCSG("", "tk"), + mock.Anything, multisync.FromOpenCSG("http://foo.com", "tk"), ).Return(nil) tester.scheduler.Execute("sync-as-client-schedule", tester.cronEnv) diff --git a/api/workflow/workflow_ce_test.go b/api/workflow/workflow_ce_test.go index 7f81837e..bcebc71e 100644 --- a/api/workflow/workflow_ce_test.go +++ b/api/workflow/workflow_ce_test.go @@ -42,6 +42,7 @@ func newWorkflowTester(t *testing.T) (*workflowTester, error) { scanner := mock_component.NewMockRuntimeArchitectureComponent(t) cfg := &config.Config{} + cfg.MultiSync.SaasAPIDomain = "http://foo.com" mtc := mock_temporal.NewMockClient(t) mtc.EXPECT().NewWorker(workflow.HandlePushQueueName, mock.Anything).Return(tester.env) mtc.EXPECT().NewWorker(workflow.CronJobQueueName, mock.Anything).Return(tester.cronEnv) diff --git a/builder/store/database/db.go b/builder/store/database/db.go index cf847db9..cf6f238a 100644 --- a/builder/store/database/db.go +++ b/builder/store/database/db.go @@ -51,6 +51,10 @@ type Operator struct { Core bun.IDB } +func GetDB() *DB { + return defaultDB +} + func InitDB(config DBConfig) { bg, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() diff --git a/common/config/config.go b/common/config/config.go index d558baf5..55201bc6 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -2,8 +2,10 @@ package config import ( "context" + "log/slog" "os" "reflect" + "sync" "github.com/naoina/toml" "github.com/sethvargo/go-envconfig" @@ -261,9 +263,30 @@ type Config struct { func SetConfigFile(file string) { configFile = file + // The config file is provided via the command line, which should be the only entry point for running + // all CSGHub server commands. However, if some code accidentally calls LoadConfig in the init function + // before the config file is loaded from the command line, the config system will be broken entirely. + // To address this, we reset the `once` variable when SetConfigFile is called, ensuring that even if + // the config is incorrect during initialization, it will be corrected when the config is loaded again + // from the command line. + // This is a temporary fix. We should avoid relying on a global config and instead use DI + // to pass the config wherever it's needed. + once = sync.OnceValues(func() (*Config, error) { + return loadConfig() + }) } +var once = sync.OnceValues(func() (*Config, error) { + return loadConfig() +}) + func LoadConfig() (*Config, error) { + return once() +} + +func loadConfig() (*Config, error) { + defer slog.Debug("end load config") + slog.Debug("start load config") cfg := &Config{} toml.DefaultConfig.MissingField = func(typ reflect.Type, key string) error { diff --git a/common/config/config_test.go b/common/config/config_test.go index c72203a8..24e5ca7d 100644 --- a/common/config/config_test.go +++ b/common/config/config_test.go @@ -8,6 +8,8 @@ import ( func TestConfig_LoadConfig(t *testing.T) { t.Run("config env", func(t *testing.T) { + // force reload, because config is a global var and may affected by other tests + SetConfigFile("") t.Setenv("STARHUB_SERVER_INSTANCE_ID", "foo") t.Setenv("STARHUB_SERVER_SERVER_PORT", "6789") cfg, err := LoadConfig() @@ -23,9 +25,8 @@ func TestConfig_LoadConfig(t *testing.T) { cfg, err := LoadConfig() require.Nil(t, err) - require.Equal(t, "bar", cfg.InstanceID) - require.Equal(t, 4321, cfg.APIServer.Port) - require.Equal(t, "git@localhost:2222", cfg.APIServer.SSHDomain) + require.Equal(t, 9091, cfg.APIServer.Port) + require.Equal(t, "ssh://git@localhost:2222", cfg.APIServer.SSHDomain) }) t.Run("file and env", func(t *testing.T) { @@ -35,7 +36,38 @@ func TestConfig_LoadConfig(t *testing.T) { require.Nil(t, err) require.Equal(t, "foobar", cfg.InstanceID) - require.Equal(t, 4321, cfg.APIServer.Port) - require.Equal(t, "git@localhost:2222", cfg.APIServer.SSHDomain) + require.Equal(t, 9091, cfg.APIServer.Port) + require.Equal(t, "ssh://git@localhost:2222", cfg.APIServer.SSHDomain) }) } + +func TestConfig_LoadConfigOnce(t *testing.T) { + // force reload, because config is a global var and may affected by other tests + SetConfigFile("") + cfg, err := LoadConfig() + require.NoError(t, err) + require.Equal(t, 8080, cfg.APIServer.Port) + + SetConfigFile("test.toml") + cfg, err = LoadConfig() + require.NoError(t, err) + require.Equal(t, 9091, cfg.APIServer.Port) + +} + +func TestConfig_Update(t *testing.T) { + // force reload, because config is a global var and may affected by other tests + SetConfigFile("") + cfg, err := LoadConfig() + require.NoError(t, err) + require.Equal(t, "", cfg.InstanceID) + + cfg, err = LoadConfig() + require.NoError(t, err) + cfg.InstanceID = "abc" + + cfg, err = LoadConfig() + require.NoError(t, err) + require.Equal(t, "abc", cfg.InstanceID) + +} diff --git a/common/config/test.toml b/common/config/test.toml index 8d2f2d57..3372fe13 100644 --- a/common/config/test.toml +++ b/common/config/test.toml @@ -1,4 +1,105 @@ -instance_id = "bar" +saas = false +instance_id = "" +enable_swagger = false +enable_https = false +api_token = "f3a7b9c1d6e5f8e2a1b5d4f9e6a2b8d7c3a4e2b1d9f6e7a8d2c5a7b4c1e3f5b8a1d4f9b7d6e2f8a5d3b1e7f9c6a8b2d1e4f7d5b6e9f2a4b3c8e1d7f995hd82hf" [api_server] -port = 4321 +port = 9091 +public_domain = "http://localhost:9091" +ssh_domain = "ssh://git@localhost:2222" + +[database] +driver = "pg" +dsn = "" +timezone = "Asia/Shanghai" + +[redis] +endpoint = "localhost:6379" +max_retries = 3 +min_idle_connections = 0 +user = "" +password = "" +sentinel_mode = false +sentinel_master = "" +sentinel_endpoint = "" + +[git_server] +type = "gitaly" + +[gitaly_server] +address = "tcp://localhost:9876" +storge = "default" +token = "abc123secret" +jwt_secret = "signing-key" + +[jwt] +signing_key = "signing-key" +valid_hour = 24 + +[model] +deploy_timeout_in_min = 60 +download_endpoint = "https://hub.opencsg.com" +docker_reg_base = "opencsg-registry.cn-beijing.cr.aliyuncs.com/public/" +nim_docker_secret_name = "ngc-secret" +nim_ngc_secret_name = "nvidia-nim-secrets" + +[event] +sync_interval = 1 + +[nats] +url = "nats://natsadmin:AkxjzcHaK4uRqInuPRLeoUYYV5xYKIWv3jlzCehgoe@127.0.0.1:4222" +msg_fetch_timeout_in_sec = 5 +fee_notify_no_balance_subject = "accounting.notify.nobalance" +fee_request_subject = "accounting.fee.>" +fee_send_subject = "accounting.fee.credit" +token_send_subject = "accounting.fee.token" +quota_send_subject = "accounting.fee.quota" +meter_request_subject = "accounting.metering.>" +meter_duration_send_subject = "accounting.metering.duration" +meter_token_send_subject = "accounting.metering.token" +meter_quota_send_subject = "accounting.metering.quota" +order_expired_subject = "accounting.order.expired" + +[user] +host = "http://localhost" +port = 9092 +signin_success_redirect_url = "http://localhost:3000/server/callback" +codesouler_vscode_redirect_url = "http://127.0.0.1:37678/callback" +codesouler_jetbrains_redirect_url = "http://127.0.0.1:37679/callback" + +[multi_sync] +saasApiDomain = "" +saasSyncDomain = "" + +[telemetry] +enable = false + +[dataset] +promptMaxJsonlFileSize = 1048576 # 1MB + +[workflow] +endpoint = "localhost:7233" + +[cron_job] +sync_as_client_cron_expression = "0 * * * *" +calc_recom_score_cron_expression = "0 1 * * *" + +[dataviewer] +host = "http://localhost" +port = 9093 + +[proxy] +hosts = ["opencsg.com", "sync.opencsg.com"] + +[casdoor] +certificate = "go.mod" + +[s3] +access_key_id = "abc" +access_key_secret = "def" +endpoint = "" +bucket = "testcsg" +enable_ssl = false +url_upload_max_file_size = 5153960755 +bucket_lookup = "path" \ No newline at end of file diff --git a/common/tests/testutils.go b/common/tests/testutils.go index 55f8a553..f90beef0 100644 --- a/common/tests/testutils.go +++ b/common/tests/testutils.go @@ -69,13 +69,18 @@ func chProjectRoot() { // Init a test db, must call `defer db.Close()` in the test func InitTestDB() *database.DB { + db, _ := CreateTestDB("csghub_test") + return db +} + +func CreateTestDB(name string) (*database.DB, string) { ctx := context.TODO() // reuse the container, so we don't need to recreate the db for each test // https://github.com/testcontainers/testcontainers-go/issues/2726 reuse := testcontainers.CustomizeRequestOption( func(req *testcontainers.GenericContainerRequest) error { req.Reuse = true - req.Name = "csghub_test" + req.Name = name return nil }, ) @@ -83,7 +88,7 @@ func InitTestDB() *database.DB { pc, err := postgres.Run(ctx, "postgres:15.7", reuse, - postgres.WithDatabase("csghub_test"), + postgres.WithDatabase(name), testcontainers.WithWaitStrategy( wait.ForLog("database system is ready to accept connections"). WithOccurrence(2). @@ -141,7 +146,7 @@ func InitTestDB() *database.DB { return &database.DB{ Operator: database.Operator{Core: bdb}, BunDB: bdb, - } + }, dsn } // Create a random test postgres Database without txdb, diff --git a/component/git_http.go b/component/git_http.go index b28dbe11..a155e12a 100644 --- a/component/git_http.go +++ b/component/git_http.go @@ -194,7 +194,7 @@ func (c *gitHTTPComponentImpl) lfsBatchDownloadInfo(ctx context.Context, req typ } if !obj.Valid() { objs = append(objs, &types.ObjectResponse{ - Error: &types.ObjectError{}, + Error: &types.ObjectError{Message: "object not valid"}, }) continue } @@ -207,7 +207,7 @@ func (c *gitHTTPComponentImpl) lfsBatchDownloadInfo(ctx context.Context, req typ url, err := c.s3Client.PresignedGetObject(ctx, c.config.S3.Bucket, objectKey, types.OssFileExpire, reqParams) if err != nil { objs = append(objs, &types.ObjectResponse{ - Error: &types.ObjectError{}, + Error: &types.ObjectError{Message: err.Error()}, }) continue } diff --git a/go.mod b/go.mod index 0188b097..e1a4474c 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/golang-jwt/jwt/v5 v5.2.1 github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f github.com/google/wire v0.6.0 + github.com/johannesboyne/gofakes3 v0.0.0-20250106100439-5c39aecd6999 github.com/marcboeker/go-duckdb v1.5.6 github.com/minio/minio-go/v7 v7.0.84 github.com/minio/sha256-simd v1.0.1 @@ -37,6 +38,7 @@ require ( github.com/swaggo/swag v1.16.2 github.com/testcontainers/testcontainers-go v0.33.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.33.0 + github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/uptrace/bun v1.2.8 github.com/uptrace/bun/dialect/pgdialect v1.2.8 @@ -177,6 +179,7 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/robfig/cron v1.2.0 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/samber/lo v1.47.0 // indirect github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a // indirect github.com/shirou/gopsutil/v3 v3.23.12 // indirect @@ -186,7 +189,6 @@ require ( github.com/temporalio/ringpop-go v0.0.0-20241119001152-e505ebd8f887 // indirect github.com/temporalio/sqlparser v0.0.0-20231115171017-f4060bcfa6cb // indirect github.com/temporalio/tchannel-go v1.22.1-0.20240528171429-1db37fdea938 // indirect - github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tinylib/msgp v1.1.2 // indirect @@ -207,6 +209,7 @@ require ( go.opentelemetry.io/otel/exporters/prometheus v0.53.0 // indirect go.opentelemetry.io/otel/metric v1.33.0 // indirect go.opentelemetry.io/proto/otlp v1.4.0 // indirect + go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d // indirect go.temporal.io/version v0.3.0 // indirect go.uber.org/dig v1.17.1 // indirect go.uber.org/fx v1.22.0 // indirect @@ -314,7 +317,7 @@ require ( golang.org/x/crypto v0.32.0 golang.org/x/net v0.34.0 // indirect golang.org/x/oauth2 v0.24.0 // indirect - golang.org/x/sync v0.10.0 // indirect + golang.org/x/sync v0.10.0 golang.org/x/sys v0.29.0 // indirect golang.org/x/term v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect diff --git a/go.sum b/go.sum index 2e395638..6c724ff5 100644 --- a/go.sum +++ b/go.sum @@ -93,6 +93,7 @@ github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/argoproj/argo-workflows/v3 v3.5.12 h1:BD7fEPY+uF9Iellb/hYpsJkDlku8wAJwMVvTskUkZco= github.com/argoproj/argo-workflows/v3 v3.5.12/go.mod h1:DecB01a8UXDCjtIh0udY8XfIMIRrWrlbob7hk/uMmg0= +github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go v1.53.15 h1:FtZmkg7xM8RfP2oY6p7xdKBYrRgkITk9yve2QV7N938= github.com/aws/aws-sdk-go v1.53.15/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/beevik/ntp v1.3.1 h1:Y/srlT8L1yQr58kyPWFPZIxRL8ttx2SRIpVYJqZIlAM= @@ -141,6 +142,8 @@ github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cevatbarisyilmaz/ara v0.0.4 h1:SGH10hXpBJhhTlObuZzTuFn1rrdmjQImITXnZVPSodc= +github.com/cevatbarisyilmaz/ara v0.0.4/go.mod h1:BfFOxnUd6Mj6xmcvRxHN3Sr21Z1T3U2MYkYOmoQe4Ts= github.com/chenyahui/gin-cache v1.9.0 h1:lQlx4qa+Xh3eEuRtmZsviZx4X1uVJ9mOcroti2f+408= github.com/chenyahui/gin-cache v1.9.0/go.mod h1:wh30aYY5rRMUAJmQvw1qoIIcEVRV1EkMJkpXzgipe8U= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -431,12 +434,15 @@ github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 h1:liMMTbpW34dhU4az1GN0pTPADwNmvoRSeoZ6PItiqnY= github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w= github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= +github.com/johannesboyne/gofakes3 v0.0.0-20250106100439-5c39aecd6999 h1:CMbkEl1h9JvRURFFprSbyy2f4Gf71SFz9h74iSAETGo= +github.com/johannesboyne/gofakes3 v0.0.0-20250106100439-5c39aecd6999/go.mod h1:t6osVdP++3g4v2awHz4+HFccij23BbdT1rX3W7IijqQ= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -635,6 +641,8 @@ github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWN github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= github.com/samber/slog-multi v1.3.3 h1:qhFXaYdW73FIWLt8SrXMXfPwY58NpluzKDwRdPvhWWY= @@ -661,6 +669,7 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/sony/gobreaker v1.0.0 h1:feX5fGGXSl3dYd4aHZItw+FpHLvvoaqkawKjVNiFMNQ= github.com/sony/gobreaker v1.0.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spf13/afero v0.0.0-20170901052352-ee1bd8ee15a1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.1.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= @@ -783,6 +792,7 @@ gitlab.com/gitlab-org/go/reopen v1.0.0 h1:6BujZ0lkkjGIejTUJdNO1w56mN1SI10qcVQyQl gitlab.com/gitlab-org/go/reopen v1.0.0/go.mod h1:D6OID8YJDzEVZNYW02R/Pkj0v8gYFSIhXFTArAsBQw8= gitlab.com/gitlab-org/labkit v1.21.2 h1:GlFHh8OdkrIMH3Qi0ByOzva0fGYXMICsuahGpJe4KNQ= gitlab.com/gitlab-org/labkit v1.21.2/go.mod h1:Q++SWyCH/abH2pytnX2SU/3mrCX6aK/xKz/WpM1hLbA= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= @@ -826,6 +836,8 @@ go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qq go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= +go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d h1:Ns9kd1Rwzw7t0BR8XMphenji4SmIoNZPn8zhYmaVKP8= +go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d/go.mod h1:92Uoe3l++MlthCm+koNi0tcUCX3anayogF0Pa/sp24k= go.temporal.io/api v1.43.0 h1:lBhq+u5qFJqGMXwWsmg/i8qn1UA/3LCwVc88l2xUMHg= go.temporal.io/api v1.43.0/go.mod h1:1WwYUMo6lao8yl0371xWUm13paHExN5ATYT/B7QtFis= go.temporal.io/sdk v1.31.0 h1:CLYiP0R5Sdj0gq8LyYKDDz4ccGOdJPR8wNGJU0JGwj8= @@ -901,6 +913,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= @@ -927,8 +940,10 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= @@ -970,6 +985,7 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -990,8 +1006,10 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1003,7 +1021,9 @@ golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= @@ -1016,6 +1036,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= @@ -1034,6 +1055,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1046,6 +1068,7 @@ golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= @@ -1125,6 +1148,7 @@ gopkg.in/ini.v1 v1.56.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/validator.v2 v2.0.1 h1:xF0KWyGWXm/LM2G1TrEjqOu4pa6coO9AlWSf3msVfDY= diff --git a/tests/0.parquet b/tests/0.parquet new file mode 100644 index 00000000..75fabeb8 Binary files /dev/null and b/tests/0.parquet differ diff --git a/tests/1.parquet b/tests/1.parquet new file mode 100644 index 00000000..21901310 Binary files /dev/null and b/tests/1.parquet differ diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..fa7e17df --- /dev/null +++ b/tests/README.md @@ -0,0 +1,74 @@ +# Integration Test + +This directory contains integration tests for the CSGHub server. + +### Starting and Stopping the Test Environment + +**Note:** The test server requires Docker and an internet connection (if the images have not been pulled yet) to run correctly, as we use Testcontainers to start various services and components. + +Setting up the test environment for integration testing is simple. Follow the steps below: + +```go +ctx := context.TODO() +env, err := testinfra.StartTestEnv() +defer func() { _ = env.Shutdown(ctx) }() +``` + +However, before proceeding with writing your tests, please read the following explanation to fully understand what happens in these three lines of code. A lot is happening behind the scenes. + +When `testinfra.StartTestEnv()` is called, the following actions are performed in sequence: + +1. **Load the configuration file**: The `common/config/test.toml` configuration file is loaded. This config is used during integration tests. +2. **Create a test PostgreSQL database**: A PostgreSQL database is created on a random port using test containers. The database configuration in the test config is updated accordingly. +3. **Start the Gitaly server**: A Gitaly server is started using test containers. The configuration used for Gitaly is either `tests/gitaly.toml` or `tests/gitaly_github.toml` (used when running on GitHub). Please see the comment in the `testinfra.go` explaining why two config files are required. The Gitaly server configuration is updated once the container is started. +4. **Start the Temporal test server**: A local Temporal test server is started using the [temporaltest package](https://github.com/temporalio/temporal/blob/main/temporaltest/README.md). The workflow endpoint config is also updated. By default, the Temporal test server uses a random namespace to avoid conflicts, but we force the registration of the default namespace to ensure tests run. +5. **Start the in-memory S3 server**: A local in-memory S3 server is started using [GoFakeS3](https://github.com/johannesboyne/gofakes3). This server is used with the MinIO Go SDK for testing LFS (Large File Storage) functionality. The S3 configuration is updated accordingly. +6. **Start the Redis server**: A Redis server is started using test containers, and the Redis endpoint configuration is updated. +7. **Start the CSGHub user server**: The CSGHub user server and its workflows are started. +8. **Start the CSGHub dataset viewer server**: The CSGHub dataset viewer server and its workflows are started. +9. **Start the CSGHub main API server**: The CSGHub main API server and its workflows are started. + +That’s all. All docker containers are started using Testcontainers, so all data will be removed after test done.Note that not all services are started by default. For example, the NATS server or runner server is not started. If you need to test functionality related to these services, be sure to add them to the environment startup function. + +After the test environment is started, always defer the call to `env.Shutdown(ctx)` to ensure all resources are properly cleaned up. + +### Writing Tests + +There are two example test files: + +- **api/model_test.go**: This tests CRUD operations for models and Git-related functionality. Since the model, dataset, space, and code all share the same repo/Git code, you can consider this file to also test dataset, space, and code-related features. +- **api/dataset_viewer_test.go**: This file tests the Temporal workflows for the dataset viewer server. + +The test code in these files clearly demonstrates how to test the API, Git, and workflows. There’s no need to repeat this here. + +One important thing to remember is that for all integration tests, you should add the following snippet at the beginning of your test function: + +```go +if testing.Short() { + t.Skip("skipping integration test") +} +``` + +This allows you to differentiate between unit tests and integration tests. + +### What to Test in Integration Tests + +Integration tests involve starting multiple services and save data to database (as opposed to database unit tests, which use in-memory transaction and roll back changes after the test). Writing too many integration tests can significantly slow down the testing process, so here are three key suggestions: + +1. **Group related actions into single test cases**: For example, basic CRUD operations can be tested in a single test case. However, avoid overloading tests. Separate unrelated actions, such as API and Git operations, into different tests. +2. **Prioritize the most used features**: Focus on testing the 80% of use cases that are most commonly used in your service. These are the most critical and should not fail. +3. **Test important but less common features**: For features that may not be used frequently but are critical (e.g., those that could result in significant issues, like financial losses), consider adding integration tests for them as well. + +### Starting a Test API Server + +You can also use the test environment to start a temporary test server. To do so, run the following command: + +``` +make run_test_server +``` + +To integrate the test server into CSGHub portal, change CSGHUB_PORTAL_STARHUB_BASE_URL env before make: + +``` +CSGHUB_PORTAL_STARHUB_BASE_URL=http://localhost:9091 make +``` diff --git a/tests/api/dataset_viewer_test.go b/tests/api/dataset_viewer_test.go new file mode 100644 index 00000000..1fcc6b21 --- /dev/null +++ b/tests/api/dataset_viewer_test.go @@ -0,0 +1,129 @@ +package api_test + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" + "opencsg.com/csghub-server/common/types" + "opencsg.com/csghub-server/tests/testinfra" +) + +func TestIntegrationDatasetViewer_Workflow(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + ctx := context.TODO() + env, err := testinfra.StartTestEnv() + defer func() { _ = env.Shutdown(ctx) }() + require.NoError(t, err) + token, err := env.CreateUser(ctx, "user") + require.NoError(t, err) + userClient := testinfra.GetClient(token) + + data := `{"name":"test","nickname":"","namespace":"user","license":"apache-2.0","description":"","private":false}` + req, err := http.NewRequest( + "POST", "http://localhost:9091/api/v1/datasets", bytes.NewBuffer([]byte(data)), + ) + require.NoError(t, err) + resp, err := userClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, 200, resp.StatusCode) + resp, err = userClient.Get("http://localhost:9091/api/v1/datasets/user/test") + require.NoError(t, err) + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + var model types.Model + err = json.Unmarshal([]byte(gjson.GetBytes(body, "data").Raw), &model) + require.NoError(t, err) + cloneURL := model.Repository.HTTPCloneURL + + token, err = env.CreateAccessToken(ctx, "user", types.AccessTokenAppGit) + require.NoError(t, err) + url := strings.ReplaceAll(cloneURL, "http://", fmt.Sprintf("http://%s:%s@", "user", token)) + dir := "dataset_clone" + err = gitClone(url, dir) + require.NoError(t, err) + defer os.RemoveAll(dir) + // add yaml config to readme + file, err := os.OpenFile(dir+"/README.md", os.O_RDWR|os.O_CREATE, 0644) + require.NoError(t, err) + defer file.Close() + fileContent := "" + buf := make([]byte, 1024) + for { + n, err := file.Read(buf) + if err != nil { + break + } + fileContent += string(buf[:n]) + } + configContent := `--- +configs: +- config_name: defaultgo + data_files: + - split: traingo + path: "train/*.parquet" + - split: testgo + path: "test/*.parquet" +--- +` + newContent := configContent + fileContent + _, err = file.Seek(0, 0) + require.NoError(t, err) + _, err = file.WriteString(newContent) + require.NoError(t, err) + + err = exec.Command("mkdir", dir+"/train").Run() + require.NoError(t, err) + err = exec.Command("mkdir", dir+"/test").Run() + require.NoError(t, err) + err = exec.Command("cp", "tests/0.parquet", dir+"/train/0.parquet").Run() + require.NoError(t, err) + err = exec.Command("cp", "tests/1.parquet", dir+"/test/1.parquet").Run() + require.NoError(t, err) + err = gitCommitAndPush(dir) + require.NoError(t, err) + // wait temporal workflow + time.Sleep(5 * time.Second) + + resp, err = userClient.Get("http://localhost:9091/api/v1/datasets/user/test/dataviewer/catalog") + require.NoError(t, err) + defer resp.Body.Close() + body, err = io.ReadAll(resp.Body) + require.NoError(t, err) + expected := `{"msg":"OK","data":{"configs":[{"config_name":"defaultgo","data_files":[{"split":"traingo","path":["train/*.parquet"]},{"split":"testgo","path":["test/*.parquet"]}]}],"dataset_info":[{"config_name":"defaultgo","splits":[{"name":"traingo","num_examples":20},{"name":"testgo","num_examples":20}]}],"status":2,"logs":"Done"}} +` + require.Equal(t, expected, string(body)) + + // check auto created branch + maxAttempts := 10 + for attempts := 1; attempts <= maxAttempts; attempts++ { + resp, err = userClient.Get("http://localhost:9091/api/v1/datasets/user/test/branches") + require.NoError(t, err) + defer resp.Body.Close() + body, err = io.ReadAll(resp.Body) + require.NoError(t, err) + branches := gjson.GetBytes(body, "data.#.name").String() + if branches == `["main","refs-convert-parquet"]` { + break + } + if attempts < maxAttempts { + time.Sleep(1 * time.Second) + } else { + require.FailNow(t, "Branch check failed") + } + } +} diff --git a/tests/api/model_test.go b/tests/api/model_test.go new file mode 100644 index 00000000..c95dd5ff --- /dev/null +++ b/tests/api/model_test.go @@ -0,0 +1,315 @@ +package api_test + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "strings" + "testing" + + "github.com/spf13/cast" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" + "go.opentelemetry.io/otel" + "opencsg.com/csghub-server/builder/parquet" + "opencsg.com/csghub-server/common/types" + "opencsg.com/csghub-server/tests/testinfra" +) + +func TestIntegrationModel_CRUD(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + ctx := context.TODO() + env, err := testinfra.StartTestEnv() + defer func() { _ = env.Shutdown(ctx) }() + require.NoError(t, err) + token, err := env.CreateUser(ctx, "user1") + require.NoError(t, err) + userClientA := testinfra.GetClient(token) + token, err = env.CreateUser(ctx, "user2") + require.NoError(t, err) + userClientB := testinfra.GetClient(token) + anonymousClient := testinfra.GetClient("") + + type triResponse struct { + codes []int + bodys [][]byte + } + tripleDo := func(method string, url string, body string) *triResponse { + rp := &triResponse{} + for _, client := range []*http.Client{anonymousClient, userClientA, userClientB} { + buf := bytes.NewBuffer([]byte(body)) + req, err := http.NewRequest(method, url, buf) + require.NoError(t, err) + resp, err := client.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + bodyBytes, err := io.ReadAll(resp.Body) + require.NoError(t, err) + rp.codes = append(rp.codes, resp.StatusCode) + rp.bodys = append(rp.bodys, bodyBytes) + } + return rp + } + + // create model anonymous + data := `{"name":"test1","nickname":"","namespace":"user1","license":"apache-2.0","description":"","private":false}` + req, err := http.NewRequest( + "POST", "http://localhost:9091/api/v1/models", bytes.NewBuffer([]byte(data)), + ) + require.NoError(t, err) + resp, err := anonymousClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, 401, resp.StatusCode) + + // create model login + req, err = http.NewRequest( + "POST", "http://localhost:9091/api/v1/models", bytes.NewBuffer([]byte(data)), + ) + require.NoError(t, err) + resp, err = userClientA.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, 200, resp.StatusCode) + + // get models, all 3 clients should be able to see new model + rp := tripleDo( + "GET", "http://localhost:9091/api/v1/models?page=1&per=16&search=&sort=trending", "", + ) + require.Equal(t, []int{200, 200, 200}, rp.codes) + for _, b := range rp.bodys { + require.Equal(t, int64(1), gjson.GetBytes(b, "total").Int()) + require.Equal(t, "test1", gjson.GetBytes(b, "data.0.name").String()) + } + + // get model detail, all 3 clients should be able to access + rp = tripleDo( + "GET", "http://localhost:9091/api/v1/models/user1/test1", "", + ) + require.Equal(t, []int{200, 200, 200}, rp.codes) + for _, b := range rp.bodys { + require.Equal(t, "test1", gjson.GetBytes(b, "data.name").String()) + // FIXME: user email leak + require.Equal(t, "user1@csg.com", gjson.GetBytes(b, "data.user.email").String()) + } + + // create private model + data = `{"name":"test2","nickname":"","namespace":"user1","license":"apache-2.0","description":"","private":true}` + req, err = http.NewRequest( + "POST", "http://localhost:9091/api/v1/models", bytes.NewBuffer([]byte(data)), + ) + require.NoError(t, err) + resp, err = userClientA.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, 200, resp.StatusCode) + + // get model, only user A can see the new model in list + rp = tripleDo( + "GET", "http://localhost:9091/api/v1/models?page=1&per=16&search=&sort=trending", "", + ) + require.Equal(t, []int{200, 200, 200}, rp.codes) + for i, b := range rp.bodys { + if i == 1 { + require.Equal(t, int64(2), gjson.GetBytes(b, "total").Int()) + require.Equal(t, "test2", gjson.GetBytes(b, "data.1.name").String()) + } else { + require.Equal(t, int64(1), gjson.GetBytes(b, "total").Int()) + require.Equal(t, "test1", gjson.GetBytes(b, "data.0.name").String()) + } + } + + // get model detail, only user A can access the new model + rp = tripleDo( + "GET", "http://localhost:9091/api/v1/models/user1/test2", "", + ) + require.Equal(t, []int{403, 200, 403}, rp.codes) + for i, b := range rp.bodys { + if i == 1 { + require.Equal(t, "test2", gjson.GetBytes(b, "data.name").String()) + } + } + + // update model file, public + rp = tripleDo( + "PUT", "http://localhost:9091/api/v1/models/user1/test1/raw/README.md", + `{"content":"Ci0tLQpsaWNlbnNlOiBnZW1tYQotLS0Keg==","message":"Update README.md","branch":"main","new_branch":"main","sha":"4d1cb859ec3b14226026a965517d0e0c07c9883e"}`, + ) + require.Equal(t, []int{401, 200, 500}, rp.codes) + + // update model file, private + rp = tripleDo( + "PUT", "http://localhost:9091/api/v1/models/user1/test2/raw/README.md", + `{"content":"Ci0tLQpsaWNlbnNlOiBnZW1tYQotLS0Keg==","message":"Update README.md","branch":"main","new_branch":"main","sha":"4d1cb859ec3b14226026a965517d0e0c07c9883e"}`, + ) + require.Equal(t, []int{401, 200, 500}, rp.codes) + + // delete model, public + rp = tripleDo("DELETE", "http://localhost:9091/api/v1/models/user1/test1", "") + require.Equal(t, []int{401, 200, 500}, rp.codes) + + // delete model, private + rp = tripleDo("DELETE", "http://localhost:9091/api/v1/models/user1/test2", "") + require.Equal(t, []int{401, 200, 500}, rp.codes) + + // model list empty + rp = tripleDo( + "GET", "http://localhost:9091/api/v1/models?page=1&per=16&search=&sort=trending", "", + ) + require.Equal(t, []int{200, 200, 200}, rp.codes) + for _, b := range rp.bodys { + require.Equal(t, int64(0), gjson.GetBytes(b, "total").Int()) + } + +} + +func gitClone(url, dir string) error { + cmd := exec.Command("git", "clone", url, dir) + // cmd.Env = os.Environ() + // cmd.Env = append(cmd.Env, "GIT_CURL_VERBOSE=1") + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + return cmd.Run() +} + +func gitCommitAndPush(dir string) error { + emailCheck := exec.Command("git", "-C", dir, "config", "--get", "user.email") + emailCheckOutput, err := emailCheck.Output() + if err != nil || string(emailCheckOutput) == "" { + err = exec.Command("git", "-C", dir, "config", "user.email", "you@example.com").Run() + if err != nil { + return err + } + } + + nameCheck := exec.Command("git", "-C", dir, "config", "--get", "user.name") + nameCheckOutput, err := nameCheck.Output() + if err != nil || string(nameCheckOutput) == "" { + err = exec.Command("git", "-C", dir, "config", "user.name", "you").Run() + if err != nil { + return err + } + } + + err = exec.Command("git", "-C", dir, "add", ".").Run() + if err != nil { + return err + } + cmd := exec.Command("git", "-C", dir, "commit", "-m", "Update") + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + err = cmd.Run() + if err != nil { + return err + } + cmd = exec.Command("git", "-C", dir, "push") + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, "GIT_CURL_VERBOSE=1") + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + return cmd.Run() +} + +func TestIntegrationModel_Git(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + ctx := context.TODO() + env, err := testinfra.StartTestEnv() + defer func() { _ = env.Shutdown(ctx) }() + require.NoError(t, err) + token, err := env.CreateUser(ctx, "user1") + require.NoError(t, err) + userClientA := testinfra.GetClient(token) + _, err = env.CreateUser(ctx, "user2") + require.NoError(t, err) + + data := `{"name":"test1","nickname":"","namespace":"user1","license":"apache-2.0","description":"","private":false}` + req, err := http.NewRequest( + "POST", "http://localhost:9091/api/v1/models", bytes.NewBuffer([]byte(data)), + ) + require.NoError(t, err) + resp, err := userClientA.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, 200, resp.StatusCode) + resp, err = userClientA.Get("http://localhost:9091/api/v1/models/user1/test1") + require.NoError(t, err) + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + var model types.Model + err = json.Unmarshal([]byte(gjson.GetBytes(body, "data").Raw), &model) + require.NoError(t, err) + cloneURL := model.Repository.HTTPCloneURL + + // git clone repo without token + err = gitClone(cloneURL, "model_clone_1") + require.NoError(t, err) + defer os.RemoveAll("model_clone_1") + // change and git push + err = exec.Command("cp", "Makefile", "model_clone_1/Makefile").Run() + require.NoError(t, err) + err = gitCommitAndPush("model_clone_1") + require.Error(t, err) + + // clone and push + for _, user := range []string{"user1", "user2"} { + token, err := env.CreateAccessToken(ctx, user, types.AccessTokenAppGit) + require.NoError(t, err) + url := strings.ReplaceAll(cloneURL, "http://", fmt.Sprintf("http://%s:%s@", user, token)) + dir := "model_clone_" + user + err = gitClone(url, dir) + require.NoError(t, err) + defer os.RemoveAll(dir) + // change and push + err = exec.Command("cp", "Makefile", dir+"/Makefile").Run() + require.NoError(t, err) + // test lfs + err = exec.Command("cp", "tests/0.parquet", dir+"/0.parquet").Run() + require.NoError(t, err) + + err = gitCommitAndPush(dir) + if user == "user1" { + require.NoError(t, err) + } else { + require.Error(t, err) + } + } + + // clone and validate parquet file + token, err = env.CreateAccessToken(ctx, "user1", types.AccessTokenAppGit) + require.NoError(t, err) + url := strings.ReplaceAll(cloneURL, "http://", fmt.Sprintf("http://%s:%s@", "user1", token)) + dir := "model_clone_user1_again" + err = gitClone(url, dir) + require.NoError(t, err) + defer os.RemoveAll(dir) + reader := parquet.NewParquetGoReader(&parquet.OSFileClient{}, otel.Tracer("test"), "") + columns, columnTypes, pdata, total, err := reader.RowsWithCount( + context.TODO(), + []string{dir + "/0.parquet"}, + 30, + 0, + ) + require.NoError(t, err) + require.Equal(t, []string{"Id", "Name"}, columns) + require.Equal(t, []string{"INT64", "INT64"}, columnTypes) + require.Equal(t, int64(20), total) + var current int64 + for _, row := range pdata { + id := cast.ToInt64(row[0]) + name := cast.ToInt64(row[1]) + require.Equal(t, current, id) + require.Equal(t, current, name) + current += 1 + } + require.Equal(t, int64(20), current) +} diff --git a/tests/gitaly.toml b/tests/gitaly.toml new file mode 100644 index 00000000..c1e1d753 --- /dev/null +++ b/tests/gitaly.toml @@ -0,0 +1,13 @@ +listen_addr = "0.0.0.0:9876" +bin_dir = "/usr/bin" + +[[storage]] +name = "default" +path = "/home/git/repositories" + +[gitlab] +url = "http://host.docker.internal:9091" +secret = "signing-key" + +[git] +use_bundled_binaries = true \ No newline at end of file diff --git a/tests/gitaly_github.toml b/tests/gitaly_github.toml new file mode 100644 index 00000000..56f3956a --- /dev/null +++ b/tests/gitaly_github.toml @@ -0,0 +1,13 @@ +listen_addr = "0.0.0.0:9876" +bin_dir = "/usr/bin" + +[[storage]] +name = "default" +path = "/home/git/repositories" + +[gitlab] +url = "http://172.17.0.1:9091" +secret = "signing-key" + +[git] +use_bundled_binaries = true \ No newline at end of file diff --git a/tests/main.go b/tests/main.go new file mode 100644 index 00000000..1d123951 --- /dev/null +++ b/tests/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "context" + "log" + "os" + "os/signal" + + "opencsg.com/csghub-server/tests/testinfra" +) + +func main() { + ctx := context.Background() + env, err := testinfra.StartTestEnv() + defer func() { _ = env.Shutdown(ctx) }() + if err != nil { + panic(err) + } + quit := make(chan os.Signal, 1) + signal.Notify(quit, os.Interrupt) + <-quit + log.Println("Shutdown Server ...") +} diff --git a/tests/testinfra/testinfra.go b/tests/testinfra/testinfra.go new file mode 100644 index 00000000..ec5409d7 --- /dev/null +++ b/tests/testinfra/testinfra.go @@ -0,0 +1,345 @@ +package testinfra + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/johannesboyne/gofakes3" + "github.com/johannesboyne/gofakes3/backend/s3mem" + "github.com/spf13/cast" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/sdk/client" + "go.temporal.io/sdk/log" + "go.temporal.io/server/temporaltest" + "google.golang.org/protobuf/types/known/durationpb" + "opencsg.com/csghub-server/api/httpbase" + api_router "opencsg.com/csghub-server/api/router" + api_workflow "opencsg.com/csghub-server/api/workflow" + "opencsg.com/csghub-server/builder/store/database" + "opencsg.com/csghub-server/builder/temporal" + "opencsg.com/csghub-server/common/config" + "opencsg.com/csghub-server/common/tests" + "opencsg.com/csghub-server/common/types" + dv_router "opencsg.com/csghub-server/dataviewer/router" + user_router "opencsg.com/csghub-server/user/router" + user_workflow "opencsg.com/csghub-server/user/workflow" +) + +var chMu sync.Mutex + +func chProjectRoot() { + chMu.Lock() + defer chMu.Unlock() + for { + _, err := os.Stat("builder/store/database/migrations") + if err != nil { + err = os.Chdir("../") + if err != nil { + panic(err) + } + continue + } + return + } +} + +type TestEnv struct { + userStore database.UserStore + accessTokenStore database.AccessTokenStore + temporalServer *temporaltest.TestServer + userServer *httpbase.GracefulServer + datasetViewerServer *httpbase.GracefulServer + apiServer *httpbase.GracefulServer + s3Server *httptest.Server +} + +func (t *TestEnv) Shutdown(ctx context.Context) error { + var err error + if t.temporalServer != nil { + t.temporalServer.Stop() + } + if t.userServer != nil { + err = errors.Join(err, t.userServer.Shutdown(ctx)) + } + if t.datasetViewerServer != nil { + err = errors.Join(err, t.datasetViewerServer.Shutdown(ctx)) + } + if t.apiServer != nil { + err = errors.Join(err, t.apiServer.Shutdown(ctx)) + } + if t.s3Server != nil { + t.s3Server.Close() + } + return err +} + +func (t *TestEnv) CreateAccessToken(ctx context.Context, userName string, app types.AccessTokenApp) (string, error) { + uw, err := t.userStore.FindByUsername(ctx, userName) + if err != nil { + return "", err + } + token := cast.ToString(time.Now().UnixNano()) + err = t.accessTokenStore.Create(ctx, &database.AccessToken{ + Token: token, + User: &uw, + UserID: uw.ID, + IsActive: true, + ExpiredAt: time.Now().Add(10 * time.Hour), + Application: app, + }) + if err != nil { + return "", err + } + return token, nil +} + +// Create a new user in database and return an access token +func (t *TestEnv) CreateUser(ctx context.Context, userName string) (string, error) { + namespace := &database.Namespace{ + Path: userName, + } + user := &database.User{ + Username: userName, + NickName: userName, + Email: userName + "@csg.com", + UUID: userName + "uuid", + RegProvider: "casdoor", + EmailVerified: true, + RoleMask: "user", + } + err := t.userStore.Create(ctx, user, namespace) + if err != nil { + return "", err + } + token, err := t.CreateAccessToken(ctx, userName, types.AccessTokenAppCSGHub) + if err != nil { + return "", err + } + return token, nil +} + +func StartTestEnv() (*TestEnv, error) { + env := &TestEnv{} + chProjectRoot() + ctx := context.TODO() + config.SetConfigFile("common/config/test.toml") + cfg, err := config.LoadConfig() + if err != nil { + return nil, err + } + // create test postgres container + _, dsn := tests.CreateTestDB("csghub_integration_test" + cast.ToString(time.Now().Unix())) + cfg.Database.DSN = dsn + dbConfig := database.DBConfig{ + Dialect: database.DatabaseDialect(cfg.Database.Driver), + DSN: cfg.Database.DSN + "sslmode=disable", + } + // init db from test container + database.InitDB(dbConfig) + env.userStore = database.NewUserStoreWithDB(database.GetDB()) + env.accessTokenStore = database.NewAccessTokenStoreWithDB(database.GetDB()) + + // create test gitaly + configFile := "tests/gitaly.toml" + // http://host.docker.internal:9091 is not accessible in github CI, + // use http://172.17.0.1:9091 + if os.Getenv("GITHUB") == "true" { + configFile = "tests/gitaly_github.toml" + } + req := testcontainers.ContainerRequest{ + Name: "csghub_test_gitaly", + Image: "opencsg-registry.cn-beijing.cr.aliyuncs.com/opencsg_public/gitaly:v17.5.0", + ExposedPorts: []string{"9876/tcp"}, + User: "root", + Env: map[string]string{"GITALY_CONFIG_FILE": "/etc/gitaly.toml"}, + Files: []testcontainers.ContainerFile{ + { + HostFilePath: configFile, + ContainerFilePath: "/etc/gitaly.toml", + FileMode: 0o700, + }, + }, + Cmd: []string{"bash", "-c", "mkdir -p /home/git/repositories && rm -rf /srv/gitlab-shell/hooks/* && touch /srv/gitlab-shell/.gitlab_shell_secret && exec /scripts/process-wrapper"}, + } + gitalyContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + Reuse: true, + }) + if err != nil { + return env, err + } + time.Sleep(1 * time.Second) + gitalyLocalPort, err := gitalyContainer.MappedPort(ctx, "9876") + if err != nil { + return env, err + } + tmp := strings.Split(string(gitalyLocalPort), "/") + cfg.GitalyServer.Address = strings.ReplaceAll(cfg.GitalyServer.Address, ":9876", ":"+tmp[0]) + + // create local temporal + env.temporalServer = temporaltest.NewServer() + cfg.WorkFLow.Endpoint = env.temporalServer.GetFrontendHostPort() + + nsclient, err := client.NewNamespaceClient(client.Options{HostPort: cfg.WorkFLow.Endpoint}) + if err != nil { + return env, err + } + err = nsclient.Register(ctx, &workflowservice.RegisterNamespaceRequest{ + Namespace: "default", + WorkflowExecutionRetentionPeriod: &durationpb.Duration{Seconds: 1000000000}, + }) + if err != nil { + return env, err + } + + // create local s3 + backend := s3mem.New() + faker := gofakes3.New( + backend, gofakes3.WithHostBucket(false), + gofakes3.WithLogger(gofakes3.GlobalLog()), + ) + ts := httptest.NewServer(faker.Server()) + env.s3Server = ts + cfg.S3.Endpoint = strings.TrimPrefix(ts.URL, "http://") + err = backend.CreateBucket(cfg.S3.Bucket) + if err != nil { + return env, err + } + + // start redis + req = testcontainers.ContainerRequest{ + Image: "opencsg-registry.cn-beijing.cr.aliyuncs.com/opencsg_public/redis:7.2.5", + ExposedPorts: []string{"6379/tcp"}, + WaitingFor: wait.ForLog("Ready to accept connections"), + } + rc, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + if err != nil { + return env, err + } + redisLocalPort, err := rc.MappedPort(ctx, "6379") + if err != nil { + return env, err + } + cfg.Redis.Endpoint = strings.ReplaceAll(cfg.Redis.Endpoint, "6379", redisLocalPort.Port()) + + // start user service + err = user_workflow.StartWorker(cfg) + if err != nil { + return env, err + } + ur, err := user_router.NewRouter(cfg) + if err != nil { + return env, err + } + env.userServer = httpbase.NewGracefulServer( + httpbase.GraceServerOpt{ + Port: cfg.User.Port, + }, + ur, + ) + go env.userServer.Run() + + // start dataset viewer service + client, err := temporal.NewClient(client.Options{ + HostPort: cfg.WorkFLow.Endpoint, + Logger: log.NewStructuredLogger(slog.Default()), + }, "dataset-viewer") + if err != nil { + return env, err + } + dr, err := dv_router.NewDataViewerRouter(cfg, client) + if err != nil { + return env, err + } + env.datasetViewerServer = httpbase.NewGracefulServer( + httpbase.GraceServerOpt{ + Port: cfg.DataViewer.Port, + }, + dr, + ) + go env.datasetViewerServer.Run() + + // start api + as, err := api_router.NewServer(cfg, false) + if err != nil { + return env, err + } + env.apiServer = as + go env.apiServer.Run() + + err = api_workflow.StartWorkflow(cfg) + if err != nil { + return env, err + } + var success bool + for i := 0; i < 10; i++ { + time.Sleep(1 * time.Second) + resp, err := http.Get("http://localhost:9091/api/v1/models?page=1&per=1") + if err != nil { + fmt.Println("health check failed, retry in 1 second") + continue + } else { + defer resp.Body.Close() + if resp.StatusCode != 200 { + fmt.Println("health check failed, retry in 1 second") + continue + } + success = true + break + } + } + if !success { + return env, errors.New("api health check failed") + } + return env, nil +} + +func StartTestServer(t *testing.T) { + ctx := context.Background() + env, err := StartTestEnv() + require.NoError(t, err) + t.Cleanup(func() { + if err := env.Shutdown(ctx); err != nil { + t.Error("Timed out waiting on server to shut down") + } + }) +} + +type AddHeaderTransport struct { + t http.RoundTripper + headers map[string]string +} + +func (adt *AddHeaderTransport) RoundTrip(req *http.Request) (*http.Response, error) { + for k, v := range adt.headers { + req.Header.Add(k, v) + } + return adt.t.RoundTrip(req) +} + +func GetClient(accessToken string) *http.Client { + header := map[string]string{} + if accessToken != "" { + header["Authorization"] = fmt.Sprintf("Bearer %s", accessToken) + } + return &http.Client{Transport: &AddHeaderTransport{ + t: http.DefaultTransport, + headers: header, + }} +} diff --git a/tests/testserver/testserver.go b/tests/testserver/testserver.go new file mode 100644 index 00000000..47d25065 --- /dev/null +++ b/tests/testserver/testserver.go @@ -0,0 +1,32 @@ +package testserver + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "os/signal" + + "opencsg.com/csghub-server/tests/testinfra" +) + +func StartTestServer() { + ctx := context.Background() + env, err := testinfra.StartTestEnv() + if err != nil { + err = errors.Join(err, env.Shutdown(ctx)) + panic(err) + } + defer func() { + err = env.Shutdown(ctx) + if err != nil { + fmt.Println("shutdown test env error: ", err) + } + }() + + quit := make(chan os.Signal, 1) + signal.Notify(quit, os.Interrupt) + <-quit + log.Println("Shutdown Server ...") +} diff --git a/user/workflow/worker.go b/user/workflow/worker.go index 89d9a880..b5b5a4c4 100644 --- a/user/workflow/worker.go +++ b/user/workflow/worker.go @@ -2,6 +2,7 @@ package workflow import ( "fmt" + "log/slog" "go.temporal.io/sdk/client" "go.temporal.io/sdk/worker" @@ -18,6 +19,7 @@ func StartWorker(config *config.Config) error { var err error wfClient, err = client.Dial(client.Options{ HostPort: config.WorkFLow.Endpoint, + Logger: slog.Default(), }) if err != nil { return fmt.Errorf("unable to create workflow client, error:%w", err)