diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..7aa7643c1 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,104 @@ +# AGENTS Guidelines for This Repository + +This repository is a Go project following the **microservice** architecture design. Services including: + +- **API**: The API service handles HTTP requests and responses. It is the entry point for external clients to interact with the system. +- **User**: The User service handles user-related operations, such as user registration, login, and profile management. All requests are proxied to this service from the API service. +- **Accounting**: The Accounting service handles accounting-related operations, such as recording user token or hardware resource usage, or updating user balances. All requests are proxied to this service from the API service. +- **Moderation**: The Moderation service handles content moderation operations, such as flagging inappropriate text or images. All requests are proxied to this service from the API service. +- **DataViewer**: The DataViewer service handles dataset preview operations, such as fetching dataset metadata or previewing dataset files. All requests are proxied to this service from the API service. +- **Notification**: The Notification service handles sending notifications to users, such as email or push notifications. All requests are proxied to this service from the API service. +- **Payment**: The Payment service handles payment operations, such as processing payments or refunding payments. All requests are proxied to this service from the API service. +- **AIGateway**: The AIGateway service handles AI model inference operations, such as running AI models or generating AI outputs. It's another entry point for external clients to interact with the AI models. +- **Runner**: The Runner service is a bridge between api service and Kubernetes cluster. It handles deployment of models, spaces. +- **LogCollector**: The LogCollector service handles collecting logs from Kubernetes cluster. All logs are sent to this service from the Runner service and API service. + +# Structure +Every service follows the layered architecture design: handler -> component -> builder (database, rpc, git, etc.). + +- The handler layer handles HTTP requests and responses. +- The component layer handles business logic and coordinates between different layers. +- The builder layer handles low-level operations, such as database access, RPC calls, or Git operations. +- Every golang file should have a corresponding `*_test.go` file for unit tests. + +Folders relative to the root of the repository for each service: + +| Service | Folder | +|---------|--------| +| API | api | +| User | user | +| Accounting | accounting | +| Moderation | moderation | +| DataViewer | dataviewer | +| Notification | notification | +| Payment | payment | +| AIGateway | aigateway | +| Runner | runner | +| LogCollector | logcollector | + +## Examples + +### Router + +- `api/router/api.go` is an example of a router that registers HTTP routes and their corresponding handlers for common functionality across services. +- `accounting/router/api.go` is an example of a router that registers HTTP routes and their corresponding handlers for the Accounting service. +- `runner/router/api.go` is an example of a router that registers HTTP routes and their corresponding handlers for the runner service. + +### Handler Layer + +- `api/handler/space.go` is an example of a handler that deals with space-related HTTP requests. +- `api/handler/evaluation.go` is an example of a handler that deals with evaluation-related HTTP requests. + +### Component Layer + +- `component/space.go` is an example of a component that deals with space-related business logic. +- `component/evaluation.go` is an example of a component that deals with evaluation-related business logic. + +### Database Builder Layer + +- `builder/store/database/space.go` is an example of a builder that deals with space-related database operations. + +### Database Migration + +- `builder/store/database/migrations/20240201061926_create_spaces.go` is an example of a database migration script that creates a space table. +- use `go run cmd/csghub-server/main.go migration create_go` to generate a go database migration script. +- use `go run cmd/csghub-server/main.go migration create_sql` to generate a sql database migration script. + +### Space Deploy + +- `builder/deploy/deployer.go` create build and deploy task in database, then create temporal workflow to run the task. +- `api/workflow/activity/deploy_activity.go` impletements temporal activities to run the build and deploy task by call runner api. +- `runner/handler/imagebuilder.go` implements runner api to trigger image builder process by call image builder component. +- `runner/component/imagebuilder.go` implements runner component to trigger deploy process by call knative api. +- `runner/handler/service.go` implements runner api to trigger deploy process by call deploy component. +- `runner/component/service.go` implements runner component to trigger deploy process by call knative api. +- `docker/spaces/builder/Dockerfile*` are Dockerfile that builds the space image. + +## Code Style & Conventions: + +- Each layer's interface should only expose data structures defined within its own layer or common type definitions from the common.types package. For example, interfaces in the Component layer (such as UserComponent) should not return data structures from the underlying database layer (such as database.User structure), as the database layer is considered lower-level than the component layer. +- Write unit tests for new code. +- Use struct data types instead of primitive types for function parameters and return values. +- All variables should be named in camelCase. +- Variables should be declared at the smallest possible scope under `common/types`. + +### Do + +### Do Not + +## Testing + +- Use `make mock_gen GO_TAGS={go.buildTags}` to generate mock implementations for the interfaces. +- Use `make test GO_TAGS={go.buildTags}` to run all tests in project. +- Mock dependencies (e.g., database, RPC clients) using tools like `mockery`. + +## Tools + +- Search `Makefile` for running, building, testing, and linting tools. +- Swagger doc is generated by `swag` tool, and it will be served by handler layer. + +## Commit & Pull Request Guidelines: + +- Each PR must include a clear description of the changes made and their impact, including root cause analysis if applicable, and solution details, and local test result. + +## Specific Instructions diff --git a/_mocks/opencsg.com/csghub-server/builder/store/database/mock_DeployTaskStore.go b/_mocks/opencsg.com/csghub-server/builder/store/database/mock_DeployTaskStore.go index 90f164c25..e7071085e 100644 --- a/_mocks/opencsg.com/csghub-server/builder/store/database/mock_DeployTaskStore.go +++ b/_mocks/opencsg.com/csghub-server/builder/store/database/mock_DeployTaskStore.go @@ -618,6 +618,65 @@ func (_c *MockDeployTaskStore_GetLatestDeployBySpaceID_Call) RunAndReturn(run fu return _c } +// GetLatestDeploysBySpaceIDs provides a mock function with given fields: ctx, spaceIDs +func (_m *MockDeployTaskStore) GetLatestDeploysBySpaceIDs(ctx context.Context, spaceIDs []int64) (map[int64]*database.Deploy, error) { + ret := _m.Called(ctx, spaceIDs) + + if len(ret) == 0 { + panic("no return value specified for GetLatestDeploysBySpaceIDs") + } + + var r0 map[int64]*database.Deploy + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, []int64) (map[int64]*database.Deploy, error)); ok { + return rf(ctx, spaceIDs) + } + if rf, ok := ret.Get(0).(func(context.Context, []int64) map[int64]*database.Deploy); ok { + r0 = rf(ctx, spaceIDs) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[int64]*database.Deploy) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, []int64) error); ok { + r1 = rf(ctx, spaceIDs) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockDeployTaskStore_GetLatestDeploysBySpaceIDs_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetLatestDeploysBySpaceIDs' +type MockDeployTaskStore_GetLatestDeploysBySpaceIDs_Call struct { + *mock.Call +} + +// GetLatestDeploysBySpaceIDs is a helper method to define mock.On call +// - ctx context.Context +// - spaceIDs []int64 +func (_e *MockDeployTaskStore_Expecter) GetLatestDeploysBySpaceIDs(ctx interface{}, spaceIDs interface{}) *MockDeployTaskStore_GetLatestDeploysBySpaceIDs_Call { + return &MockDeployTaskStore_GetLatestDeploysBySpaceIDs_Call{Call: _e.mock.On("GetLatestDeploysBySpaceIDs", ctx, spaceIDs)} +} + +func (_c *MockDeployTaskStore_GetLatestDeploysBySpaceIDs_Call) Run(run func(ctx context.Context, spaceIDs []int64)) *MockDeployTaskStore_GetLatestDeploysBySpaceIDs_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].([]int64)) + }) + return _c +} + +func (_c *MockDeployTaskStore_GetLatestDeploysBySpaceIDs_Call) Return(_a0 map[int64]*database.Deploy, _a1 error) *MockDeployTaskStore_GetLatestDeploysBySpaceIDs_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockDeployTaskStore_GetLatestDeploysBySpaceIDs_Call) RunAndReturn(run func(context.Context, []int64) (map[int64]*database.Deploy, error)) *MockDeployTaskStore_GetLatestDeploysBySpaceIDs_Call { + _c.Call.Return(run) + return _c +} + // GetNewTaskAfter provides a mock function with given fields: ctx, currentDeployTaskID func (_m *MockDeployTaskStore) GetNewTaskAfter(ctx context.Context, currentDeployTaskID int64) (*database.DeployTask, error) { ret := _m.Called(ctx, currentDeployTaskID) diff --git a/_mocks/opencsg.com/csghub-server/component/mock_SpaceComponent.go b/_mocks/opencsg.com/csghub-server/component/mock_SpaceComponent.go index bfdf072ae..f12405b70 100644 --- a/_mocks/opencsg.com/csghub-server/component/mock_SpaceComponent.go +++ b/_mocks/opencsg.com/csghub-server/component/mock_SpaceComponent.go @@ -915,6 +915,65 @@ func (_c *MockSpaceComponent_Status_Call) RunAndReturn(run func(context.Context, return _c } +// StatusByPaths provides a mock function with given fields: ctx, paths +func (_m *MockSpaceComponent) StatusByPaths(ctx context.Context, paths []string) (map[string]string, error) { + ret := _m.Called(ctx, paths) + + if len(ret) == 0 { + panic("no return value specified for StatusByPaths") + } + + var r0 map[string]string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, []string) (map[string]string, error)); ok { + return rf(ctx, paths) + } + if rf, ok := ret.Get(0).(func(context.Context, []string) map[string]string); ok { + r0 = rf(ctx, paths) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[string]string) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, []string) error); ok { + r1 = rf(ctx, paths) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockSpaceComponent_StatusByPaths_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'StatusByPaths' +type MockSpaceComponent_StatusByPaths_Call struct { + *mock.Call +} + +// StatusByPaths is a helper method to define mock.On call +// - ctx context.Context +// - paths []string +func (_e *MockSpaceComponent_Expecter) StatusByPaths(ctx interface{}, paths interface{}) *MockSpaceComponent_StatusByPaths_Call { + return &MockSpaceComponent_StatusByPaths_Call{Call: _e.mock.On("StatusByPaths", ctx, paths)} +} + +func (_c *MockSpaceComponent_StatusByPaths_Call) Run(run func(ctx context.Context, paths []string)) *MockSpaceComponent_StatusByPaths_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].([]string)) + }) + return _c +} + +func (_c *MockSpaceComponent_StatusByPaths_Call) Return(_a0 map[string]string, _a1 error) *MockSpaceComponent_StatusByPaths_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockSpaceComponent_StatusByPaths_Call) RunAndReturn(run func(context.Context, []string) (map[string]string, error)) *MockSpaceComponent_StatusByPaths_Call { + _c.Call.Return(run) + return _c +} + // Stop provides a mock function with given fields: ctx, namespace, name, deleteSpace func (_m *MockSpaceComponent) Stop(ctx context.Context, namespace string, name string, deleteSpace bool) error { ret := _m.Called(ctx, namespace, name, deleteSpace) diff --git a/api/workflow/activity/deploy_activity.go b/api/workflow/activity/deploy_activity.go index c64d59984..99f72652b 100644 --- a/api/workflow/activity/deploy_activity.go +++ b/api/workflow/activity/deploy_activity.go @@ -470,8 +470,6 @@ func (a *DeployActivity) createBuildRequest(ctx context.Context, task *database. return nil, fmt.Errorf("invalid repository path format: %s", repoInfo.Path) } - sdkVersion := a.determineSDKVersion(repoInfo) - lastCommitReq := gitserver.GetRepoLastCommitReq{ RepoType: types.RepositoryType(repoInfo.RepoType), Namespace: pathParts[0], @@ -490,7 +488,7 @@ func (a *DeployActivity) createBuildRequest(ctx context.Context, task *database. PythonVersion: "3.10", Sdk: repoInfo.Sdk, DriverVersion: repoInfo.DriverVersion, - Sdk_version: sdkVersion, + Sdk_version: repoInfo.SdkVersion, SpaceURL: repoInfo.HTTPCloneURL, GitRef: task.Deploy.GitBranch, UserId: accessToken.User.Username, @@ -588,21 +586,6 @@ func (a *DeployActivity) createDeployRequest(ctx context.Context, task *database }, nil } -func (a *DeployActivity) determineSDKVersion(repoInfo common.RepoInfo) string { - if repoInfo.SdkVersion != "" { - return repoInfo.SdkVersion - } - - switch repoInfo.Sdk { - case types.GRADIO.Name: - return types.GRADIO.Version - case types.STREAMLIT.Name: - return types.STREAMLIT.Version - default: - return "" - } -} - func (a *DeployActivity) parseHardware(input string) string { if strings.Contains(input, "GPU") || strings.Contains(input, "NVIDIA") { return "gpu" diff --git a/api/workflow/activity/deploy_activity_test.go b/api/workflow/activity/deploy_activity_test.go index c9f5012f8..58574d7d0 100644 --- a/api/workflow/activity/deploy_activity_test.go +++ b/api/workflow/activity/deploy_activity_test.go @@ -7,6 +7,7 @@ import ( "time" "opencsg.com/csghub-server/builder/deploy/common" + "opencsg.com/csghub-server/builder/git/gitserver" "opencsg.com/csghub-server/builder/store/database" "opencsg.com/csghub-server/common/config" "opencsg.com/csghub-server/common/types" @@ -91,56 +92,41 @@ func setupTest(t *testing.T) *testEnv { } } -// TestActivities_determineSDKVersion tests the determineSDKVersion method -func TestActivities_determineSDKVersion(t *testing.T) { +// TestActivities_createBuildRequest tests the createBuildRequest method +func TestActivities_createBuildRequest(t *testing.T) { tester := setupTest(t) - // Test cases - testCases := []struct { - name string - repoInfo common.RepoInfo - expectedSDK string - }{ - { - name: "With explicit SDK version", - repoInfo: common.RepoInfo{ - SdkVersion: "custom-version", - Sdk: "some-other-sdk", - }, - expectedSDK: "custom-version", - }, - { - name: "With GRADIO SDK", - repoInfo: common.RepoInfo{ - SdkVersion: "", - Sdk: types.GRADIO.Name, - }, - expectedSDK: types.GRADIO.Version, - }, - { - name: "With STREAMLIT SDK", - repoInfo: common.RepoInfo{ - SdkVersion: "", - Sdk: types.STREAMLIT.Name, - }, - expectedSDK: types.STREAMLIT.Version, - }, - { - name: "With unknown SDK", - repoInfo: common.RepoInfo{ - SdkVersion: "", - Sdk: "unknown-sdk", - }, - expectedSDK: "", + task := &database.DeployTask{ + Deploy: &database.Deploy{ + UserID: 1, }, } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - sdkVersion := tester.activities.determineSDKVersion(tc.repoInfo) - require.Equal(t, tc.expectedSDK, sdkVersion) - }) + repoInfo := common.RepoInfo{ + Path: "org/space", + SdkVersion: "6.2.0", } + + tester.mockTokenStore.EXPECT().FindByUID(tester.ctx, task.Deploy.UserID).Return(&database.AccessToken{ + Token: "test-token", + User: &database.User{ + Username: "uname", + }, + }, nil) + + tester.mockGitServer.EXPECT().GetRepoLastCommit(tester.ctx, gitserver.GetRepoLastCommitReq{ + RepoType: types.RepositoryType(repoInfo.RepoType), + Namespace: "org", + Name: "space", + Ref: task.Deploy.GitBranch, + }).Return(&types.Commit{ + ID: "id", + }, nil) + + r, err := tester.activities.createBuildRequest(tester.ctx, task, repoInfo) + require.Nil(t, err) + require.Equal(t, r.Sdk_version, repoInfo.SdkVersion) + require.Equal(t, r.LastCommitID, "id") } // TestActivities_handleDeployError tests the handleDeployError method diff --git a/builder/store/database/deploy_task.go b/builder/store/database/deploy_task.go index 677a2af18..a1a4e4959 100644 --- a/builder/store/database/deploy_task.go +++ b/builder/store/database/deploy_task.go @@ -81,6 +81,7 @@ type DeployTaskStore interface { CreateDeploy(ctx context.Context, deploy *Deploy) error UpdateDeploy(ctx context.Context, deploy *Deploy) error GetLatestDeployBySpaceID(ctx context.Context, spaceID int64) (*Deploy, error) + GetLatestDeploysBySpaceIDs(ctx context.Context, spaceIDs []int64) (map[int64]*Deploy, error) CreateDeployTask(ctx context.Context, deployTask *DeployTask) error UpdateDeployTask(ctx context.Context, deployTask *DeployTask) error GetDeployTask(ctx context.Context, id int64) (*DeployTask, error) @@ -138,6 +139,33 @@ func (s *deployTaskStoreImpl) GetLatestDeployBySpaceID(ctx context.Context, spac return deploy, err } +// GetLatestDeploysBySpaceIDs gets the latest deploy for each space ID in a single query +func (s *deployTaskStoreImpl) GetLatestDeploysBySpaceIDs(ctx context.Context, spaceIDs []int64) (map[int64]*Deploy, error) { + if len(spaceIDs) == 0 { + return make(map[int64]*Deploy), nil + } + + var deploys []Deploy + err := s.db.Core.NewSelect(). + ColumnExpr("DISTINCT ON (space_id) *"). + Model(&deploys). + Where("space_id IN (?)", bun.In(spaceIDs)). + Order("space_id", "created_at DESC"). + Scan(ctx) + + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return nil, errorx.HandleDBError(err, nil) + } + + result := make(map[int64]*Deploy, len(deploys)) + for i := range deploys { + deploy := &deploys[i] + result[deploy.SpaceID] = deploy + } + + return result, nil +} + func (s *deployTaskStoreImpl) CreateDeployTask(ctx context.Context, deployTask *DeployTask) error { _, err := s.db.Core.NewInsert().Model(deployTask).Exec(ctx, deployTask) err = errorx.HandleDBError(err, nil) diff --git a/builder/store/database/deploy_task_test.go b/builder/store/database/deploy_task_test.go index d3258cdd7..fef1fbd57 100644 --- a/builder/store/database/deploy_task_test.go +++ b/builder/store/database/deploy_task_test.go @@ -653,3 +653,100 @@ func TestDeployTaskStore_DeleteDeployByID(t *testing.T) { err = store.DeleteDeployByID(ctx, 100, 999999) require.NotNil(t, err) } + +func TestDeployTaskStore_GetLatestDeploysBySpaceIDs(t *testing.T) { + db := tests.InitTestDB() + defer db.Close() + ctx := context.TODO() + store := database.NewDeployTaskStoreWithDB(db) + + // Test with empty spaceIDs + result, err := store.GetLatestDeploysBySpaceIDs(ctx, []int64{}) + require.Nil(t, err) + require.NotNil(t, result) + require.Equal(t, 0, len(result)) + + // Create test data: multiple deploys for different space IDs + // Space 100: 3 deploys (should return the latest) + // Space 200: 2 deploys (should return the latest) + // Space 300: 1 deploy (should return that one) + // Space 400: no deploys (should not appear in result) + + now := time.Now().UTC() + space100Deploys := []database.Deploy{ + {SpaceID: 100, DeployName: "space100-old", SvcName: "svc100-1", UserID: 1, RepoID: 1, GitPath: "test", GitBranch: "main", Template: "test", Hardware: "test"}, + {SpaceID: 100, DeployName: "space100-middle", SvcName: "svc100-2", UserID: 1, RepoID: 1, GitPath: "test", GitBranch: "main", Template: "test", Hardware: "test"}, + {SpaceID: 100, DeployName: "space100-latest", SvcName: "svc100-3", UserID: 1, RepoID: 1, GitPath: "test", GitBranch: "main", Template: "test", Hardware: "test"}, + } + + space200Deploys := []database.Deploy{ + {SpaceID: 200, DeployName: "space200-old", SvcName: "svc200-1", UserID: 1, RepoID: 2, GitPath: "test", GitBranch: "main", Template: "test", Hardware: "test"}, + {SpaceID: 200, DeployName: "space200-latest", SvcName: "svc200-2", UserID: 1, RepoID: 2, GitPath: "test", GitBranch: "main", Template: "test", Hardware: "test"}, + } + + space300Deploy := database.Deploy{ + SpaceID: 300, DeployName: "space300-single", SvcName: "svc300-1", UserID: 1, RepoID: 3, GitPath: "test", GitBranch: "main", Template: "test", Hardware: "test", + } + + // Create deploys with different timestamps + for i, dp := range space100Deploys { + err := store.CreateDeploy(ctx, &dp) + require.Nil(t, err) + // Set created_at to different times (oldest first) + _, err = db.BunDB.ExecContext(ctx, "UPDATE deploys SET created_at = ?, updated_at = ? WHERE id = ?", + now.Add(-time.Duration(3-i)*time.Hour), now.Add(-time.Duration(3-i)*time.Hour), dp.ID) + require.NoError(t, err) + } + + for i, dp := range space200Deploys { + err := store.CreateDeploy(ctx, &dp) + require.Nil(t, err) + // Set created_at to different times (oldest first) + _, err = db.BunDB.ExecContext(ctx, "UPDATE deploys SET created_at = ?, updated_at = ? WHERE id = ?", + now.Add(-time.Duration(2-i)*time.Hour), now.Add(-time.Duration(2-i)*time.Hour), dp.ID) + require.NoError(t, err) + } + + err = store.CreateDeploy(ctx, &space300Deploy) + require.Nil(t, err) + + // Test: Get latest deploys for space 100, 200, 300, 400 + spaceIDs := []int64{100, 200, 300, 400} + result, err = store.GetLatestDeploysBySpaceIDs(ctx, spaceIDs) + require.Nil(t, err) + require.NotNil(t, result) + + // Should have 3 results (space 400 has no deploys, so won't appear) + require.Equal(t, 3, len(result)) + + // Verify space 100 has the latest deploy + deploy100, exists := result[100] + require.True(t, exists) + require.NotNil(t, deploy100) + require.Equal(t, "space100-latest", deploy100.DeployName) + require.Equal(t, "svc100-3", deploy100.SvcName) + + // Verify space 200 has the latest deploy + deploy200, exists := result[200] + require.True(t, exists) + require.NotNil(t, deploy200) + require.Equal(t, "space200-latest", deploy200.DeployName) + require.Equal(t, "svc200-2", deploy200.SvcName) + + // Verify space 300 has its deploy + deploy300, exists := result[300] + require.True(t, exists) + require.NotNil(t, deploy300) + require.Equal(t, "space300-single", deploy300.DeployName) + require.Equal(t, "svc300-1", deploy300.SvcName) + + // Verify space 400 is not in the result (no deploys) + _, exists = result[400] + require.False(t, exists) + + // Test with only space IDs that don't exist + result, err = store.GetLatestDeploysBySpaceIDs(ctx, []int64{999, 998}) + require.Nil(t, err) + require.NotNil(t, result) + require.Equal(t, 0, len(result)) +} diff --git a/builder/store/database/space.go b/builder/store/database/space.go index b3b142eb9..dcad13bd6 100644 --- a/builder/store/database/space.go +++ b/builder/store/database/space.go @@ -88,6 +88,17 @@ func (s *spaceStoreImpl) FindByPath(ctx context.Context, namespace, name string) return nil, fmt.Errorf("select space by path, error: %w", err) } + err = s.db.Operator.Core.NewSelect(). + Model(resSpace.Repository). + WherePK(). + Relation("Tags", func(sq *bun.SelectQuery) *bun.SelectQuery { + return sq.Where("repository_tag.count > 0") + }). + Scan(ctx) + err = errorx.HandleDBError(err, errorx.Ctx(). + Set("path", fmt.Sprintf("%s/%s", namespace, name)), + ) + return resSpace, err } diff --git a/common/types/space.go b/common/types/space.go index abd965545..4f326b951 100644 --- a/common/types/space.go +++ b/common/types/space.go @@ -14,7 +14,7 @@ type SDKConfig struct { var ( GRADIO = SDKConfig{ Name: "gradio", - Version: "3.37.0", + Version: "6.2.0", Port: 7860, Image: "", } diff --git a/common/utils/common/string.go b/common/utils/common/string.go index cbfc20554..256ae81f8 100644 --- a/common/utils/common/string.go +++ b/common/utils/common/string.go @@ -49,3 +49,19 @@ func SHA256(s string) string { hashBytes := hash.Sum(nil) return hex.EncodeToString(hashBytes) } + +func SafeDeref(p *string) string { + if p != nil { + return *p + } + return "" +} + +// AllocStringPtr returns a pointer to a heap-allocated copy of the string +// This ensures the string has a stable lifetime and can be safely stored in structs +func AllocStringPtr(s string) *string { + // Allocate on heap explicitly to ensure stable lifetime + ptr := new(string) + *ptr = s + return ptr +} diff --git a/component/callback/git_callback.go b/component/callback/git_callback.go index f38f5a0fd..d0a37ad05 100644 --- a/component/callback/git_callback.go +++ b/component/callback/git_callback.go @@ -581,11 +581,10 @@ func getTagScopeByRepoType(repoType string) (types.TagScope, error) { tagScope = types.MCPTagScope case fmt.Sprintf("%ss", types.CodeRepo): tagScope = types.CodeTagScope + case fmt.Sprintf("%ss", types.SpaceRepo): + tagScope = types.SpaceTagScope default: return types.UnknownScope, fmt.Errorf("get tag scope by invalid repo type %s", repoType) - // TODO: support space - // case SpaceRepoType: - // tagScope = types.SpaceTagScope } return tagScope, nil diff --git a/component/callback/git_callback_test.go b/component/callback/git_callback_test.go index 904c20548..d1227803a 100644 --- a/component/callback/git_callback_test.go +++ b/component/callback/git_callback_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "opencsg.com/csghub-server/builder/git/gitserver" @@ -133,3 +134,53 @@ func TestGitCallbackComponent_SetRepoUpdateTime(t *testing.T) { }) } } + +func TestGitCallbackComponentImpl_UpdateRepoInfos(t *testing.T) { + ctx := mock.Anything + + req := &types.GiteaCallbackPushReq{ + Ref: "refs/heads/main", + Repository: types.GiteaCallbackPushReq_Repository{ + FullName: "models_namespace/repo", + }, + Commits: []types.GiteaCallbackPushReq_Commit{ + { + Added: []string{"README.md", "new_file.txt"}, + Removed: []string{"old_file.txt"}, + Modified: []string{"config.json"}, + }, + }, + } + repo := &database.Repository{ID: 1, Path: "namespace/repo"} + + t.Run("should update repo infos successfully", func(t *testing.T) { + gc := initializeTestGitCallbackComponent(context.Background(), t) + // Expectations for modifyFiles + gc.mocks.stores.RepoMock().EXPECT().FindByPath(ctx, types.ModelRepo, "namespace", "repo").Return(repo, nil) + modelInfo := &types.ModelInfo{} + gc.mocks.runtimeArchComponent.EXPECT().UpdateModelMetadata(ctx, repo).Return(modelInfo, nil) + gc.mocks.runtimeArchComponent.EXPECT().UpdateRuntimeFrameworkTag(ctx, modelInfo, repo).Return(nil) + + // Expectations for removeFiles + gc.mocks.tagComponent.EXPECT().UpdateLibraryTags(ctx, types.ModelTagScope, "namespace", "repo", "old_file.txt", "").Return(nil) + + // Expectations for addFiles + gc.mocks.tagComponent.EXPECT().UpdateLibraryTags(ctx, types.ModelTagScope, "namespace", "repo", "", "new_file.txt").Return(nil) + + getFileRawReq := gitserver.GetRepoInfoByPathReq{ + Namespace: "namespace", + Name: "repo", + Ref: "refs/heads/main", + Path: "README.md", + RepoType: types.ModelRepo, + } + gc.mocks.gitServer.EXPECT().GetRepoFileRaw(ctx, getFileRawReq).Return("---\nlicense: apache-2.0\nlanguage:\n - zh\n---\nreadme content", nil).Once() + gc.mocks.tagComponent.EXPECT().UpdateMetaTags(ctx, types.ModelTagScope, "namespace", "repo", "---\nlicense: apache-2.0\nlanguage:\n - zh\n---\nreadme content").Return(nil, nil) + gc.mocks.stores.RepoMock().EXPECT().FindByPath(ctx, types.ModelRepo, "namespace", "repo").Return(repo, nil) + gc.mocks.runtimeArchComponent.EXPECT().UpdateModelMetadata(ctx, repo).Return(modelInfo, nil) + gc.mocks.runtimeArchComponent.EXPECT().UpdateRuntimeFrameworkTag(ctx, modelInfo, repo).Return(nil) + + err := gc.UpdateRepoInfos(context.Background(), req) + assert.NoError(t, err) + }) +} diff --git a/component/space.go b/component/space.go index 05cc31ac9..fd1f78d91 100644 --- a/component/space.go +++ b/component/space.go @@ -54,6 +54,7 @@ type SpaceComponent interface { // FixHasEntryFile checks whether git repo has entry point file and update space's HasAppFile property in db FixHasEntryFile(ctx context.Context, s *database.Space) *database.Space Status(ctx context.Context, namespace, name string) (string, string, error) + StatusByPaths(ctx context.Context, paths []string) (map[string]string, error) Logs(ctx context.Context, namespace, name, since string) (*deploy.MultiLogReader, error) // HasEntryFile checks whether space repo has entry point file to run with HasEntryFile(ctx context.Context, space *database.Space) bool @@ -74,6 +75,14 @@ func (c *spaceComponentImpl) Create(ctx context.Context, req types.CreateSpaceRe req.DefaultBranch = types.MainBranch } + if req.Sdk == types.GRADIO.Name && len(req.SdkVersion) < 1 { + req.SdkVersion = types.GRADIO.Version + } + + if req.Sdk == types.STREAMLIT.Name && len(req.SdkVersion) < 1 { + req.SdkVersion = types.STREAMLIT.Version + } + req.Nickname = nickname req.RepoType = types.SpaceRepo req.Readme = generateReadmeData(req.License) @@ -389,6 +398,16 @@ func (c *spaceComponentImpl) Show(ctx context.Context, namespace, name, currentU } repository := common.BuildCloneInfo(c.config, space.Repository) + for _, tag := range space.Repository.Tags { + tags = append(tags, types.RepoTag{ + Name: tag.Name, + Category: tag.Category, + Group: tag.Group, + BuiltIn: tag.BuiltIn, + ShowName: tag.I18nKey, //ShowName: tag.ShowName, + }) + } + resSpace := &types.Space{ ID: space.ID, Name: space.Repository.Name, @@ -1049,6 +1068,64 @@ func (c *spaceComponentImpl) Status(ctx context.Context, namespace, name string) return spaceStatus.SvcName, spaceStatus.Status, err } +// StatusByPaths queries status for multiple spaces in a single query +// paths is a slice of space paths in the format "namespace/name" +// Returns a map where key is the path and value is the status string +func (c *spaceComponentImpl) StatusByPaths(ctx context.Context, paths []string) (map[string]string, error) { + if len(paths) == 0 { + return make(map[string]string), nil + } + + dbSpaces, err := c.spaceStore.ListByPath(ctx, paths) + if err != nil { + return nil, fmt.Errorf("failed to find spaces by paths, error: %w", err) + } + + pathToDBSpace := make(map[string]*database.Space) + spaceIDs := make([]int64, 0, len(dbSpaces)) + for i := range dbSpaces { + space := &dbSpaces[i] + path := space.Repository.Path + pathToDBSpace[path] = space + spaceIDs = append(spaceIDs, space.ID) + } + + // Get latest deploys for all spaces in one query + deployMap, err := c.deployTaskStore.GetLatestDeploysBySpaceIDs(ctx, spaceIDs) + if err != nil { + return nil, err + } + + result := make(map[string]string) + for _, path := range paths { + dbSpace, exists := pathToDBSpace[path] + if !exists { + // Space not found, set status to Empty + result[path] = SpaceStatusEmpty + continue + } + + if !dbSpace.HasAppFile { + if dbSpace.Sdk == types.NGINX.Name { + result[path] = SpaceStatusNoNGINXConf + } else { + result[path] = SpaceStatusNoAppFile + } + continue + } + + deploy, hasDeploy := deployMap[dbSpace.ID] + if !hasDeploy || deploy == nil { + result[path] = SpaceStatusStopped + continue + } + + result[path] = deployStatusCodeToString(deploy.Status) + } + + return result, nil +} + func (c *spaceComponentImpl) Logs(ctx context.Context, namespace, name, since string) (*deploy.MultiLogReader, error) { s, err := c.spaceStore.FindByPath(ctx, namespace, name) if err != nil { diff --git a/component/space_test.go b/component/space_test.go index 601c331ea..597549392 100644 --- a/component/space_test.go +++ b/component/space_test.go @@ -21,16 +21,15 @@ func TestSpaceComponent_Show(t *testing.T) { ctx := context.TODO() sc := initializeTestSpaceComponent(ctx, t) - sc.mocks.stores.SpaceMock().EXPECT().FindByPath(ctx, "ns", "n").Return(&database.Space{ + dbRepo := &database.Repository{ID: 123, Name: "n", Path: "foo/bar", Tags: []database.Tag{{Name: "t1"}}} + dbSpace := &database.Space{ ID: 1, - Repository: &database.Repository{ID: 123, Name: "n", Path: "foo/bar"}, + Repository: dbRepo, HasAppFile: true, - }, nil) - sc.mocks.components.repo.EXPECT().GetUserRepoPermission(ctx, "user", &database.Repository{ - ID: 123, - Name: "n", - Path: "foo/bar", - }).Return( + } + + sc.mocks.stores.SpaceMock().EXPECT().FindByPath(ctx, "ns", "n").Return(dbSpace, nil) + sc.mocks.components.repo.EXPECT().GetUserRepoPermission(ctx, "user", dbRepo).Return( &types.UserRepoPermission{CanRead: true, CanAdmin: true}, nil, ) sc.mocks.components.repo.EXPECT().GetNameSpaceInfo(ctx, "ns").Return(&types.Namespace{Path: "ns"}, nil) @@ -67,9 +66,11 @@ func TestSpaceComponent_Show(t *testing.T) { HTTPCloneURL: "/s/foo/bar.git", SSHCloneURL: ":s/foo/bar.git", }, + Tags: []types.RepoTag{{Name: "t1"}}, Endpoint: "endpoint/svc", SvcName: "svc", }, space) + require.Equal(t, []types.RepoTag{{Name: "t1"}}, space.Tags) } func TestSpaceComponent_Index(t *testing.T) { @@ -290,111 +291,6 @@ func TestSpaceComponent_AllowCallApi(t *testing.T) { } -func TestSpaceComponent_Delete(t *testing.T) { - ctx := context.TODO() - sc := initializeTestSpaceComponent(ctx, t) - - sc.mocks.stores.SpaceMock().EXPECT().FindByPath(mock.Anything, "ns", "n").Return(&database.Space{ID: 1}, nil) - sc.mocks.components.repo.EXPECT().DeleteRepo(ctx, types.DeleteRepoReq{ - Username: "user", - Namespace: "ns", - Name: "n", - RepoType: types.SpaceRepo, - }).Return(&database.Repository{ - User: database.User{ - UUID: "user-uuid", - }, - Path: "ns/n", - }, nil) - sc.mocks.stores.SpaceMock().EXPECT().Delete(mock.Anything, database.Space{ID: 1}).Return(nil) - - sc.mocks.stores.DeployTaskMock().EXPECT().GetLatestDeployBySpaceID(mock.Anything, int64(1)).Return( - &database.Deploy{ - RepoID: 2, - UserID: 3, - ID: 4, - }, nil, - ) - var wgstop sync.WaitGroup - wgstop.Add(1) - sc.mocks.deployer.EXPECT().Stop(mock.Anything, mock.MatchedBy(func(req types.DeployRepo) bool { - return req.SpaceID == 1 && - req.Namespace == "ns" && - req.Name == "n" - })). - RunAndReturn(func(ctx context.Context, req types.DeployRepo) error { - wgstop.Done() - return nil - }).Once() - sc.mocks.stores.DeployTaskMock().EXPECT().StopDeploy( - mock.Anything, types.SpaceRepo, int64(2), int64(3), int64(4), - ).Return(nil) - - var wg sync.WaitGroup - wg.Add(1) - sc.mocks.components.repo.EXPECT(). - SendAssetManagementMsg(mock.Anything, mock.MatchedBy(func(req types.RepoNotificationReq) bool { - return req.RepoType == types.SpaceRepo && - req.Operation == types.OperationDelete && - req.RepoPath == "ns/n" && - req.UserUUID == "user-uuid" - })). - RunAndReturn(func(ctx context.Context, req types.RepoNotificationReq) error { - wg.Done() - return nil - }).Once() - - err := sc.Delete(ctx, "ns", "n", "user") - require.Nil(t, err) - wg.Wait() - wgstop.Wait() -} - -func TestSpaceComponent_Deploy(t *testing.T) { - ctx := context.TODO() - sc := initializeTestSpaceComponent(ctx, t) - - sc.mocks.stores.UserMock().EXPECT().FindByUsername(ctx, "user").Return(database.User{ - Username: "user1", - }, nil) - t.Run("Deploy", func(t *testing.T) { - sc.mocks.stores.SpaceMock().EXPECT().FindByPath(ctx, "ns1", "n1").Return(&database.Space{ - ID: 1, - Repository: &database.Repository{Path: "foo1/bar1"}, - SKU: "1", - HasAppFile: true, - }, nil) - sc.mocks.stores.SpaceResourceMock().EXPECT().FindByID(ctx, int64(1)).Return(&database.SpaceResource{ - ID: 1, - }, nil) - sc.mocks.components.repo.EXPECT().CheckAccountAndResource(ctx, "user", "", int64(0), &database.SpaceResource{ - ID: 1, - }).Return(nil) - sc.mocks.deployer.EXPECT().Deploy(ctx, types.DeployRepo{ - SpaceID: 1, - Path: "foo1/bar1", - Annotation: "{\"hub-deploy-user\":\"user1\",\"hub-res-name\":\"ns1/n1\",\"hub-res-type\":\"space\"}", - ContainerPort: 8080, - SKU: "1", - }).Return(123, nil) - - id, err := sc.Deploy(ctx, "ns1", "n1", "user") - require.Nil(t, err) - require.Equal(t, int64(123), id) - }) - t.Run("DeployWithoutAppFile", func(t *testing.T) { - sc.mocks.stores.SpaceMock().EXPECT().FindByPath(ctx, "ns2", "n2").Return(&database.Space{ - ID: 1, - Repository: &database.Repository{Path: "foo2/bar2"}, - SKU: "1", - HasAppFile: false, - }, nil) - id, err := sc.Deploy(ctx, "ns2", "n2", "user") - require.Equal(t, true, errors.Is(err, errorx.ErrNoEntryFile)) - require.Equal(t, int64(-1), id) - }) -} - func TestSpaceComponent_Wakeup(t *testing.T) { ctx := context.TODO() sc := initializeTestSpaceComponent(ctx, t) @@ -642,3 +538,325 @@ func TestSpaceComponent_GetMCPServiceBySvcName(t *testing.T) { require.Contains(t, err.Error(), "failed to get space by id") }) } + +func TestSpaceComponent_StatusByPaths(t *testing.T) { + ctx := context.TODO() + sc := initializeTestSpaceComponent(ctx, t) + + t.Run("empty paths", func(t *testing.T) { + result, err := sc.StatusByPaths(ctx, []string{}) + require.Nil(t, err) + require.NotNil(t, result) + require.Equal(t, 0, len(result)) + }) + + t.Run("space not found", func(t *testing.T) { + sc.mocks.stores.SpaceMock().EXPECT().ListByPath(ctx, []string{"nonexistent/space"}). + Return([]database.Space{}, nil).Once() + + // When no spaces are found, spaceIDs will be empty, but GetLatestDeploysBySpaceIDs is still called + sc.mocks.stores.DeployTaskMock().EXPECT().GetLatestDeploysBySpaceIDs(ctx, []int64{}). + Return(map[int64]*database.Deploy{}, nil).Once() + + result, err := sc.StatusByPaths(ctx, []string{"nonexistent/space"}) + require.Nil(t, err) + require.NotNil(t, result) + require.Equal(t, 1, len(result)) + require.Equal(t, SpaceStatusEmpty, result["nonexistent/space"]) + }) + + t.Run("space without HasAppFile - NGINX", func(t *testing.T) { + dbRepo := &database.Repository{ID: 1, Path: "ns1/space1"} + dbSpace := &database.Space{ + ID: 1, + Repository: dbRepo, + HasAppFile: false, + Sdk: types.NGINX.Name, + } + + sc.mocks.stores.SpaceMock().EXPECT().ListByPath(ctx, []string{"ns1/space1"}). + Return([]database.Space{*dbSpace}, nil).Once() + + // GetLatestDeploysBySpaceIDs is called with all found space IDs, even if they don't have HasAppFile + sc.mocks.stores.DeployTaskMock().EXPECT().GetLatestDeploysBySpaceIDs(ctx, []int64{1}). + Return(map[int64]*database.Deploy{}, nil).Once() + + result, err := sc.StatusByPaths(ctx, []string{"ns1/space1"}) + require.Nil(t, err) + require.NotNil(t, result) + require.Equal(t, 1, len(result)) + require.Equal(t, SpaceStatusNoNGINXConf, result["ns1/space1"]) + }) + + t.Run("space without HasAppFile - non-NGINX", func(t *testing.T) { + dbRepo := &database.Repository{ID: 2, Path: "ns2/space2"} + dbSpace := &database.Space{ + ID: 2, + Repository: dbRepo, + HasAppFile: false, + Sdk: "streamlit", + } + + sc.mocks.stores.SpaceMock().EXPECT().ListByPath(ctx, []string{"ns2/space2"}). + Return([]database.Space{*dbSpace}, nil).Once() + + // GetLatestDeploysBySpaceIDs is called with all found space IDs, even if they don't have HasAppFile + sc.mocks.stores.DeployTaskMock().EXPECT().GetLatestDeploysBySpaceIDs(ctx, []int64{2}). + Return(map[int64]*database.Deploy{}, nil).Once() + + result, err := sc.StatusByPaths(ctx, []string{"ns2/space2"}) + require.Nil(t, err) + require.NotNil(t, result) + require.Equal(t, 1, len(result)) + require.Equal(t, SpaceStatusNoAppFile, result["ns2/space2"]) + }) + + t.Run("space with HasAppFile but no deploy", func(t *testing.T) { + dbRepo := &database.Repository{ID: 3, Path: "ns3/space3"} + dbSpace := &database.Space{ + ID: 3, + Repository: dbRepo, + HasAppFile: true, + } + + sc.mocks.stores.SpaceMock().EXPECT().ListByPath(ctx, []string{"ns3/space3"}). + Return([]database.Space{*dbSpace}, nil).Once() + + sc.mocks.stores.DeployTaskMock().EXPECT().GetLatestDeploysBySpaceIDs(ctx, []int64{3}). + Return(map[int64]*database.Deploy{}, nil).Once() + + result, err := sc.StatusByPaths(ctx, []string{"ns3/space3"}) + require.Nil(t, err) + require.NotNil(t, result) + require.Equal(t, 1, len(result)) + require.Equal(t, SpaceStatusStopped, result["ns3/space3"]) + }) + + t.Run("space with HasAppFile and deploy - Running", func(t *testing.T) { + dbRepo := &database.Repository{ID: 4, Path: "ns4/space4"} + dbSpace := &database.Space{ + ID: 4, + Repository: dbRepo, + HasAppFile: true, + } + + deploy := &database.Deploy{ + ID: 1, + Status: 23, // Running + } + + sc.mocks.stores.SpaceMock().EXPECT().ListByPath(ctx, []string{"ns4/space4"}). + Return([]database.Space{*dbSpace}, nil).Once() + + sc.mocks.stores.DeployTaskMock().EXPECT().GetLatestDeploysBySpaceIDs(ctx, []int64{4}). + Return(map[int64]*database.Deploy{4: deploy}, nil).Once() + + result, err := sc.StatusByPaths(ctx, []string{"ns4/space4"}) + require.Nil(t, err) + require.NotNil(t, result) + require.Equal(t, 1, len(result)) + require.Equal(t, SpaceStatusRunning, result["ns4/space4"]) + }) + + t.Run("space with HasAppFile and deploy - Stopped", func(t *testing.T) { + dbRepo := &database.Repository{ID: 5, Path: "ns5/space5"} + dbSpace := &database.Space{ + ID: 5, + Repository: dbRepo, + HasAppFile: true, + } + + deploy := &database.Deploy{ + ID: 2, + Status: 26, // Stopped + } + + sc.mocks.stores.SpaceMock().EXPECT().ListByPath(ctx, []string{"ns5/space5"}). + Return([]database.Space{*dbSpace}, nil).Once() + + sc.mocks.stores.DeployTaskMock().EXPECT().GetLatestDeploysBySpaceIDs(ctx, []int64{5}). + Return(map[int64]*database.Deploy{5: deploy}, nil).Once() + + result, err := sc.StatusByPaths(ctx, []string{"ns5/space5"}) + require.Nil(t, err) + require.NotNil(t, result) + require.Equal(t, 1, len(result)) + require.Equal(t, SpaceStatusStopped, result["ns5/space5"]) + }) + + t.Run("multiple spaces with different scenarios", func(t *testing.T) { + paths := []string{"ns6/space6", "ns7/space7", "ns8/space8", "ns9/space9"} + + dbRepo6 := &database.Repository{ID: 6, Path: "ns6/space6"} + dbSpace6 := &database.Space{ + ID: 6, + Repository: dbRepo6, + HasAppFile: false, + Sdk: types.NGINX.Name, + } + + dbRepo7 := &database.Repository{ID: 7, Path: "ns7/space7"} + dbSpace7 := &database.Space{ + ID: 7, + Repository: dbRepo7, + HasAppFile: true, + } + + dbRepo8 := &database.Repository{ID: 8, Path: "ns8/space8"} + dbSpace8 := &database.Space{ + ID: 8, + Repository: dbRepo8, + HasAppFile: true, + } + + deploy8 := &database.Deploy{ + ID: 3, + Status: 23, // Running + } + + sc.mocks.stores.SpaceMock().EXPECT().ListByPath(ctx, paths). + Return([]database.Space{*dbSpace6, *dbSpace7, *dbSpace8}, nil).Once() + + sc.mocks.stores.DeployTaskMock().EXPECT().GetLatestDeploysBySpaceIDs(ctx, []int64{6, 7, 8}). + Return(map[int64]*database.Deploy{8: deploy8}, nil).Once() + + result, err := sc.StatusByPaths(ctx, paths) + require.Nil(t, err) + require.NotNil(t, result) + require.Equal(t, 4, len(result)) + + // Space 6: no app file, NGINX + require.Equal(t, SpaceStatusNoNGINXConf, result["ns6/space6"]) + + // Space 7: has app file but no deploy + require.Equal(t, SpaceStatusStopped, result["ns7/space7"]) + + // Space 8: has app file and running deploy + require.Equal(t, SpaceStatusRunning, result["ns8/space8"]) + + // Space 9: not found + require.Equal(t, SpaceStatusEmpty, result["ns9/space9"]) + }) + + t.Run("ListByPath error", func(t *testing.T) { + sc.mocks.stores.SpaceMock().EXPECT().ListByPath(ctx, []string{"error/space"}). + Return(nil, errors.New("database error")).Once() + + result, err := sc.StatusByPaths(ctx, []string{"error/space"}) + require.NotNil(t, err) + require.Nil(t, result) + require.Contains(t, err.Error(), "failed to find spaces by paths") + }) + + t.Run("GetLatestDeploysBySpaceIDs error", func(t *testing.T) { + dbRepo := &database.Repository{ID: 10, Path: "ns10/space10"} + dbSpace := &database.Space{ + ID: 10, + Repository: dbRepo, + HasAppFile: true, + } + + sc.mocks.stores.SpaceMock().EXPECT().ListByPath(ctx, []string{"ns10/space10"}). + Return([]database.Space{*dbSpace}, nil).Once() + + sc.mocks.stores.DeployTaskMock().EXPECT().GetLatestDeploysBySpaceIDs(ctx, []int64{10}). + Return(nil, errors.New("deploy query error")).Once() + + result, err := sc.StatusByPaths(ctx, []string{"ns10/space10"}) + require.NotNil(t, err) + require.Nil(t, result) + }) +} + +func TestSpaceComponent_CreateGradio(t *testing.T) { + ctx := context.TODO() + sc := initializeTestSpaceComponent(ctx, t) + + sc.mocks.stores.SpaceResourceMock().EXPECT().FindByID(ctx, int64(1)).Return(&database.SpaceResource{ + ID: 1, + Name: "sp", + Resources: `{"memory": "foo"}`, + }, nil) + + sc.mocks.components.repo.EXPECT().CheckAccountAndResource(ctx, "user", "cluster", int64(0), mock.Anything).Return(nil) + + sc.mocks.components.repo.EXPECT().CreateRepo(ctx, types.CreateRepoReq{ + DefaultBranch: "main", + Readme: generateReadmeData("MIT"), + License: "MIT", + Namespace: "ns", + Name: "n", + Nickname: "n", + RepoType: types.SpaceRepo, + Username: "user", + }).Return(nil, &database.Repository{ + ID: 321, + User: database.User{ + Username: "user", + Email: "foo@bar.com", + UUID: "user-uuid", + }, + Path: "ns/n", + }, &gitserver.CommitFilesReq{}, nil) + + sc.mocks.stores.SpaceMock().EXPECT().CreateAndUpdateRepoPath(ctx, database.Space{ + RepositoryID: 321, + Sdk: types.GRADIO.Name, + SdkVersion: "6.2.0", + Env: "env", + Hardware: `{"memory": "foo"}`, + Secrets: "sss", + SKU: "1", + ClusterID: "cluster", + }, "ns/n").Return(&database.Space{ + RepositoryID: 321, + Repository: &database.Repository{ID: 321, Path: "ns/n"}, + }, nil) + + sc.mocks.gitServer.EXPECT().CommitFiles(ctx, gitserver.CommitFilesReq{}).Return(nil).Once() + + sc.mocks.gitServer.EXPECT().CommitFiles(mock.Anything, mock.Anything).Return(nil) + + var wg sync.WaitGroup + wg.Add(1) + sc.mocks.components.repo.EXPECT().SendAssetManagementMsg(mock.Anything, mock.MatchedBy(func(req types.RepoNotificationReq) bool { + return req.RepoType == types.SpaceRepo && + req.Operation == types.OperationCreate && + req.RepoPath == "ns/n" && + req.UserUUID == "user-uuid" + })).RunAndReturn(func(ctx context.Context, req types.RepoNotificationReq) error { + wg.Done() + return nil + }).Once() + + space, err := sc.Create(ctx, types.CreateSpaceReq{ + Sdk: types.GRADIO.Name, + SdkVersion: "", + Env: "env", + Secrets: "sss", + ResourceID: 1, + ClusterID: "cluster", + CreateRepoReq: types.CreateRepoReq{ + DefaultBranch: "main", + Readme: "readme", + Namespace: "ns", + Name: "n", + License: "MIT", + Username: "user", + }, + }) + require.Nil(t, err) + + require.Equal(t, &types.Space{ + License: "MIT", + Name: "n", + Sdk: "gradio", + SdkVersion: "6.2.0", + Env: "env", + Secrets: "sss", + Hardware: `{"memory": "foo"}`, + Creator: "user", + Path: "ns/n", + }, space) + wg.Wait() +} diff --git a/docker/spaces/builder/Dockerfile-python3.10 b/docker/spaces/builder/Dockerfile-python3.10 index 450c8b35d..d84d0d6ca 100644 --- a/docker/spaces/builder/Dockerfile-python3.10 +++ b/docker/spaces/builder/Dockerfile-python3.10 @@ -1,7 +1,9 @@ ARG SPACE_BASE_IMAGE=opencsg-registry.cn-beijing.cr.aliyuncs.com ARG NAMESPACE=opencsghq +ARG SPACE_BASE_TAG=python3.10 +ARG SPACE_BASE_VERSION=1.0.4 -FROM ${SPACE_BASE_IMAGE}/${NAMESPACE}/space-base:python3.10-1.0.3 AS base +FROM ${SPACE_BASE_IMAGE}/${NAMESPACE}/space-base:${SPACE_BASE_TAG}-${SPACE_BASE_VERSION} AS base # First handle dependencies to leverage caching USER user diff --git a/docker/spaces/builder/Dockerfile-python3.10-cuda11.8.0 b/docker/spaces/builder/Dockerfile-python3.10-cuda11.8.0 index d706c3b26..78698c024 100644 --- a/docker/spaces/builder/Dockerfile-python3.10-cuda11.8.0 +++ b/docker/spaces/builder/Dockerfile-python3.10-cuda11.8.0 @@ -1,7 +1,9 @@ ARG SPACE_BASE_IMAGE=opencsg-registry.cn-beijing.cr.aliyuncs.com ARG NAMESPACE=opencsghq +ARG SPACE_BASE_TAG=python3.10-cuda11.8.0 +ARG SPACE_BASE_VERSION=1.0.4 -FROM ${SPACE_BASE_IMAGE}/${NAMESPACE}/space-base:python3.10-cuda11.8.0-1.0.3 AS base +FROM ${SPACE_BASE_IMAGE}/${NAMESPACE}/space-base:${SPACE_BASE_TAG}-${SPACE_BASE_VERSION} AS base USER user WORKDIR /home/user/app diff --git a/docker/spaces/builder/Dockerfile-python3.10-cuda12.1.0 b/docker/spaces/builder/Dockerfile-python3.10-cuda12.1.0 index 9b10d4673..03bfbd238 100644 --- a/docker/spaces/builder/Dockerfile-python3.10-cuda12.1.0 +++ b/docker/spaces/builder/Dockerfile-python3.10-cuda12.1.0 @@ -1,6 +1,9 @@ ARG SPACE_BASE_IMAGE=opencsg-registry.cn-beijing.cr.aliyuncs.com ARG NAMESPACE=opencsghq -FROM ${SPACE_BASE_IMAGE}/${NAMESPACE}/space-base:python3.10-cuda12.1.0-1.0.3 AS base +ARG SPACE_BASE_TAG=python3.10-cuda12.1.0 +ARG SPACE_BASE_VERSION=1.0.4 + +FROM ${SPACE_BASE_IMAGE}/${NAMESPACE}/space-base:${SPACE_BASE_TAG}-${SPACE_BASE_VERSION} AS base USER user WORKDIR /home/user/app diff --git a/runner/component/imagebuilder.go b/runner/component/imagebuilder.go index bfc59888d..9b2808ed6 100644 --- a/runner/component/imagebuilder.go +++ b/runner/component/imagebuilder.go @@ -252,6 +252,12 @@ func wfTemplateForImageBuilder(cfg *config.Config, params ctypes.ImageBuilderReq "--build-arg=GIT_IMAGE=" + cfg.Runner.ImageBuilderGitImage, } + if (params.Sdk == ctypes.GRADIO.Name && params.Sdk_version != ctypes.GRADIO.Version) || + (params.Sdk == ctypes.STREAMLIT.Name && params.Sdk_version != ctypes.STREAMLIT.Version) { + // Using old base image 1.0.3 for old spaces, will be removed in the future + builderArgs = append(builderArgs, "--build-arg=SPACE_BASE_VERSION=1.0.3") + } + for _, arg := range cfg.Runner.ImageBuilderKanikoArgs { if arg == "" || strings.HasPrefix(arg, "--context") || strings.HasPrefix(arg, "--destination") { continue diff --git a/runner/component/imagebuilder_test.go b/runner/component/imagebuilder_test.go index 1938a1a75..419648541 100644 --- a/runner/component/imagebuilder_test.go +++ b/runner/component/imagebuilder_test.go @@ -40,12 +40,47 @@ func TestImagebuilderComponent_Build(t *testing.T) { pool.EXPECT().GetClusterByID(context.Background(), testCluster.CID).Return(testCluster, nil).Once() // imageStore.EXPECT().CreateOrUpdateByBuildID(context.Background(), mock.Anything).Return(&database.ImageBuilderWork{}, nil).Once() // imageStore.EXPECT().FindByImagePath(context.Background(), mock.Anything).Return(nil, nil).Once() + err := ibc.Build(context.Background(), types.ImageBuilderRequest{ + ClusterID: "config", + OrgName: "test-org", + SpaceName: "test-space", + DeployId: "test-build-id", + SpaceURL: "https://github.com/test-org/test-space", + Sdk: "gradio", + Sdk_version: "6.2.0", + }) + + require.Nil(t, err) +} + +func TestImagebuilderComponent_Build_OldSpace(t *testing.T) { + conf := &config.Config{} + testCluster := &cluster.Cluster{ + CID: "config", + ID: "config", + Client: fake.NewSimpleClientset(), + ArgoClient: argofake.NewSimpleClientset(), + } + + logReporter := mockReporter.NewMockLogCollector(t) + pool := mockCluster.NewMockPool(t) + ibc := &imagebuilderComponentImpl{ + clusterPool: pool, + config: conf, + logReporter: logReporter, + } + + logReporter.EXPECT().Report(mock.Anything).Return().Once() + + pool.EXPECT().GetClusterByID(context.Background(), testCluster.CID).Return(testCluster, nil).Once() + err := ibc.Build(context.Background(), types.ImageBuilderRequest{ ClusterID: "config", OrgName: "test-org", SpaceName: "test-space", DeployId: "test-build-id", SpaceURL: "https://github.com/test-org/test-space", + Sdk: "gradio", }) require.Nil(t, err)