From 4cabbbe5e15ee42f4a1ff3de129de613239ab885 Mon Sep 17 00:00:00 2001 From: yiling Date: Wed, 27 Nov 2024 14:56:16 +0800 Subject: [PATCH 01/15] Add missing database tests Support argo workflow system Support evaluation datasets support opencompass --- .gitignore | 2 + api/handler/evaluation.go | 111 +++++ api/handler/repo.go | 2 +- api/handler/tag.go | 7 +- api/handler/user.go | 46 +++ api/router/api.go | 27 +- builder/deploy/cluster/cluster_manager.go | 7 + builder/deploy/deployer.go | 61 ++- builder/deploy/imagerunner/local_runner.go | 13 + builder/deploy/imagerunner/remote_runner.go | 50 +++ builder/deploy/imagerunner/runner.go | 4 + builder/store/database/argo_workflow.go | 104 +++++ .../20241105124716_create_table_workflows.go | 51 +++ .../20241111095821_create_tag_rules.go | 35 ++ .../20241111095847_init_tag_rule.down.sql | 9 + .../20241111095847_init_tag_rule.up.sql | 112 +++++ ...095440_update_argo_workflow_table.down.sql | 10 + ...17095440_update_argo_workflow_table.up.sql | 18 + .../store/database/seeds/tag_categories.yml | 26 +- builder/store/database/tag.go | 26 +- builder/store/database/tag_rule.go | 41 ++ cmd/csghub-server/cmd/deploy/runner.go | 12 +- common/config/config.go | 12 + common/types/accounting.go | 1 + common/types/argo_workflow.go | 128 ++++++ common/types/model.go | 29 +- common/types/tag.go | 11 + common/types/user.go | 1 + component/callback/git_callback.go | 58 ++- component/evaluation.go | 105 +++++ component/model.go | 22 +- component/runtime_architecture.go | 1 + component/tag.go | 7 +- component/user.go | 20 + docker/evaluation/Dockerfile.opencompass | 16 + .../evaluation/opencompass/get_answer_mode.py | 30 ++ docker/evaluation/opencompass/start.sh | 75 ++++ docker/evaluation/opencompass/upload_files.py | 201 +++++++++ docker/inference/supervisord.conf | 1 + docs/docs.go | 235 ++++++++++- docs/swagger.json | 235 ++++++++++- docs/swagger.yaml | 151 ++++++- go.mod | 31 +- go.sum | 68 ++-- .../component/service.go | 4 +- runner/component/workflow.go | 383 ++++++++++++++++++ .../k8s.go => runner/handler/service.go | 53 ++- runner/handler/workflow.go | 110 +++++ runner/router/api.go | 64 +++ servicerunner/router/api.go | 45 -- 50 files changed, 2674 insertions(+), 197 deletions(-) create mode 100644 api/handler/evaluation.go create mode 100644 builder/store/database/argo_workflow.go create mode 100644 builder/store/database/migrations/20241105124716_create_table_workflows.go create mode 100644 builder/store/database/migrations/20241111095821_create_tag_rules.go create mode 100644 builder/store/database/migrations/20241111095847_init_tag_rule.down.sql create mode 100644 builder/store/database/migrations/20241111095847_init_tag_rule.up.sql create mode 100644 builder/store/database/migrations/20241117095440_update_argo_workflow_table.down.sql create mode 100644 builder/store/database/migrations/20241117095440_update_argo_workflow_table.up.sql create mode 100644 builder/store/database/tag_rule.go create mode 100644 common/types/argo_workflow.go create mode 100644 component/evaluation.go create mode 100644 docker/evaluation/Dockerfile.opencompass create mode 100644 docker/evaluation/opencompass/get_answer_mode.py create mode 100644 docker/evaluation/opencompass/start.sh create mode 100644 docker/evaluation/opencompass/upload_files.py rename {servicerunner => runner}/component/service.go (97%) create mode 100644 runner/component/workflow.go rename servicerunner/handler/k8s.go => runner/handler/service.go (94%) create mode 100644 runner/handler/workflow.go create mode 100644 runner/router/api.go delete mode 100644 servicerunner/router/api.go diff --git a/.gitignore b/.gitignore index 45bfe3ab1..47bd72795 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,5 @@ local.toml .data /data */*/_api_test/ +redis/dump.rdb +run_server_local.sh diff --git a/api/handler/evaluation.go b/api/handler/evaluation.go new file mode 100644 index 000000000..ee339dc43 --- /dev/null +++ b/api/handler/evaluation.go @@ -0,0 +1,111 @@ +package handler + +import ( + "errors" + "fmt" + "log/slog" + "strconv" + + "github.com/gin-gonic/gin" + "opencsg.com/csghub-server/api/httpbase" + "opencsg.com/csghub-server/common/config" + "opencsg.com/csghub-server/common/types" + "opencsg.com/csghub-server/component" +) + +func NewEvaluationHandler(config *config.Config) (*EvaluationHandler, error) { + wkf, err := component.NewArgoWFSComponent(config) + if err != nil { + return nil, err + } + sc, err := component.NewSensitiveComponent(config) + if err != nil { + return nil, fmt.Errorf("error creating sensitive component:%w", err) + } + return &EvaluationHandler{ + c: wkf, + sc: sc, + }, nil +} + +type EvaluationHandler struct { + c *component.ArgoWFSComponent + sc *component.SensitiveComponent +} + +// create evaluation godoc +// @Security ApiKey +// @Summary run model evaluation +// @Tags Evaluation +// @Accept json +// @Produce json +// @Param namespace path string true "namespace" +// @Param name path string true "name" +// @Param body body types.EvaluationReq true "body setting of evaluation" +// @Success 200 {object} string "OK" +// @Failure 400 {object} types.APIBadRequest "Bad request" +// @Failure 500 {object} types.APIInternalServerError "Internal server error" +// @Router /evaluations [post] +func (h *EvaluationHandler) RunEvaluation(ctx *gin.Context) { + currentUser := httpbase.GetCurrentUser(ctx) + if currentUser == "" { + httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first")) + return + } + + var req types.EvaluationReq + if err := ctx.ShouldBindJSON(&req); err != nil { + slog.Error("Bad request format", "error", err) + httpbase.BadRequest(ctx, err.Error()) + return + } + _, err := h.sc.CheckRequestV2(ctx, &req) + if err != nil { + slog.Error("failed to check sensitive request", slog.Any("error", err)) + httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error()) + return + } + req.Username = currentUser + evaluation, err := h.c.CreateEvaluation(ctx, &req) + if err != nil { + slog.Error("Failed to create evaluation job", slog.Any("error", err)) + httpbase.ServerError(ctx, err) + return + } + httpbase.OK(ctx, evaluation) +} + +// deleteEvaluation godoc +// @Security ApiKey +// @Summary delete model evaluation +// @Tags Evaluation +// @Accept json +// @Produce json +// @Param id path string true "id" +// @Success 200 {object} string "OK" +// @Failure 400 {object} types.APIBadRequest "Bad request" +// @Failure 500 {object} types.APIInternalServerError "Internal server error" +// @Router /evaluations/{id} [delete] +func (h *EvaluationHandler) DeleteEvaluation(ctx *gin.Context) { + currentUser := httpbase.GetCurrentUser(ctx) + if currentUser == "" { + httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first")) + return + } + id, err := strconv.ParseInt(ctx.Param("id"), 10, 64) + if err != nil { + slog.Error("Bad request format", "error", err) + httpbase.BadRequest(ctx, err.Error()) + return + } + var req = &types.EvaluationDelReq{} + req.ID = id + req.Username = currentUser + err = h.c.DeleteEvaluation(ctx, *req) + if err != nil { + slog.Error("Failed to delete evaluation job", slog.Any("error", err)) + httpbase.ServerError(ctx, err) + return + } + httpbase.OK(ctx, nil) +} diff --git a/api/handler/repo.go b/api/handler/repo.go index 04b77443e..07ff9bfbf 100644 --- a/api/handler/repo.go +++ b/api/handler/repo.go @@ -1209,7 +1209,7 @@ func (h *RepoHandler) DeleteMirror(ctx *gin.Context) { // @Param namespace path string true "namespace" // @Param name path string true "name" // @Param current_user query string false "current user" -// @Param deploy_type query int false "deploy_type" Enums(0, 1, 2) default(1) +// @Param deploy_type query int false "deploy_type" Enums(0, 1, 2, 4) default(1) // @Success 200 {object} string "OK" // @Failure 400 {object} types.APIBadRequest "Bad request" // @Failure 500 {object} types.APIInternalServerError "Internal server error" diff --git a/api/handler/tag.go b/api/handler/tag.go index a707bd066..e4e62ca22 100644 --- a/api/handler/tag.go +++ b/api/handler/tag.go @@ -31,13 +31,16 @@ type TagsHandler struct { // @Tags Tag // @Accept json // @Produce json +// @Param category query string false "category name" +// @Param scope query string false "scope name" Enums(model, dataset) // @Success 200 {object} types.ResponseWithTotal{data=[]database.Tag,total=int} "tags" // @Failure 500 {object} types.APIInternalServerError "Internal server error" // @Router /tags [get] func (t *TagsHandler) AllTags(ctx *gin.Context) { //TODO:validate inputs - - tags, err := t.tc.AllTags(ctx) + category := ctx.Query("category") + scope := ctx.Query("scope") + tags, err := t.tc.AllTagsByScopeAndCategory(ctx, scope, category) if err != nil { slog.Error("Failed to load tags", "error", err) httpbase.ServerError(ctx, err) diff --git a/api/handler/user.go b/api/handler/user.go index b04541468..c062710f4 100644 --- a/api/handler/user.go +++ b/api/handler/user.go @@ -854,3 +854,49 @@ func (h *UserHandler) Prompts(ctx *gin.Context) { } ctx.JSON(http.StatusOK, respData) } + +// GetUserEvaluations godoc +// @Security ApiKey +// @Summary Get user evaluations +// @Description get user evaluations +// @Tags User +// @Accept json +// @Produce json +// @Param username path string true "username" +// @Param per query int false "per" default(20) +// @Param page query int false "per page" default(1) +// @Success 200 {object} types.Response{} "OK" +// @Failure 400 {object} types.APIBadRequest "Bad request" +// @Failure 500 {object} types.APIInternalServerError "Internal server error" +// @Router /user/{username}/evaluations [get] +func (h *UserHandler) GetEvaluations(ctx *gin.Context) { + currentUser := httpbase.GetCurrentUser(ctx) + if currentUser == "" { + httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first")) + return + } + var req types.UserEvaluationReq + per, page, err := common.GetPerAndPageFromContext(ctx) + if err != nil { + slog.Error("Bad request format of page and per", "error", err) + httpbase.BadRequest(ctx, err.Error()) + return + } + + req.Owner = ctx.Param("username") + req.CurrentUser = httpbase.GetCurrentUser(ctx) + req.Page = page + req.PageSize = per + ds, total, err := h.c.Evaluations(ctx, &req) + if err != nil { + slog.Error("Failed to get user evaluations", slog.Any("error", err)) + httpbase.ServerError(ctx, err) + return + } + + respData := gin.H{ + "data": ds, + "total": total, + } + ctx.JSON(http.StatusOK, respData) +} diff --git a/api/router/api.go b/api/router/api.go index 670b6f19c..9adeb6d78 100644 --- a/api/router/api.go +++ b/api/router/api.go @@ -120,8 +120,17 @@ func NewRouter(config *config.Config, enableSwagger bool) (*gin.Engine, error) { apiGroup.POST("/list/spaces_by_path", cache.CacheByRequestURI(memoryStore, 1*time.Minute), listHandler.ListSpacesByPath) } + //evaluation handler + evaluationHandler, err := handler.NewEvaluationHandler(config) + if err != nil { + return nil, fmt.Errorf("error creatring evaluation handler: %v", err) + } + + createEvaluationRoutes(config, apiGroup, evaluationHandler) + // Model routes createModelRoutes(config, apiGroup, needAPIKey, modelHandler, repoCommonHandler) + // Dataset routes createDatasetRoutes(config, apiGroup, dsHandler, repoCommonHandler) @@ -132,13 +141,6 @@ func NewRouter(config *config.Config, enableSwagger bool) (*gin.Engine, error) { // Code routes createCodeRoutes(config, apiGroup, codeHandler, repoCommonHandler) - // Dataset viewer - dsViewerHandler, err := handler.NewDatasetViewerHandler(config) - if err != nil { - return nil, fmt.Errorf("error creating dataset viewer handler:%w", err) - } - apiGroup.GET("/datasets/:namespace/:name/viewer/*file_path", dsViewerHandler.View) - spaceHandler, err := handler.NewSpaceHandler(config) if err != nil { return nil, fmt.Errorf("error creating space handler:%w", err) @@ -386,6 +388,15 @@ func NewRouter(config *config.Config, enableSwagger bool) (*gin.Engine, error) { return r, nil } +func createEvaluationRoutes(config *config.Config, apiGroup *gin.RouterGroup, evaluationHandler *handler.EvaluationHandler) { + // Models routes + evaluationsGroup := apiGroup.Group("/evaluations") + { + evaluationsGroup.POST("", evaluationHandler.RunEvaluation) + evaluationsGroup.DELETE("/:id", evaluationHandler.DeleteEvaluation) + } +} + func createModelRoutes(config *config.Config, apiGroup *gin.RouterGroup, needAPIKey gin.HandlerFunc, modelHandler *handler.ModelHandler, repoCommonHandler *handler.RepoHandler) { // Models routes modelsGroup := apiGroup.Group("/models") @@ -648,6 +659,8 @@ func createUserRoutes(apiGroup *gin.RouterGroup, needAPIKey gin.HandlerFunc, use apiGroup.GET("/user/:username/likes/datasets", userHandler.LikesDatasets) apiGroup.GET("/user/:username/run/:repo_type", userHandler.GetRunDeploys) apiGroup.GET("/user/:username/finetune/instances", userHandler.GetFinetuneInstances) + // User evaluations + apiGroup.GET("/user/:username/evaluations", userHandler.GetEvaluations) } // User collection diff --git a/builder/deploy/cluster/cluster_manager.go b/builder/deploy/cluster/cluster_manager.go index a5692c5b9..b87e25be5 100644 --- a/builder/deploy/cluster/cluster_manager.go +++ b/builder/deploy/cluster/cluster_manager.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" + "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -27,6 +28,7 @@ type Cluster struct { ConfigPath string // Path to the kubeconfig file Client *kubernetes.Clientset // Kubernetes client KnativeClient *knative.Clientset // Knative client + ArgoClient *versioned.Clientset // Argo client StorageClass string } @@ -61,6 +63,10 @@ func NewClusterPool() (*ClusterPool, error) { if err != nil { return nil, err } + argoClient, err := versioned.NewForConfig(config) + if err != nil { + return nil, err + } knativeClient, err := knative.NewForConfig(config) if err != nil { slog.Error("falied to create knative client", "error", err) @@ -72,6 +78,7 @@ func NewClusterPool() (*ClusterPool, error) { ConfigPath: kubeconfig, Client: client, KnativeClient: knativeClient, + ArgoClient: argoClient, }) err = pool.ClusterStore.Add(context.TODO(), id, fmt.Sprintf("region-%d", i)) if err != nil { diff --git a/builder/deploy/deployer.go b/builder/deploy/deployer.go index d20c17d51..518ac09b6 100644 --- a/builder/deploy/deployer.go +++ b/builder/deploy/deployer.go @@ -38,7 +38,10 @@ type Deployer interface { UpdateCluster(ctx context.Context, data types.ClusterRequest) (*types.UpdateClusterResponse, error) UpdateDeploy(ctx context.Context, dur *types.DeployUpdateReq, deploy *database.Deploy) error StartDeploy(ctx context.Context, deploy *database.Deploy) error - CheckResourceAvailable(ctx context.Context, clusterId string, hardWare *types.HardWare) (bool, error) + CheckResourceAvailable(ctx context.Context, clusterId string, orderDetailID int64, hardWare *types.HardWare) (bool, error) + SubmitEvaluation(ctx context.Context, req types.EvaluationReq) (*types.ArgoWorkFlowRes, error) + ListEvaluations(context.Context, string, int, int) (*types.ArgoWorkFlowListRes, error) + DeleteEvaluation(ctx context.Context, req types.ArgoWorkFlowDeleteReq) error } var _ Deployer = (*deployer)(nil) @@ -775,3 +778,59 @@ func CheckResource(clusterResources *types.ClusterRes, hardware *types.HardWare) } return false } + +// SubmitEvaluation +func (d *deployer) SubmitEvaluation(ctx context.Context, req types.EvaluationReq) (*types.ArgoWorkFlowRes, error) { + env := make(map[string]string) + env["REVISION"] = "main" + env["MODEL_ID"] = req.ModelId + env["DATASET_IDS"] = strings.Join(req.Datasets, ",") + env["REVISION"] = "main" + env["ACCESS_TOKEN"] = req.Token + env["HF_ENDPOINT"] = req.DownloadEndpoint + + if req.Hardware.Gpu.Num != "" { + env["GPU_NUM"] = req.Hardware.Gpu.Num + } + templates := []types.ArgoFlowTemplate{} + templates = append(templates, types.ArgoFlowTemplate{ + Name: "evaluation", + Env: env, + HardWare: req.Hardware, + Image: req.Image, + }, + ) + uniqueFlowName := d.sfNode.Generate().Base36() + flowReq := &types.ArgoWorkFlowReq{ + TaskName: req.TaskName, + TaskId: uniqueFlowName, + TaskType: req.TaskType, + TaskDesc: req.TaskDesc, + Image: req.Image, + Datasets: req.Datasets, + Username: req.Username, + UserUUID: req.UserUUID, + RepoIds: []string{req.ModelId}, + Entrypoint: "evaluation", + ClusterID: req.ClusterID, + Templates: templates, + RepoType: req.RepoType, + ResourceId: req.ResourceId, + ResourceName: req.ResourceName, + } + if req.ResourceId == 0 { + flowReq.ShareMode = true + } + return d.ir.SubmitWorkFlow(ctx, flowReq) +} +func (d *deployer) ListEvaluations(ctx context.Context, username string, per int, page int) (*types.ArgoWorkFlowListRes, error) { + return d.ir.ListWorkFlows(ctx, username, per, page) +} + +func (d *deployer) DeleteEvaluation(ctx context.Context, req types.ArgoWorkFlowDeleteReq) error { + _, err := d.ir.DeleteWorkFlow(ctx, req) + if err != nil { + return err + } + return nil +} diff --git a/builder/deploy/imagerunner/local_runner.go b/builder/deploy/imagerunner/local_runner.go index 4afa391ce..6394c9b75 100644 --- a/builder/deploy/imagerunner/local_runner.go +++ b/builder/deploy/imagerunner/local_runner.go @@ -3,6 +3,7 @@ package imagerunner import ( "context" + "opencsg.com/csghub-server/api/httpbase" "opencsg.com/csghub-server/builder/deploy/common" "opencsg.com/csghub-server/common/types" ) @@ -89,3 +90,15 @@ func (h *LocalRunner) GetClusterById(ctx context.Context, clusterId string) (*ty func (h *LocalRunner) UpdateCluster(ctx context.Context, data *types.ClusterRequest) (*types.UpdateClusterResponse, error) { return nil, nil } + +func (h *LocalRunner) SubmitWorkFlow(ctx context.Context, req *types.ArgoWorkFlowReq) (*types.ArgoWorkFlowRes, error) { + return nil, nil +} + +func (h *LocalRunner) ListWorkFlows(ctx context.Context, username string, per, page int) (*types.ArgoWorkFlowListRes, error) { + return nil, nil +} + +func (h *LocalRunner) DeleteWorkFlow(ctx context.Context, req types.ArgoWorkFlowDeleteReq) (*httpbase.R, error) { + return nil, nil +} diff --git a/builder/deploy/imagerunner/remote_runner.go b/builder/deploy/imagerunner/remote_runner.go index 7b02a4e6e..ed00e1a8d 100644 --- a/builder/deploy/imagerunner/remote_runner.go +++ b/builder/deploy/imagerunner/remote_runner.go @@ -12,6 +12,7 @@ import ( "net/url" "time" + "opencsg.com/csghub-server/api/httpbase" "opencsg.com/csghub-server/common/types" ) @@ -315,3 +316,52 @@ func (h *RemoteRunner) UpdateCluster(ctx context.Context, data *types.ClusterReq return &resp, nil } + +// submit argo workflow +func (h *RemoteRunner) SubmitWorkFlow(ctx context.Context, req *types.ArgoWorkFlowReq) (*types.ArgoWorkFlowRes, error) { + url := fmt.Sprintf("%s/api/v1/workflows", h.remote) + // Create a new HTTP client with a timeout + response, err := h.doRequest(http.MethodPost, url, req) + if err != nil { + return nil, fmt.Errorf("failed to submit evaluation job, %w", err) + } + defer response.Body.Close() + + var res types.ArgoWorkFlowRes + if err := json.NewDecoder(response.Body).Decode(&res); err != nil { + return nil, err + } + return &res, nil +} + +// list workflows +func (h *RemoteRunner) ListWorkFlows(ctx context.Context, username string, per, page int) (*types.ArgoWorkFlowListRes, error) { + url := fmt.Sprintf("%s/api/v1/workflows?username=%s&per=%d&page=%d", h.remote, username, per, page) + // Create a new HTTP client with a timeout + response, err := h.doRequest(http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("failed to list evaluation jobs, %w", err) + } + var res types.ArgoWorkFlowListRes + if err := json.NewDecoder(response.Body).Decode(&res); err != nil { + return nil, err + } + return &res, nil +} + +// delete workflow +func (h *RemoteRunner) DeleteWorkFlow(ctx context.Context, req types.ArgoWorkFlowDeleteReq) (*httpbase.R, error) { + url := fmt.Sprintf("%s/api/v1/workflows/%d", h.remote, req.ID) + // Create a new HTTP client with a timeout + response, err := h.doRequest(http.MethodDelete, url, req) + if err != nil { + return nil, fmt.Errorf("failed to delete evaluation job, %w", err) + } + defer response.Body.Close() + var res httpbase.R + err = json.NewDecoder(response.Body).Decode(&res) + if err != nil { + return nil, err + } + return &res, nil +} diff --git a/builder/deploy/imagerunner/runner.go b/builder/deploy/imagerunner/runner.go index a21a41fa3..b3e702e1e 100644 --- a/builder/deploy/imagerunner/runner.go +++ b/builder/deploy/imagerunner/runner.go @@ -3,6 +3,7 @@ package imagerunner import ( "context" + "opencsg.com/csghub-server/api/httpbase" "opencsg.com/csghub-server/common/types" ) @@ -19,4 +20,7 @@ type Runner interface { ListCluster(ctx context.Context) ([]types.ClusterResponse, error) GetClusterById(ctx context.Context, clusterId string) (*types.ClusterResponse, error) UpdateCluster(ctx context.Context, data *types.ClusterRequest) (*types.UpdateClusterResponse, error) + SubmitWorkFlow(context.Context, *types.ArgoWorkFlowReq) (*types.ArgoWorkFlowRes, error) + ListWorkFlows(context.Context, string, int, int) (*types.ArgoWorkFlowListRes, error) + DeleteWorkFlow(context.Context, types.ArgoWorkFlowDeleteReq) (*httpbase.R, error) } diff --git a/builder/store/database/argo_workflow.go b/builder/store/database/argo_workflow.go new file mode 100644 index 000000000..2ddb1b0da --- /dev/null +++ b/builder/store/database/argo_workflow.go @@ -0,0 +1,104 @@ +package database + +import ( + "context" + "fmt" + "time" + + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + "opencsg.com/csghub-server/common/types" +) + +type ArgoWorkFlowStore struct { + db *DB +} + +func NewArgoWorkFlowStore() *ArgoWorkFlowStore { + return &ArgoWorkFlowStore{ + db: defaultDB, + } +} + +type ArgoWorkflow struct { + ID int64 `bun:",pk,autoincrement" json:"id"` + Username string `bun:",notnull" json:"username"` + UserUUID string `bun:",notnull" json:"user_uuid"` + TaskName string `bun:",notnull" json:"task_name"` // user input name + TaskId string `bun:",notnull" json:"task_id"` // generated task id + TaskType types.TaskType `bun:",notnull" json:"task_type"` + ClusterID string `bun:",notnull" json:"cluster_id"` + Namespace string `bun:",notnull" json:"namespace"` + RepoIds []string `bun:",notnull,type:jsonb" json:"repo_ids"` + RepoType string `bun:",notnull" json:"repo_type"` + TaskDesc string `bun:"," json:"task_desc"` + Status v1alpha1.WorkflowPhase `bun:"," json:"status"` + Reason string `bun:"," json:"reason"` // reason for status + Image string `bun:",notnull" json:"image"` // ArgoWorkFlow framework + Datasets []string `bun:",notnull,type:jsonb" json:"datasets"` + ResourceId int64 `bun:",nullzero" json:"resource_id"` + ResourceName string `bun:"," json:"resource_name"` + SubmitTime time.Time `bun:",nullzero,notnull,default:current_timestamp" json:"submit_time"` + StartTime time.Time `bun:",nullzero" json:"start_time"` + EndTime time.Time `bun:",nullzero" json:"end_time"` + ResultURL string `bun:"," json:"result_url"` + DownloadURL string `bun:"," json:"download_url"` + FailuresURL string `bun:"," json:"failures_url"` +} + +func (s *ArgoWorkFlowStore) FindByID(ctx context.Context, id int64) (WorkFlow ArgoWorkflow, err error) { + err = s.db.Operator.Core.NewSelect().Model(&WorkFlow).Where("id = ?", id).Scan(ctx, &WorkFlow) + if err != nil { + return + } + return +} + +func (s *ArgoWorkFlowStore) FindByTaskID(ctx context.Context, id string) (WorkFlow ArgoWorkflow, err error) { + err = s.db.Operator.Core.NewSelect().Model(&WorkFlow).Where("task_id = ?", id).Scan(ctx, &WorkFlow) + if err != nil { + return + } + return +} + +func (s *ArgoWorkFlowStore) FindByUsername(ctx context.Context, username string, per, page int) (WorkFlows []ArgoWorkflow, total int, err error) { + query := s.db.Operator.Core. + NewSelect(). + Model(&WorkFlows). + Where("username = ?", username) + + query = query.Order("submit_time DESC"). + Limit(per). + Offset((page - 1) * per) + + err = query.Scan(ctx) + if err != nil { + return + } + total, err = query.Count(ctx) + if err != nil { + return + } + return +} + +func (s *ArgoWorkFlowStore) CreateWorkFlow(ctx context.Context, workFlow ArgoWorkflow) (*ArgoWorkflow, error) { + res, err := s.db.Core.NewInsert().Model(&workFlow).Exec(ctx, &workFlow) + if err := assertAffectedOneRow(res, err); err != nil { + return nil, fmt.Errorf("failed to save WorkFlow in db, error:%w", err) + } + + return &workFlow, nil +} + +// mainly for update status +func (s *ArgoWorkFlowStore) UpdateWorkFlow(ctx context.Context, workFlow ArgoWorkflow) (*ArgoWorkflow, error) { + _, err := s.db.Core.NewUpdate().Model(&workFlow).WherePK().Exec(ctx) + return &workFlow, err +} + +// delete workflow by id +func (s *ArgoWorkFlowStore) DeleteWorkFlow(ctx context.Context, id int64) error { + _, err := s.db.Core.NewDelete().Model(&ArgoWorkflow{}).Where("id = ?", id).Exec(ctx) + return err +} diff --git a/builder/store/database/migrations/20241105124716_create_table_workflows.go b/builder/store/database/migrations/20241105124716_create_table_workflows.go new file mode 100644 index 000000000..8b8b780ac --- /dev/null +++ b/builder/store/database/migrations/20241105124716_create_table_workflows.go @@ -0,0 +1,51 @@ +package migrations + +import ( + "context" + "time" + + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + "github.com/uptrace/bun" + "opencsg.com/csghub-server/common/types" +) + +type ArgoWorkflow struct { + ID int64 `bun:",pk,autoincrement" json:"id"` + Username string `bun:",notnull" json:"username"` + UserUUID string `bun:",notnull" json:"user_uuid"` + TaskName string `bun:",notnull" json:"task_name"` // user input name + TaskId string `bun:",notnull" json:"task_id"` // generated task id + TaskType types.TaskType `bun:",notnull" json:"task_type"` + RepoIds []string `bun:",notnull,type:jsonb" json:"repo_ids"` + RepoType string `bun:",notnull" json:"repo_type"` + TaskDesc string `bun:"," json:"task_desc"` + Status v1alpha1.WorkflowPhase `bun:"," json:"status"` + Reason string `bun:"," json:"reason"` // reason for status + Image string `bun:",notnull" json:"image"` // ArgoWorkFlow framework + Datasets []string `bun:",notnull,type:jsonb" json:"datasets"` + ResourceId int64 `bun:",nullzero" json:"resource_id"` + SubmitTime time.Time `bun:",nullzero,notnull,default:current_timestamp" json:"submit_time"` + StartTime time.Time `bun:",nullzero" json:"start_time"` + EndTime time.Time `bun:",nullzero" json:"end_time"` + ResultURL string `bun:",nullzero" json:"result_url"` + DownloadURL string `bun:",nullzero" json:"download_url"` + FailuresURL string `bun:",nullzero" json:"failures_url"` +} + +func init() { + Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error { + err := createTables(ctx, db, ArgoWorkflow{}) + if err != nil { + return err + } + _, err = db.NewCreateIndex(). + Model((*ArgoWorkflow)(nil)). + Index("idx_workflow_user_uuid"). + Column("username", "task_id"). + Exec(ctx) + return err + + }, func(ctx context.Context, db *bun.DB) error { + return dropTables(ctx, db, ArgoWorkflow{}) + }) +} diff --git a/builder/store/database/migrations/20241111095821_create_tag_rules.go b/builder/store/database/migrations/20241111095821_create_tag_rules.go new file mode 100644 index 000000000..fd9aa2885 --- /dev/null +++ b/builder/store/database/migrations/20241111095821_create_tag_rules.go @@ -0,0 +1,35 @@ +package migrations + +import ( + "context" + "time" + + "github.com/uptrace/bun" +) + +type TagRule struct { + ID int64 `bun:",pk,autoincrement" json:"id"` + RepoName string `bun:",notnull" json:"repo_name"` + RepoType string `bun:",notnull" json:"repo_type"` + Category string `bun:",notnull" json:"category"` + TagName string `bun:",notnull" json:"tag_name"` + Tag Tag `bun:",rel:has-one,join:tag_name=name"` + CreatedAt time.Time `bun:",nullzero,notnull,skipupdate,default:current_timestamp" json:"created_at"` +} + +func init() { + Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error { + err := createTables(ctx, db, TagRule{}) + if err != nil { + return err + } + _, err = db.NewCreateIndex(). + Model((*TagRule)(nil)). + Index("idx_dataset_tag_name_type"). + Column("repo_name", "repo_type"). + Exec(ctx) + return err + }, func(ctx context.Context, db *bun.DB) error { + return dropTables(ctx, db, TagRule{}) + }) +} diff --git a/builder/store/database/migrations/20241111095847_init_tag_rule.down.sql b/builder/store/database/migrations/20241111095847_init_tag_rule.down.sql new file mode 100644 index 000000000..cced29bfc --- /dev/null +++ b/builder/store/database/migrations/20241111095847_init_tag_rule.down.sql @@ -0,0 +1,9 @@ +SET statement_timeout = 0; + +--bun:split + +Delete from public.tags where name in ('Knowledge','Reasoning','Examination','Understanding','Code','Other'); + +--bun:split + +Delete from public.tag_rules; diff --git a/builder/store/database/migrations/20241111095847_init_tag_rule.up.sql b/builder/store/database/migrations/20241111095847_init_tag_rule.up.sql new file mode 100644 index 000000000..09b4f284d --- /dev/null +++ b/builder/store/database/migrations/20241111095847_init_tag_rule.up.sql @@ -0,0 +1,112 @@ +SET statement_timeout = 0; + +--bun:split + +INSERT INTO public.tag_categories ("name", "scope") VALUES( 'evaluation', 'dataset') ON CONFLICT ("name", scope) DO NOTHING; + +--bun:split +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name, created_at, updated_at) VALUES('Knowledge', 'evaluation', '', 'dataset', true, '知识', '2024-11-11 10:42:12.939', '2024-11-11 10:42:12.939') ON CONFLICT ("name", category, scope) DO NOTHING; + +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name, created_at, updated_at) VALUES('Reasoning', 'evaluation', '', 'dataset', true, '推理', '2024-11-11 10:42:12.939', '2024-11-11 10:42:12.939') ON CONFLICT ("name", category, scope) DO NOTHING; + +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name, created_at, updated_at) VALUES('Examination', 'evaluation', '', 'dataset', true, '考试', '2024-11-11 10:42:12.939', '2024-11-11 10:42:12.939') ON CONFLICT ("name", category, scope) DO NOTHING; + +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name, created_at, updated_at) VALUES('Understanding', 'evaluation', '', 'dataset', true, '理解', '2024-11-11 10:42:12.939', '2024-11-11 10:42:12.939') ON CONFLICT ("name", category, scope) DO NOTHING; + +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name, created_at, updated_at) VALUES('Code', 'evaluation', '', 'dataset', true, '代码', '2024-11-11 10:42:12.939', '2024-11-11 10:42:12.939') ON CONFLICT ("name", category, scope) DO NOTHING; + +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name, created_at, updated_at) VALUES('Other', 'evaluation', '', 'dataset', true, '其他', '2024-11-11 10:42:12.939', '2024-11-11 10:42:12.939') ON CONFLICT ("name", category, scope) DO NOTHING; + +--bun:split +ALTER TABLE public.tag_rules ADD CONSTRAINT unique_tag_rules UNIQUE (repo_name, category); +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('wic', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('summedits', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('chid', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('afqmc', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('bustm', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('cluewsc', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('wsc', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('winogrande', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('flores', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('iwslt2017', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('tydiqa', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('xcopa', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('xlsum', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('leval', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('longbench', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('govreports', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('narrativeqa', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('qasper', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('civilcomments', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('crowspairs', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('cvalues', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('jigsawmultilingual', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('truthfulqa', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('advglue', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('ifeval', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; + +--bun:split +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('boolq', 'evaluation', 'Knowledge', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('commonsense_qa', 'evaluation', 'Knowledge', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('natural_question', 'evaluation', 'Knowledge', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('trivia_qa', 'evaluation', 'Knowledge', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; + + +--bun:split +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('cmnli', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('ocnli', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('ocnli_fc', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('ax-b', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('ax-g', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('rte', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('anli', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('xstory_cloze', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('copa', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('record', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('hellaswag', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('piqa', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('siqa', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('math', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('gsm8k', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('theoremqa', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('strategy_qa', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('scibench', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('bbh', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('musr', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; + +--bun:split +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('ceval-exam', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('agieval', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('mmlu', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('gaokao-bench', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('cmmlu', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('ai2_arc', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('xiezhi', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('cmb', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('mmlu-pro', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('chembench', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('mmmlu_lite', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('wikibench', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; + +--bun:split +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('cmrc_dev', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('drcd_dev', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('race', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('openbookqa', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('squad2.0', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('lcsts', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('xsum', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('summscreen', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('eprstmt', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('lambada', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('tnews', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; + +--bun:split +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('humaneval', 'evaluation', 'Code', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('humanevalx', 'evaluation', 'Code', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('mbpp', 'evaluation', 'Code', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('apps', 'evaluation', 'Code', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('ds1000', 'evaluation', 'Code', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('code_generation_lite', 'evaluation', 'Code', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('execution-v2', 'evaluation', 'Code', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('test_generation', 'evaluation', 'Code', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; \ No newline at end of file diff --git a/builder/store/database/migrations/20241117095440_update_argo_workflow_table.down.sql b/builder/store/database/migrations/20241117095440_update_argo_workflow_table.down.sql new file mode 100644 index 000000000..722ddf237 --- /dev/null +++ b/builder/store/database/migrations/20241117095440_update_argo_workflow_table.down.sql @@ -0,0 +1,10 @@ +SET statement_timeout = 0; + +--bun:split + +ALTER TABLE public.argo_workflows DROP COLUMN IF EXISTS cluster_id; +ALTER TABLE public.tag_rules DROP COLUMN IF EXISTS runtime_framework; +ALTER TABLE public.tag_rules DROP COLUMN IF EXISTS resource_name; + +--bun:split +Delete from public.tags where name='opencompass' and category='runtime_framework'; diff --git a/builder/store/database/migrations/20241117095440_update_argo_workflow_table.up.sql b/builder/store/database/migrations/20241117095440_update_argo_workflow_table.up.sql new file mode 100644 index 000000000..15a0e6522 --- /dev/null +++ b/builder/store/database/migrations/20241117095440_update_argo_workflow_table.up.sql @@ -0,0 +1,18 @@ +SET statement_timeout = 0; + +--bun:split + +ALTER TABLE public.argo_workflows ADD COLUMN IF NOT EXISTS cluster_id VARCHAR; +ALTER TABLE public.argo_workflows ADD COLUMN IF NOT EXISTS namespace VARCHAR; +ALTER TABLE public.argo_workflows ADD COLUMN IF NOT EXISTS resource_name VARCHAR; + +--bun:split + +--add opencompass as runtime framework for all tag_rules created in 20241111095847_init_tag_rule.up.sql +--then drop the default the value +ALTER TABLE public.tag_rules ADD COLUMN IF NOT EXISTS runtime_framework VARCHAR DEFAULT 'opencompass'; +ALTER TABLE public.tag_rules ALTER COLUMN runtime_framework DROP DEFAULT; + +--bun:split +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name) VALUES('opencompass', 'runtime_framework', 'evaluation', 'model', true, 'OpenCompass') ON CONFLICT ("name", category, scope) DO NOTHING; +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name) VALUES('opencompass', 'runtime_framework', 'evaluation', 'dataset', true, 'OpenCompass') ON CONFLICT ("name", category, scope) DO NOTHING; diff --git a/builder/store/database/seeds/tag_categories.yml b/builder/store/database/seeds/tag_categories.yml index 9f45a2d83..126bc8b29 100644 --- a/builder/store/database/seeds/tag_categories.yml +++ b/builder/store/database/seeds/tag_categories.yml @@ -1,13 +1,15 @@ categories: -- name: task - scope: model -- name: license - scope: model -- name: framework - scope: model -- name: task - scope: dataset -- name: license - scope: dataset -- name: size - scope: dataset + - name: task + scope: model + - name: license + scope: model + - name: framework + scope: model + - name: task + scope: dataset + - name: license + scope: dataset + - name: size + scope: dataset + - name: evaluation + scope: dataset diff --git a/builder/store/database/tag.go b/builder/store/database/tag.go index 3c0b03fa5..b9c339ce9 100644 --- a/builder/store/database/tag.go +++ b/builder/store/database/tag.go @@ -39,6 +39,7 @@ type TagStore interface { UpsertRepoTags(ctx context.Context, repoID int64, oldTagIDs, newTagIDs []int64) (err error) RemoveRepoTags(ctx context.Context, repoID int64, tagIDs []int64) (err error) FindOrCreate(ctx context.Context, tag Tag) (*Tag, error) + FindTag(ctx context.Context, name, scope, category string) (*Tag, error) } func NewTagStore() TagStore { @@ -105,9 +106,15 @@ func (ts *tagStoreImpl) AllTagsByScope(ctx context.Context, scope TagScope) ([]* func (ts *tagStoreImpl) AllTagsByScopeAndCategory(ctx context.Context, scope TagScope, category string) ([]*Tag, error) { var tags []*Tag - err := ts.db.Operator.Core.NewSelect().Model(&tags). - Where("scope = ? and category = ?", scope, category). - Scan(ctx) + query := ts.db.Operator.Core.NewSelect().Model(&tags) + if scope != "" { + query.Where("scope = ?", scope) + } + if category != "" { + query.Where("category = ?", category) + } + + err := query.Scan(ctx) if err != nil { return nil, fmt.Errorf("failed to select tags by scope,cause: %w", err) } @@ -368,3 +375,16 @@ func (ts *tagStoreImpl) FindOrCreate(ctx context.Context, tag Tag) (*Tag, error) } return &tag, err } + +// find tag by name +func (ts *tagStoreImpl) FindTag(ctx context.Context, name, scope, category string) (*Tag, error) { + var tag Tag + err := ts.db.Operator.Core.NewSelect(). + Model(&tag). + Where("name = ? and scope = ? and category = ?", name, scope, category). + Scan(ctx) + if err != nil { + return nil, err + } + return &tag, nil +} diff --git a/builder/store/database/tag_rule.go b/builder/store/database/tag_rule.go new file mode 100644 index 000000000..cbb49c68f --- /dev/null +++ b/builder/store/database/tag_rule.go @@ -0,0 +1,41 @@ +package database + +import ( + "context" + "time" +) + +type TagRuleStore struct { + db *DB +} + +func NewTagRuleStore() *TagRuleStore { + return &TagRuleStore{db: defaultDB} +} + +// internal use for the relation between dataset and tag +type TagRule struct { + ID int64 `bun:",pk,autoincrement" json:"id"` + RepoName string `bun:",notnull" json:"repo_name"` + RepoType string `bun:",notnull" json:"repo_type"` + Category string `bun:",notnull" json:"category"` + TagName string `bun:",notnull" json:"tag_name"` + RuntimeFramework string `bun:"," json:"runtime_framework"` + Tag Tag `bun:",rel:has-one,join:tag_name=name"` + CreatedAt time.Time `bun:",nullzero,notnull,skipupdate,default:current_timestamp" json:"created_at"` +} + +// find dataset tag by name and related tag with tag name +func (s *TagRuleStore) FindByRepo(ctx context.Context, category string, repoName string, repoType string) (*TagRule, error) { + var TagRule TagRule + err := s.db.Core.NewSelect(). + Model(&TagRule). + Relation("Tag"). + Where("LOWER(tag_rule.repo_name) = LOWER(?) AND tag_rule.repo_type = ?", repoName, repoType). + Limit(1). + Scan(ctx) + if err != nil { + return nil, err + } + return &TagRule, nil +} diff --git a/cmd/csghub-server/cmd/deploy/runner.go b/cmd/csghub-server/cmd/deploy/runner.go index 3fa67b9dd..a9b970fc6 100644 --- a/cmd/csghub-server/cmd/deploy/runner.go +++ b/cmd/csghub-server/cmd/deploy/runner.go @@ -1,16 +1,19 @@ package deploy import ( + "fmt" + "github.com/spf13/cobra" "opencsg.com/csghub-server/api/httpbase" + "opencsg.com/csghub-server/builder/event" "opencsg.com/csghub-server/builder/store/database" "opencsg.com/csghub-server/common/config" - "opencsg.com/csghub-server/servicerunner/router" + "opencsg.com/csghub-server/runner/router" ) var startRunnerCmd = &cobra.Command{ Use: "runner", - Short: "start space runner service", + Short: "start runner service", PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { return }, @@ -25,7 +28,10 @@ var startRunnerCmd = &cobra.Command{ DSN: config.Database.DSN, } database.InitDB(dbConfig) - + err = event.InitEventPublisher(config) + if err != nil { + return fmt.Errorf("fail to initialize message queue, %w", err) + } s, err := router.NewHttpServer(config) if err != nil { return err diff --git a/common/config/config.go b/common/config/config.go index 3018f1f97..f93318094 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -210,6 +210,18 @@ type Config struct { WorkFLow struct { Endpoint string `env:"OPENCSG_WORKFLOW_SERVER_ENDPOINT, default=localhost:7233"` } + + Argo struct { + Namespace string `env:"STARHUB_SERVER_ARGO_NAMESPACE, default=workflows"` + // NamespaceQuota is used to create evaluation with free of charge + QuotaNamespace string `env:"STARHUB_SERVER_ARGO_QUOTA_NAMESPACE, default=workflows-quota"` + QuotaGPUNumber string `env:"STARHUB_SERVER_ARGO_QUOTA_GPU_NUMBER, default=1"` + //job will be deleted after JobTTL seconds once the jobs was done + JobTTL int `env:"STARHUB_SERVER_ARGO_TTL, default=120"` + ServiceAccountName string `env:"STARHUB_SERVER_ARGO_SERVICE_ACCOUNT, default=executor"` + // S3PublicBucket is used to store public files, should set bucket same with portal + S3PublicBucket string `env:"STARHUB_SERVER_ARGO_S3_PUBLIC_BUCKET"` + } } func SetConfigFile(file string) { diff --git a/common/types/accounting.go b/common/types/accounting.go index 2856cc499..303f6a913 100644 --- a/common/types/accounting.go +++ b/common/types/accounting.go @@ -31,6 +31,7 @@ var ( SceneSpace SceneType = 11 // csghub space SceneModelFinetune SceneType = 12 // model finetune SceneMultiSync SceneType = 13 // multi sync + SceneEvaluation SceneType = 14 // model evaluation SceneStarship SceneType = 20 // starship SceneUnknow SceneType = 99 // unknow ) diff --git a/common/types/argo_workflow.go b/common/types/argo_workflow.go new file mode 100644 index 000000000..2dda6853b --- /dev/null +++ b/common/types/argo_workflow.go @@ -0,0 +1,128 @@ +package types + +import ( + "time" + + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" +) + +var WorkFlowFinished = map[v1alpha1.WorkflowPhase]struct{}{ + v1alpha1.WorkflowSucceeded: {}, + v1alpha1.WorkflowFailed: {}, + v1alpha1.WorkflowError: {}, +} + +var _ SensitiveRequestV2 = (*EvaluationReq)(nil) + +func (c *EvaluationReq) GetSensitiveFields() []SensitiveField { + return []SensitiveField{ + { + Name: "task_name", + Value: func() string { + return c.TaskName + }, + Scenario: "nickname_detection", + }, + { + Name: "task_desc", + Value: func() string { + return c.TaskDesc + }, + Scenario: "comment_detection", + }, + } +} + +type TaskType string + +const ( + TaskTypeEvaluation TaskType = "evaluation" + TaskTypeTraining TaskType = "training" + TaskTypeComparison TaskType = "comparison" + TaskTypeLeaderBoard TaskType = "leaderboard" +) + +type EvaluationReq struct { + Username string `json:"-"` + TaskName string `json:"task_name"` + TaskDesc string `json:"task_desc"` + RuntimeFrameworkId int64 `json:"runtime_framework_id"` // ArgoWorkFlow framework + Datasets []string `json:"datasets,omitempty"` + ResourceId int64 `json:"resource_id"` + ModelId string `json:"model_id"` + ShareMode bool `json:"share_mode"` + Token string `json:"-"` + Hardware HardWare `json:"-"` + UserUUID string `json:"-"` + ClusterID string `json:"-"` + Image string `json:"-"` + RepoType string `json:"-"` + TaskType TaskType `json:"-"` + DownloadEndpoint string `json:"-"` + ResourceName string `json:"-"` +} + +type ArgoFlowTemplate struct { + Name string `json:"name"` + Image string `json:"image"` + Command []string `json:"command"` + Args []string `json:"args"` + HardWare HardWare `json:"hardware,omitempty"` + Env map[string]string `json:"env,omitempty"` + Annotation map[string]string `json:"annotation,omitempty"` +} + +type ArgoWorkFlowReq struct { + ClusterID string `json:"cluster_id"` + RepoType string `json:"repo_type"` + Templates []ArgoFlowTemplate `json:"templates,omitempty"` + Entrypoint string `json:"entrypoint"` + Username string `json:"username"` + TaskName string `json:"task_name"` + TaskId string `json:"task_id"` + TaskType TaskType `json:"task_type"` + RepoIds []string `json:"repo_ids"` + TaskDesc string `json:"task_desc"` + Image string `json:"image"` // ArgoWorkFlow framework + Datasets []string `json:"datasets,omitempty"` + ResourceId int64 `json:"resource_id"` + ResourceName string `json:"resource_name"` + UserUUID string `json:"user_uuid"` + ShareMode bool `json:"share_mode"` +} + +type ArgoWorkFlowListRes struct { + List []ArgoWorkFlowRes `json:"list"` + Total int `json:"total"` +} + +type ArgoWorkFlowRes struct { + ID int64 `json:"id"` + RepoIds []string `json:"repo_ids"` + RepoType string `json:"repo_type,omitempty"` + Username string `json:"username"` + TaskName string `json:"task_name"` + TaskId string `json:"task_id"` + TaskType TaskType `json:"task_type"` + TaskDesc string `json:"task_desc"` + Datasets []string `json:"datasets,omitempty"` + ResourceId int64 `json:"resource_id,omitempty"` + Status v1alpha1.WorkflowPhase `json:"status"` + Reason string `json:"reason,omitempty"` + Image string `bun:",notnull" json:"image"` + SubmitTime time.Time `json:"submit_time"` + StartTime time.Time `json:"start_time,omitempty"` + EndTime time.Time `json:"end_time,omitempty"` + ResultURL string `json:"result_url"` + DownloadURL string `json:"download_url"` + FailuresURL string `json:"failures_url"` +} + +type ( + EvaluationDelReq = ArgoWorkFlowDeleteReq +) + +type ArgoWorkFlowDeleteReq struct { + ID int64 `json:"id"` + Username string `json:"username"` +} diff --git a/common/types/model.go b/common/types/model.go index 263394a57..028aa4a1d 100644 --- a/common/types/model.go +++ b/common/types/model.go @@ -154,18 +154,21 @@ type Model struct { // widget UI style: generation,chat WidgetType ModelWidgetType `json:"widget_type" example:"generation"` // url to interact with the model - Status string `json:"status" example:"RUNNING"` - UserLikes bool `json:"user_likes"` - Source RepositorySource `json:"source"` - SyncStatus RepositorySyncStatus `json:"sync_status"` - EnableInference bool `json:"enable_inference"` - EnableFinetune bool `json:"enable_finetune"` - BaseModel string `json:"base_model"` - License string `json:"license"` - CanWrite bool `json:"can_write"` - CanManage bool `json:"can_manage"` - Namespace *Namespace `json:"namespace"` - MirrorLastUpdatedAt time.Time `json:"mirror_last_updated_at"` + Status string `json:"status" example:"RUNNING"` + UserLikes bool `json:"user_likes"` + Source RepositorySource `json:"source"` + SyncStatus RepositorySyncStatus `json:"sync_status"` + EnableInference bool `json:"enable_inference"` + EnableFinetune bool `json:"enable_finetune"` + EnableEvaluation bool `json:"enable_evaluation"` + BaseModel string `json:"base_model"` + License string `json:"license"` + CanWrite bool `json:"can_write"` + CanManage bool `json:"can_manage"` + Namespace *Namespace `json:"namespace"` + RecomOpWeight int `json:"recom_op_weight,omitempty"` + SensitiveCheckStatus string `json:"sensitive_check_status"` + MirrorLastUpdatedAt time.Time `json:"mirror_last_updated_at"` } type SDKModelInfo struct { @@ -275,6 +278,8 @@ const ( InferenceType = 1 // inference endpoint FinetuneType = 2 // finetune ServerlessType = 3 // serverless + EvaluationType = 4 // evaluation + UnknownType = -1 // unknown case ) type DeployActReq struct { diff --git a/common/types/tag.go b/common/types/tag.go index 847e03105..7e20074f0 100644 --- a/common/types/tag.go +++ b/common/types/tag.go @@ -11,3 +11,14 @@ type RepoTag struct { CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` } + +type TagCategory string + +const ( + TaskCategory TagCategory = "task" + LicenseCategory TagCategory = "license" + FrameworkCategory TagCategory = "framework" + SizeCategory TagCategory = "size" + LanguageCategory TagCategory = "language" + EvaluationCategory TagCategory = "evaluation" +) diff --git a/common/types/user.go b/common/types/user.go index 5c4b8b0f6..18d4ab65a 100644 --- a/common/types/user.go +++ b/common/types/user.go @@ -141,6 +141,7 @@ type ( UserCollectionReq = UserDatasetsReq DeleteUserTokenRequest = CreateUserTokenRequest UserPromptsReq = UserDatasetsReq + UserEvaluationReq = UserDatasetsReq ) type PageOpts struct { diff --git a/component/callback/git_callback.go b/component/callback/git_callback.go index 4c41798fc..0de8bfe5a 100644 --- a/component/callback/git_callback.go +++ b/component/callback/git_callback.go @@ -37,6 +37,7 @@ type GitCallbackComponent struct { ras database.RuntimeArchitecturesStore rfs database.RuntimeFrameworksStore ts database.TagStore + dt database.TagRuleStore // set visibility if file content is sensitive setRepoVisibility bool pp component.PromptComponent @@ -79,6 +80,7 @@ func NewGitCallback(config *config.Config) (*GitCallbackComponent, error) { if config.SensitiveCheck.Enable { modSvcClient = rpc.NewModerationSvcHttpClient(fmt.Sprintf("%s:%d", config.Moderation.Host, config.Moderation.Port)) } + dt := database.NewTagRuleStore() return &GitCallbackComponent{ config: config, gs: gs, @@ -97,6 +99,7 @@ func NewGitCallback(config *config.Config) (*GitCallbackComponent, error) { rfs: rfs, pp: pp, ts: ts, + dt: dt, maxPromptFS: config.Dataset.PromptMaxJsonlFileSize, }, nil } @@ -211,7 +214,7 @@ func (c *GitCallbackComponent) modifyFiles(ctx context.Context, repoType, namesp for _, fileName := range fileNames { slog.Debug("modify file", slog.String("file", fileName)) // update model runtime - c.updateModelRuntimeFrameworks(ctx, repoType, namespace, repoName, ref, fileName, false) + c.updateRepoRelations(ctx, repoType, namespace, repoName, ref, fileName, false) // only care about readme file under root directory if fileName != types.ReadmeFileName { continue @@ -233,7 +236,7 @@ func (c *GitCallbackComponent) removeFiles(ctx context.Context, repoType, namesp for _, fileName := range fileNames { slog.Debug("remove file", slog.String("file", fileName)) // update model runtime - c.updateModelRuntimeFrameworks(ctx, repoType, namespace, repoName, ref, fileName, true) + c.updateRepoRelations(ctx, repoType, namespace, repoName, ref, fileName, true) // only care about readme file under root directory if fileName == types.ReadmeFileName { // use empty content to clear all the meta tags @@ -278,7 +281,7 @@ func (c *GitCallbackComponent) addFiles(ctx context.Context, repoType, namespace for _, fileName := range fileNames { slog.Debug("add file", slog.String("file", fileName)) // update model runtime - c.updateModelRuntimeFrameworks(ctx, repoType, namespace, repoName, ref, fileName, false) + c.updateRepoRelations(ctx, repoType, namespace, repoName, ref, fileName, false) // only care about readme file under root directory if fileName == types.ReadmeFileName { content, err := c.getFileRaw(repoType, namespace, repoName, ref, fileName) @@ -374,10 +377,55 @@ func (c *GitCallbackComponent) getFileRaw(repoType, namespace, repoName, ref, fi return content, nil } -func (c *GitCallbackComponent) updateModelRuntimeFrameworks(ctx context.Context, repoType, namespace, repoName, ref, fileName string, deleteAction bool) { +// update repo relations +func (c *GitCallbackComponent) updateRepoRelations(ctx context.Context, repoType, namespace, repoName, ref, fileName string, deleteAction bool) { slog.Debug("update model relation for git callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("repoType", repoType), slog.Any("fileName", fileName), slog.Any("branch", ref)) + if repoType == fmt.Sprintf("%ss", types.ModelRepo) { + c.updateModelRuntimeFrameworks(ctx, repoType, namespace, repoName, ref, fileName, deleteAction) + } + if repoType == fmt.Sprintf("%ss", types.DatasetRepo) { + c.updateDatasetTags(ctx, namespace, repoName) + } +} + +// update dataset tags for evaluation +func (c *GitCallbackComponent) updateDatasetTags(ctx context.Context, namespace, repoName string) { + + repo, err := c.rs.FindByPath(ctx, types.DatasetRepo, namespace, repoName) + if err != nil || repo == nil { + slog.Warn("fail to query repo for in callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("error", err)) + return + } + // check if it's evaluation dataset + evalDataset, err := c.dt.FindByRepo(ctx, string(types.EvaluationCategory), repoName, string(types.DatasetRepo)) + if err != nil { + if errors.Is(err, component.ErrNotFound) { + // no need add tag + return + } + slog.Warn("fail to query dataset tag in git callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("error", err)) + } + tagIds := []int64{} + tagIds = append(tagIds, evalDataset.Tag.ID) + if evalDataset.RuntimeFramework != "" { + rTag, _ := c.ts.FindTag(ctx, evalDataset.RuntimeFramework, string(types.DatasetRepo), evalDataset.Category) + if rTag != nil { + tagIds = append(tagIds, rTag.ID) + } + } + + err = c.ts.UpsertRepoTags(ctx, repo.ID, []int64{}, tagIds) + if err != nil { + slog.Warn("fail to add dataset tag to model repo", slog.Any("repoId", repo.ID), slog.Any("tag id", evalDataset.Tag.ID), slog.Any("error", err)) + slog.Warn("fail to add dataset tag", slog.Any("repoId", repo.ID), slog.Any("tag id", tagIds), slog.Any("error", err)) + } + +} + +// update model runtime frameworks +func (c *GitCallbackComponent) updateModelRuntimeFrameworks(ctx context.Context, repoType, namespace, repoName, ref, fileName string, deleteAction bool) { // must be model repo and config.json - if repoType != fmt.Sprintf("%ss", types.ModelRepo) || fileName != component.ConfigFileName || ref != ("refs/heads/"+component.MainBranch) { + if repoType != fmt.Sprintf("%ss", types.ModelRepo) || fileName != component.ConfigFileName || (ref != ("refs/heads/"+component.MainBranch) && ref != ("refs/heads/"+component.MasterBranch)) { return } repo, err := c.rs.FindByPath(ctx, types.ModelRepo, namespace, repoName) diff --git a/component/evaluation.go b/component/evaluation.go new file mode 100644 index 000000000..70f6d8b77 --- /dev/null +++ b/component/evaluation.go @@ -0,0 +1,105 @@ +package component + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "opencsg.com/csghub-server/builder/deploy" + "opencsg.com/csghub-server/builder/store/database" + "opencsg.com/csghub-server/common/config" + "opencsg.com/csghub-server/common/types" +) + +type evaluationComponentImpl struct { + deployer deploy.Deployer + us database.UserStore + ms database.ModelStore + ss database.SpaceResourceStore + tokenStore database.AccessTokenStore + rtfm database.RuntimeFrameworksStore + config *config.Config + ac AccountingComponent +} + +type EvaluationComponent interface { + // Create argo workflow + CreateEvaluation(ctx context.Context, req *types.EvaluationReq) (*types.ArgoWorkFlowRes, error) + DeleteEvaluation(ctx context.Context, req types.ArgoWorkFlowDeleteReq) error +} + +func NewEvaluationComponent(config *config.Config) (EvaluationComponent, error) { + c := &evaluationComponentImpl{} + c.deployer = deploy.NewDeployer() + c.us = database.NewUserStore() + c.ms = database.NewModelStore() + c.ss = database.NewSpaceResourceStore() + c.tokenStore = database.NewAccessTokenStore() + c.rtfm = database.NewRuntimeFrameworksStore() + c.config = config + ac, err := NewAccountingComponent(config) + if err != nil { + return nil, fmt.Errorf("failed to create accounting component, %w", err) + } + c.ac = ac + return c, nil +} + +// Create argo workflow +func (c *evaluationComponentImpl) CreateEvaluation(ctx context.Context, req *types.EvaluationReq) (*types.ArgoWorkFlowRes, error) { + user, err := c.us.FindByUsername(ctx, req.Username) + if err != nil { + return nil, fmt.Errorf("failed to get current user %s, error:%w", req.Username, err) + } + result := strings.Split(req.ModelId, "/") + _, err = c.ms.FindByPath(ctx, result[0], result[1]) + if err != nil { + return nil, fmt.Errorf("cannot find model, %w", err) + } + + token, err := c.tokenStore.FindByUID(context.Background(), user.ID) + if err != nil { + return nil, fmt.Errorf("cant get git access token:%w", err) + } + req.Token = token.Token + var hardware types.HardWare + if req.ResourceId != 0 { + resource, err := c.ss.FindByID(ctx, req.ResourceId) + if err != nil { + return nil, fmt.Errorf("cannot find resource, %w", err) + } + err = json.Unmarshal([]byte(resource.Resources), &hardware) + if err != nil { + return nil, fmt.Errorf("invalid hardware setting, %w", err) + } + if hardware.Gpu.Num == "" { + return nil, fmt.Errorf("evaluation requires GPU or NPU resources") + } + req.ClusterID = resource.ClusterID + req.ResourceName = resource.Name + } else { + // for share mode + hardware.Gpu.Num = c.config.Argo.QuotaGPUNumber + hardware.Gpu.ResourceName = "nvidia.com/gpu" + hardware.Cpu.Num = "12" + hardware.Memory = "32Gi" + } + frame, err := c.rtfm.FindEnabledByID(ctx, req.RuntimeFrameworkId) + if err != nil { + return nil, fmt.Errorf("cannot find available runtime framework, %w", err) + } + req.Hardware = hardware + // choose image + containerImg := frame.FrameImage + req.UserUUID = user.UUID + req.Image = containerImg + req.RepoType = string(types.ModelRepo) + req.TaskType = types.TaskTypeEvaluation + req.DownloadEndpoint = c.config.Model.DownloadEndpoint + return c.deployer.SubmitEvaluation(ctx, *req) +} + +func (c *evaluationComponentImpl) DeleteEvaluation(ctx context.Context, req types.ArgoWorkFlowDeleteReq) error { + return c.deployer.DeleteEvaluation(ctx, req) +} diff --git a/component/model.go b/component/model.go index 937fa64d4..2b5afe17f 100644 --- a/component/model.go +++ b/component/model.go @@ -476,6 +476,10 @@ func (c *modelComponentImpl) Show(ctx context.Context, namespace, name, currentU if len(finetunes) > 0 { resModel.EnableFinetune = true } + evaluations, _ := c.rrtfms.GetByRepoIDsAndType(ctx, model.Repository.ID, types.EvaluationType) + if len(evaluations) > 0 { + resModel.EnableEvaluation = true + } return resModel, nil } @@ -1129,17 +1133,19 @@ func (c *modelComponentImpl) ListModelsOfRuntimeFrameworks(ctx context.Context, // define EnableInference enableInference := deployType == types.InferenceType enableFinetune := deployType == types.FinetuneType + enableEvaluation := deployType == types.EvaluationType for _, repo := range repos { resModels = append(resModels, types.Model{ - Name: repo.Name, - Nickname: repo.Nickname, - Description: repo.Description, - Path: repo.Path, - RepositoryID: repo.ID, - Private: repo.Private, - EnableInference: enableInference, - EnableFinetune: enableFinetune, + Name: repo.Name, + Nickname: repo.Nickname, + Description: repo.Description, + Path: repo.Path, + RepositoryID: repo.ID, + Private: repo.Private, + EnableInference: enableInference, + EnableFinetune: enableFinetune, + EnableEvaluation: enableEvaluation, }) } return resModels, total, nil diff --git a/component/runtime_architecture.go b/component/runtime_architecture.go index 138adce03..cb79f139d 100644 --- a/component/runtime_architecture.go +++ b/component/runtime_architecture.go @@ -16,6 +16,7 @@ import ( var ( MainBranch string = "main" + MasterBranch string = "master" ConfigFileName string = "config.json" ScanLock sync.Mutex ) diff --git a/component/tag.go b/component/tag.go index 495a44830..dd9579a98 100644 --- a/component/tag.go +++ b/component/tag.go @@ -38,9 +38,12 @@ type tagComponentImpl struct { sensitiveChecker rpc.ModerationSvcClient } -func (tc *tagComponentImpl) AllTags(ctx context.Context) ([]database.Tag, error) { +func (c *tagComponentImpl) AllTags(ctx context.Context) ([]database.Tag, error) { // TODO: query cache for tags at first - return tc.ts.AllTags(ctx) + return c.ts.AllTags(ctx) +} +func (c *tagComponentImpl) AllTagsByScopeAndCategory(ctx context.Context, scope string, category string) ([]*database.Tag, error) { + return c.ts.AllTagsByScopeAndCategory(ctx, database.TagScope(scope), category) } func (c *tagComponentImpl) ClearMetaTags(ctx context.Context, repoType types.RepositoryType, namespace, name string) error { diff --git a/component/user.go b/component/user.go index 544af6508..1dbc8f06c 100644 --- a/component/user.go +++ b/component/user.go @@ -74,6 +74,8 @@ func NewUserComponent(config *config.Config) (UserComponent, error) { return nil, err } c.promptStore = database.NewPromptStore() + c.promptStore = database.NewPromptStore() + c.wfs = database.NewArgoWorkFlowStore() return c, nil } @@ -96,6 +98,7 @@ type userComponentImpl struct { // srs database.SpaceResourceStore // urs *database.UserResourcesStore promptStore database.PromptStore + wfs *database.ArgoWorkFlowStore } func (c *userComponentImpl) Datasets(ctx context.Context, req *types.UserDatasetsReq) ([]types.Dataset, int, error) { @@ -736,3 +739,20 @@ func (c *userComponentImpl) Prompts(ctx context.Context, req *types.UserPromptsR return resPrompts, total, nil } + +func (c *userComponentImpl) Evaluations(ctx context.Context, req *types.UserEvaluationReq) ([]types.ArgoWorkFlowRes, int, error) { + + _, err := c.userStore.FindByUsername(ctx, req.CurrentUser) + if err != nil { + newError := fmt.Errorf("failed to check for the presence of the user, error:%w", err) + return nil, 0, newError + } + + res, err := c.deployer.ListEvaluations(ctx, req.CurrentUser, req.PageSize, req.Page) + if err != nil { + newError := fmt.Errorf("failed to get user evaluations,error:%w", err) + slog.Error(newError.Error()) + return nil, 0, newError + } + return res.List, res.Total, nil +} diff --git a/docker/evaluation/Dockerfile.opencompass b/docker/evaluation/Dockerfile.opencompass new file mode 100644 index 000000000..8efdb702d --- /dev/null +++ b/docker/evaluation/Dockerfile.opencompass @@ -0,0 +1,16 @@ +FROM nvidia/cuda:12.4.1-devel-ubuntu22.04 +RUN apt-get update && apt-get -y install python3.10 python3-pip dumb-init \ + && apt-get clean && rm -rf /var/lib/apt/lists/* +RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple \ + && pip install --no-cache-dir opencompass==0.3.5 \ + csghub-sdk==0.4.4 minio oss2 +RUN pip install --no-cache-dir vllm==0.6.3.post1 openpyxl +COPY ./opencompass/ /etc/csghub/ +RUN mkdir -p /var/log/supervisord && \ + ln -s /usr/bin/python3 /usr/bin/python &&\ + chmod +x /etc/csghub/*.sh +WORKDIR /workspace/ +ENV HUGGINGFACE_HUB_CACHE=/workspace/ \ + HF_HUB_ENABLE_HF_TRANSFER=0 +ENTRYPOINT [ "/usr/bin/dumb-init", "--" ] +CMD ["/etc/csghub/start.sh"] diff --git a/docker/evaluation/opencompass/get_answer_mode.py b/docker/evaluation/opencompass/get_answer_mode.py new file mode 100644 index 000000000..b63b7acec --- /dev/null +++ b/docker/evaluation/opencompass/get_answer_mode.py @@ -0,0 +1,30 @@ +import argparse +from opencompass.utils.datasets_info import DATASETS_MAPPING + + +def get_dataset_key_by_ms_id(ms_id): + for key, value in DATASETS_MAPPING.items(): + if value['ms_id'] == ms_id: + return key + return None + + +def main(ms_id): + names = ms_id.split("/") + # Extract the last part + name = names[-1] + # Generate the new string + opencompass_id = f"opencompass/{name}" + dataset_key = get_dataset_key_by_ms_id(opencompass_id) + if dataset_key: + print(f"{dataset_key}") + else: + return + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Get dataset key by ms_id.') + parser.add_argument('ms_id', type=str, help='The ms_id to search for') + args = parser.parse_args() + + main(args.ms_id) diff --git a/docker/evaluation/opencompass/start.sh b/docker/evaluation/opencompass/start.sh new file mode 100644 index 000000000..08f1d0b74 --- /dev/null +++ b/docker/evaluation/opencompass/start.sh @@ -0,0 +1,75 @@ +#!/bin/bash +mkdir -p /workspace/data +ANSWER_MODE=${ANSWER_MODE:-"gen"} +# download model +csghub-cli download $MODEL_ID -k $ACCESS_TOKEN -e $HF_ENDPOINT -cd /workspace/ +insert_string=$(cat << 'EOF' +"chat_template":"{%- if messages[0]['role'] == 'system' -%}{%- set system_message = messages[0]['content'] -%}{%- set messages = messages[1:] -%}{%- else -%}{% set system_message = '' -%}{%- endif -%}{{ bos_token + system_message }}{%- for message in messages -%}{%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{%- endif -%}{%- if message['role'] == 'user' -%}{{ 'USER: ' + message['content'] + '\\n' }}{%- elif message['role'] == 'assistant' -%}{{ 'ASSISTANT: ' + message['content'] + eos_token + '\\n' }}{%- endif -%}{%- endfor -%}{%- if add_generation_prompt -%}{{ 'ASSISTANT:' }}{% endif %}", +EOF +) +repo_tokenizer_config="/workspace/$MODEL_ID/tokenizer_config.json" +# fix some model does not contain chat_template +if ! grep -q "chat_template" "$repo_tokenizer_config"; then + filename="/tmp/tokenizer_config.json" + cp "/workspace/$MODEL_ID/tokenizer_config.json" $filename + awk -v ins="$insert_string" '/tokenizer_class/ {print; print ins; next}1' "$filename" > tmpfile && mv -f tmpfile $repo_tokenizer_config +fi +# download datasets +IFS=',' read -r -a dataset_repos <<< "$DATASET_IDS" +# Loop through the array and print each value +dataset_tasks="" +dataset_tasks_ori="" +for repo in "${dataset_repos[@]}"; do + # check $dataset existing + if [ ! -d "/workspace/data/$repo" ]; then + echo "Start downloading dataset $repo..." + csghub-cli download $repo -t dataset -k $ACCESS_TOKEN -e $HF_ENDPOINT -cd /tmp/ + # mv "$dataset" to "/workspace/data/" + mv -f "/tmp/$repo" "/workspace/data/" + fi + # get answer mode + task_path=`python -W ignore /etc/csghub/get_answer_mode.py $repo` + if [ -z "$task_path" ]; then + echo "task_path is empty for dataset $repo" + exit 1 + fi + datasets_conf_dir="/usr/local/lib/python3.10/dist-packages/opencompass/configs/datasets/" + dataset_conf_file=`find $datasets_conf_dir -type f -name "*.py" -exec grep -rl "'$task_path'" {} + | head -n 1` + if [ -z "$dataset_conf_file" ]; then + echo "Cannot find dataset config location for $task_path" + exit 1 + fi + #get dataset_location dir + dataset_conf_dir=`dirname $dataset_conf_file` + task_conf_file=`find $dataset_conf_dir -type f -name "*$ANSWER_MODE.py" | head -n 1` + if [ -n "$task_conf_file" ]; then + task=`basename $task_conf_file | cut -d'.' -f1` + dataset_tasks="$dataset_tasks $task" + ori_name=`basename $repo` + dataset_tasks_ori="$dataset_tasks_ori $ori_name" + continue + fi +done +# start evaluation +if [ -z "$dataset_tasks" ]; then + echo "dataset_tasks is empty for dataset $DATASET_IDS" + exit 1 +fi +if [ -z "$GPU_NUM" ]; then + GPU_NUM=1 +fi +#LimitedMaxToken is gpu_num multiplied by 4096 +LimitedMaxToken=$(($GPU_NUM * 4096)) + +opencompass --datasets $dataset_tasks --work-dir /workspace/output --hf-type chat --hf-path /workspace/$MODEL_ID -a vllm --max-out-len 100 --max-seq-len $LimitedMaxToken --batch-size 8 --hf-num-gpus $GPU_NUM --max-num-workers $GPU_NUM + +# upload result to mino server +output_dir=`ls -dt /workspace/output/* |head -n 1` +csv_file=`ls -d $output_dir/summary/*.csv` +python /etc/csghub/upload_files.py convert "$csv_file" +json_file=`ls -d $output_dir/summary/*.json` +python /etc/csghub/upload_files.py summary --file $json_file --tasks $dataset_tasks_ori +upload_json_file=`ls -d $output_dir/summary/*upload.json` +upload_xlsx_file=`ls -d $output_dir/summary/*upload.xlsx` +python /etc/csghub/upload_files.py upload "$upload_json_file,$upload_xlsx_file" +echo "finish evaluation for $MODEL_ID" diff --git a/docker/evaluation/opencompass/upload_files.py b/docker/evaluation/opencompass/upload_files.py new file mode 100644 index 000000000..3e72ccd30 --- /dev/null +++ b/docker/evaluation/opencompass/upload_files.py @@ -0,0 +1,201 @@ +import os +import argparse +from datetime import datetime +from minio import Minio +from minio.error import S3Error +from pathlib import Path +import pandas as pd +import json +import oss2 +import csv + +access_key_id = os.environ['S3_ACCESS_ID'] +access_key_secret = os.environ['S3_ACCESS_SECRET'] +bucket_name = os.environ['S3_BUCKET'] +endpoint = os.environ['S3_ENDPOINT'] +s3_ssl_enabled = json.loads(os.environ['S3_SSL_ENABLED']) +if endpoint.find("aliyuncs.com") == -1: + client = Minio(endpoint, access_key=access_key_id, secret_key=access_key_secret, secure=s3_ssl_enabled) +else: + auth = oss2.Auth(access_key_id, access_key_secret) + bucket = oss2.Bucket(auth, endpoint, bucket_name) + + +def generate_file_name(name): + # Get the current date and time + now = datetime.now() + + # Format the string as YYYYMMDD_HHMMSS + formatted_uuid = now.strftime("%Y%m%d_%H%M%S") + + return f"{name}_{formatted_uuid}" + + +def upload_to_minio(object_name, location_file): + # Make the bucket if it doesn't exist. + found = client.bucket_exists(bucket_name) + if not found: + client.make_bucket(bucket_name) + print("Created bucket", bucket_name) + else: + print("Bucket", bucket_name, "already exists") + + # Upload the file, renaming it in the process + client.fput_object( + bucket_name, object_name, location_file, + ) + + +def upload_to_ali(object_name, location_file): + # Upload the file, renaming it in the process + bucket.put_object_from_file(object_name, location_file) + + +def upload(files): + output = [] + # get schema based on s3_ssl_enabled + schema = "https" if s3_ssl_enabled else "http" + fileName = generate_file_name("result") + for file in files.split(','): + suffix = Path(file).suffix + object_name = f"evaluation/{fileName}{suffix}" + # check if the endpoint is aliyun oss + if endpoint.find("aliyuncs.com") != -1: + upload_to_ali(object_name, file) + file_url = f"https://{bucket_name}.{endpoint}/{object_name}" + else: + upload_to_minio(object_name, file) + file_url = f"{schema}://{endpoint}/{bucket_name}/{object_name}" + output.append(file_url) + try: + with open('/tmp/output.txt', 'w') as file: + file.write(",".join(output)) + print("Output written to /tmp/output.txt") + print(f'Successfully uploaded to {file_url}') + except Exception as e: + print(f"Error writing to file: {e}") + + +def csv_to_json(csv_file_path): + # Read the CSV file + with open(csv_file_path, mode='r', newline='', encoding='utf-8') as csv_file: + csv_reader = csv.DictReader(csv_file) # Using DictReader to read as dictionaries + data = list(csv_reader) # Convert to a list of dictionaries + + # Write the JSON output + json_file_path = os.path.splitext(csv_file_path)[0] + '.json' + with open(json_file_path, mode='w', encoding='utf-8') as json_file: + json.dump(data, json_file, indent=4) # Pretty print the JSON output + + print(f'Successfully converted {csv_file_path} to {json_file_path}') + + +column = [ + { + "title": { + "zh-CN": "数据集", + "en-US": "Dataset" + }, + "width": 220, + "key": "dataset", + "fixed": "left" + }, + { + "title": { + "zh-CN": "指标", + "en-US": "Metric" + }, + "width": 130, + "key": "metric", + "fixed": "left" + }, + { + "title": { + "zh-CN": "模式", + "en-US": "Mode" + }, + "width": 100, + "key": "mode", + "fixed": "left" + } +] + + +def json_to_summary(jsonPath, tasks): + with open(jsonPath, 'r', encoding='utf-8') as f: + jsonObj = json.load(f) + keywords = tasks + summary_data = [] + model_name = "" + xlsx_json = {} + # generate summary data + for item in jsonObj: + item_new = item.copy() + if item_new["dataset"] in keywords: + keys = list(item_new.keys()) + model_name = item_new[keys[-1]] + item_new['id'] = len(summary_data) + 1 + summary_data.append(item_new) + column.append( + { + "title": { + "zh-CN": model_name, + "en-US": model_name + }, + "width": 200, + "key": model_name, + "customizeRender": { + "sorter": "descend" + } + } + ) + summary = { + "column": column, + "data": summary_data + } + final_json = {"summary": summary} + xlsx_json['summary'] = summary_data + # generate detail data + for item in keywords: + sub_data = [] + for sub_item in jsonObj: + item_new = sub_item.copy() + if item in item_new["dataset"]: + item_new['id'] = len(sub_data) + 1 + sub_data.append(item_new) + xlsx_json[item] = sub_data + final_json[item] = {"column": column, "data": sub_data} + + json_file_path = os.path.splitext(jsonPath)[0] + '_upload.json' + with open(json_file_path, 'w', encoding='utf-8') as f: + json.dump(final_json, f, ensure_ascii=False, indent=4) + + xlsx_file = os.path.splitext(jsonPath)[0] + '_upload.xlsx' + with pd.ExcelWriter(xlsx_file) as writer: + for sheet_name, records in xlsx_json.items(): + df = pd.DataFrame(records) + df.to_excel(writer, sheet_name=sheet_name, index=False) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Get upload files.') + subparsers = parser.add_subparsers(dest='command', required=True) + + parser_a = subparsers.add_parser('upload', help='upload files') + parser_a.add_argument('files', type=str, help='Name to greet') + + parser_b = subparsers.add_parser('convert', help='Convert csv to json') + parser_b.add_argument('file', type=str, help='Convert csv to json') + + parser_c = subparsers.add_parser('summary', help='Convert json to json summary') + parser_c.add_argument('--file', type=str, help='Convert json to json summary') + parser_c.add_argument('--tasks', nargs='+', type=str, help='task list') + + args = parser.parse_args() + + if args.command == 'upload': + upload(args.files) + elif args.command == 'convert': + csv_to_json(args.file) + elif args.command == 'summary': + json_to_summary(args.file, args.tasks) diff --git a/docker/inference/supervisord.conf b/docker/inference/supervisord.conf index ee6718153..06cf52012 100644 --- a/docker/inference/supervisord.conf +++ b/docker/inference/supervisord.conf @@ -7,6 +7,7 @@ pidfile=/var/run/supervisord.pid [program:inference] +autorestart=false stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr diff --git a/docs/docs.go b/docs/docs.go index 45b5624f1..39d1ac09c 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2182,6 +2182,118 @@ const docTemplate = `{ } } }, + "/evaluations": { + "post": { + "security": [ + { + "ApiKey": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Evaluation" + ], + "summary": "run model evaluation", + "parameters": [ + { + "type": "string", + "description": "namespace", + "name": "namespace", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "name", + "name": "name", + "in": "path", + "required": true + }, + { + "description": "body setting of evaluation", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/types.EvaluationReq" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "string" + } + }, + "400": { + "description": "Bad request", + "schema": { + "$ref": "#/definitions/types.APIBadRequest" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/types.APIInternalServerError" + } + } + } + } + }, + "/evaluations/{id}": { + "delete": { + "security": [ + { + "ApiKey": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Evaluation" + ], + "summary": "delete model evaluation", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "string" + } + }, + "400": { + "description": "Bad request", + "schema": { + "$ref": "#/definitions/types.APIBadRequest" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/types.APIInternalServerError" + } + } + } + } + }, "/events": { "post": { "consumes": [ @@ -9882,6 +9994,24 @@ const docTemplate = `{ "Tag" ], "summary": "Get all tags", + "parameters": [ + { + "type": "string", + "description": "category name", + "name": "category", + "in": "query" + }, + { + "enum": [ + "model", + "dataset" + ], + "type": "string", + "description": "scope name", + "name": "scope", + "in": "query" + } + ], "responses": { "200": { "description": "tags", @@ -10646,6 +10776,69 @@ const docTemplate = `{ } } }, + "/user/{username}/evaluations": { + "get": { + "security": [ + { + "ApiKey": [] + } + ], + "description": "get user evaluations", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "User" + ], + "summary": "Get user evaluations", + "parameters": [ + { + "type": "string", + "description": "username", + "name": "username", + "in": "path", + "required": true + }, + { + "type": "integer", + "default": 20, + "description": "per", + "name": "per", + "in": "query" + }, + { + "type": "integer", + "default": 1, + "description": "per page", + "name": "page", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/types.Response" + } + }, + "400": { + "description": "Bad request", + "schema": { + "$ref": "#/definitions/types.APIBadRequest" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/types.APIInternalServerError" + } + } + } + } + }, "/user/{username}/finetune/instances": { "get": { "security": [ @@ -14277,7 +14470,8 @@ const docTemplate = `{ "enum": [ 0, 1, - 2 + 2, + 4 ], "type": "integer", "default": 1, @@ -15245,9 +15439,6 @@ const docTemplate = `{ "database.Namespace": { "type": "object", "properties": { - "created_at": { - "type": "string" - }, "id": { "type": "integer" }, @@ -15260,9 +15451,6 @@ const docTemplate = `{ "path": { "type": "string" }, - "updated_at": { - "type": "string" - }, "user": { "$ref": "#/definitions/database.User" }, @@ -16901,6 +17089,36 @@ const docTemplate = `{ } } }, + "types.EvaluationReq": { + "type": "object", + "properties": { + "datasets": { + "type": "array", + "items": { + "type": "string" + } + }, + "model_id": { + "type": "string" + }, + "resource_id": { + "type": "integer" + }, + "runtime_framework_id": { + "description": "ArgoWorkFlow framework", + "type": "integer" + }, + "share_mode": { + "type": "boolean" + }, + "task_desc": { + "type": "string" + }, + "task_name": { + "type": "string" + } + } + }, "types.Event": { "type": "object", "properties": { @@ -17161,6 +17379,9 @@ const docTemplate = `{ "downloads": { "type": "integer" }, + "enable_evaluation": { + "type": "boolean" + }, "enable_finetune": { "type": "boolean" }, diff --git a/docs/swagger.json b/docs/swagger.json index 1922d4076..a6ea36bb7 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2171,6 +2171,118 @@ } } }, + "/evaluations": { + "post": { + "security": [ + { + "ApiKey": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Evaluation" + ], + "summary": "run model evaluation", + "parameters": [ + { + "type": "string", + "description": "namespace", + "name": "namespace", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "name", + "name": "name", + "in": "path", + "required": true + }, + { + "description": "body setting of evaluation", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/types.EvaluationReq" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "string" + } + }, + "400": { + "description": "Bad request", + "schema": { + "$ref": "#/definitions/types.APIBadRequest" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/types.APIInternalServerError" + } + } + } + } + }, + "/evaluations/{id}": { + "delete": { + "security": [ + { + "ApiKey": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Evaluation" + ], + "summary": "delete model evaluation", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "string" + } + }, + "400": { + "description": "Bad request", + "schema": { + "$ref": "#/definitions/types.APIBadRequest" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/types.APIInternalServerError" + } + } + } + } + }, "/events": { "post": { "consumes": [ @@ -9871,6 +9983,24 @@ "Tag" ], "summary": "Get all tags", + "parameters": [ + { + "type": "string", + "description": "category name", + "name": "category", + "in": "query" + }, + { + "enum": [ + "model", + "dataset" + ], + "type": "string", + "description": "scope name", + "name": "scope", + "in": "query" + } + ], "responses": { "200": { "description": "tags", @@ -10635,6 +10765,69 @@ } } }, + "/user/{username}/evaluations": { + "get": { + "security": [ + { + "ApiKey": [] + } + ], + "description": "get user evaluations", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "User" + ], + "summary": "Get user evaluations", + "parameters": [ + { + "type": "string", + "description": "username", + "name": "username", + "in": "path", + "required": true + }, + { + "type": "integer", + "default": 20, + "description": "per", + "name": "per", + "in": "query" + }, + { + "type": "integer", + "default": 1, + "description": "per page", + "name": "page", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/types.Response" + } + }, + "400": { + "description": "Bad request", + "schema": { + "$ref": "#/definitions/types.APIBadRequest" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/types.APIInternalServerError" + } + } + } + } + }, "/user/{username}/finetune/instances": { "get": { "security": [ @@ -14266,7 +14459,8 @@ "enum": [ 0, 1, - 2 + 2, + 4 ], "type": "integer", "default": 1, @@ -15234,9 +15428,6 @@ "database.Namespace": { "type": "object", "properties": { - "created_at": { - "type": "string" - }, "id": { "type": "integer" }, @@ -15249,9 +15440,6 @@ "path": { "type": "string" }, - "updated_at": { - "type": "string" - }, "user": { "$ref": "#/definitions/database.User" }, @@ -16890,6 +17078,36 @@ } } }, + "types.EvaluationReq": { + "type": "object", + "properties": { + "datasets": { + "type": "array", + "items": { + "type": "string" + } + }, + "model_id": { + "type": "string" + }, + "resource_id": { + "type": "integer" + }, + "runtime_framework_id": { + "description": "ArgoWorkFlow framework", + "type": "integer" + }, + "share_mode": { + "type": "boolean" + }, + "task_desc": { + "type": "string" + }, + "task_name": { + "type": "string" + } + } + }, "types.Event": { "type": "object", "properties": { @@ -17150,6 +17368,9 @@ "downloads": { "type": "integer" }, + "enable_evaluation": { + "type": "boolean" + }, "enable_finetune": { "type": "boolean" }, diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 7386d2308..f92b2a9f9 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -263,8 +263,6 @@ definitions: type: object database.Namespace: properties: - created_at: - type: string id: type: integer mirrored: @@ -273,8 +271,6 @@ definitions: $ref: '#/definitions/database.NamespaceType' path: type: string - updated_at: - type: string user: $ref: '#/definitions/database.User' user_id: @@ -1383,6 +1379,26 @@ definitions: example: false type: boolean type: object + types.EvaluationReq: + properties: + datasets: + items: + type: string + type: array + model_id: + type: string + resource_id: + type: integer + runtime_framework_id: + description: ArgoWorkFlow framework + type: integer + share_mode: + type: boolean + task_desc: + type: string + task_name: + type: string + type: object types.Event: properties: c_id: @@ -1562,6 +1578,8 @@ definitions: type: string downloads: type: integer + enable_evaluation: + type: boolean enable_finetune: type: boolean enable_inference: @@ -3726,6 +3744,7 @@ paths: - 0 - 1 - 2 + - 4 in: query name: deploy_type type: integer @@ -5448,6 +5467,77 @@ paths: summary: Create a new discussion comment tags: - Discussion + /evaluations: + post: + consumes: + - application/json + parameters: + - description: namespace + in: path + name: namespace + required: true + type: string + - description: name + in: path + name: name + required: true + type: string + - description: body setting of evaluation + in: body + name: body + required: true + schema: + $ref: '#/definitions/types.EvaluationReq' + produces: + - application/json + responses: + "200": + description: OK + schema: + type: string + "400": + description: Bad request + schema: + $ref: '#/definitions/types.APIBadRequest' + "500": + description: Internal server error + schema: + $ref: '#/definitions/types.APIInternalServerError' + security: + - ApiKey: [] + summary: run model evaluation + tags: + - Evaluation + /evaluations/{id}: + delete: + consumes: + - application/json + parameters: + - description: id + in: path + name: id + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + type: string + "400": + description: Bad request + schema: + $ref: '#/definitions/types.APIBadRequest' + "500": + description: Internal server error + schema: + $ref: '#/definitions/types.APIInternalServerError' + security: + - ApiKey: [] + summary: delete model evaluation + tags: + - Evaluation /events: post: consumes: @@ -10287,6 +10377,18 @@ paths: consumes: - application/json description: get all tags + parameters: + - description: category name + in: query + name: category + type: string + - description: scope name + enum: + - model + - dataset + in: query + name: scope + type: string produces: - application/json responses: @@ -10770,6 +10872,47 @@ paths: summary: Get user datasets tags: - User + /user/{username}/evaluations: + get: + consumes: + - application/json + description: get user evaluations + parameters: + - description: username + in: path + name: username + required: true + type: string + - default: 20 + description: per + in: query + name: per + type: integer + - default: 1 + description: per page + in: query + name: page + type: integer + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/types.Response' + "400": + description: Bad request + schema: + $ref: '#/definitions/types.APIBadRequest' + "500": + description: Internal server error + schema: + $ref: '#/definitions/types.APIInternalServerError' + security: + - ApiKey: [] + summary: Get user evaluations + tags: + - User /user/{username}/finetune/instances: get: consumes: diff --git a/go.mod b/go.mod index 984d225f1..7fbef3f2f 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/naoina/toml v0.1.1 github.com/redis/go-redis/v9 v9.3.0 github.com/sethvargo/go-envconfig v1.1.0 - github.com/spf13/cast v1.1.0 + github.com/spf13/cast v1.5.1 github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.9.0 github.com/swaggo/files v1.0.1 @@ -87,7 +87,9 @@ require ( github.com/googleapis/gax-go/v2 v2.12.2 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/yamux v0.1.2-0.20220728231024-8f49b6f63f18 // indirect github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20210210170715-a8dfcb80d3a7 // indirect github.com/lightstep/lightstep-tracer-go v0.25.0 // indirect @@ -102,6 +104,7 @@ require ( github.com/naoina/go-stringutil v0.1.0 // indirect github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect github.com/nexus-rpc/sdk-go v0.0.11 // indirect github.com/oklog/ulid/v2 v2.0.2 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect @@ -142,6 +145,7 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed // indirect gopkg.in/DataDog/dd-trace-go.v1 v1.32.0 // indirect + modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect ) require ( @@ -154,6 +158,7 @@ require ( github.com/alibabacloud-go/tea-utils/v2 v2.0.4 // indirect github.com/alibabacloud-go/tea-xml v1.1.3 // indirect github.com/aliyun/credentials-go v1.3.2 // indirect + github.com/argoproj/argo-workflows/v3 v3.5.12 github.com/blendle/zapdriver v1.3.1 // indirect github.com/bwmarrin/snowflake v0.3.0 github.com/bytedance/sonic v1.11.2 // indirect @@ -186,7 +191,7 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/go-containerregistry v0.13.0 // indirect + github.com/google/go-containerregistry v0.16.1 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.6.0 github.com/gorilla/context v1.1.1 // indirect @@ -200,7 +205,6 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.17.4 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/leodido/go-urn v1.4.0 // indirect @@ -232,14 +236,13 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/arch v0.7.0 // indirect - golang.org/x/crypto v0.26.0 - golang.org/x/mod v0.17.0 // indirect + golang.org/x/crypto v0.27.0 golang.org/x/net v0.28.0 // indirect golang.org/x/oauth2 v0.22.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.24.0 // indirect - golang.org/x/term v0.23.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/term v0.24.0 // indirect + golang.org/x/text v0.18.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect @@ -248,18 +251,14 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 knative.dev/networking v0.0.0-20240116081125-ce0738abf051 // indirect knative.dev/pkg v0.0.0-20240116073220-b488e7be5902 // indirect - lukechampine.com/uint128 v1.3.0 // indirect mellium.im/sasl v0.3.1 // indirect - modernc.org/cc/v3 v3.41.0 // indirect - modernc.org/ccgo/v3 v3.16.15 // indirect - modernc.org/libc v1.24.1 // indirect + modernc.org/libc v1.41.0 // indirect modernc.org/mathutil v1.6.0 // indirect - modernc.org/memory v1.7.1 // indirect - modernc.org/opt v0.1.3 // indirect - modernc.org/sqlite v1.25.0 // indirect + modernc.org/memory v1.7.2 // indirect + modernc.org/sqlite v1.29.1 // indirect modernc.org/strutil v1.2.0 // indirect modernc.org/token v1.1.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/go.sum b/go.sum index db8989313..70e140cb3 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,6 @@ cloud.google.com/go v0.16.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/monitoring v1.18.0 h1:NfkDLQDG2UR3WYZVQE8kwSbUIEyIqJUPl+aOQdFH1T4= @@ -73,6 +74,9 @@ github.com/aliyun/credentials-go v1.1.2/go.mod h1:ozcZaMR5kLM7pwtCMEpVmQ242suV6q github.com/aliyun/credentials-go v1.3.1/go.mod h1:8jKYhQuDawt8x2+fusqa1Y6mPxemTsBEN04dgcAcYz0= github.com/aliyun/credentials-go v1.3.2 h1:L4WppI9rctC8PdlMgyTkF8bBsy9pyKQEzBD1bHMRl+g= github.com/aliyun/credentials-go v1.3.2/go.mod h1:tlpz4uys4Rn7Ik4/piGRrTbXy2uLKvePgQJJduE+Y5c= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/argoproj/argo-workflows/v3 v3.5.12 h1:BD7fEPY+uF9Iellb/hYpsJkDlku8wAJwMVvTskUkZco= +github.com/argoproj/argo-workflows/v3 v3.5.12/go.mod h1:DecB01a8UXDCjtIh0udY8XfIMIRrWrlbob7hk/uMmg0= github.com/aws/aws-sdk-go v1.50.36 h1:PjWXHwZPuTLMR1NIb8nEjLucZBMzmf84TLoLbD8BZqk= github.com/aws/aws-sdk-go v1.50.36/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/beevik/ntp v1.3.1 h1:Y/srlT8L1yQr58kyPWFPZIxRL8ttx2SRIpVYJqZIlAM= @@ -132,8 +136,8 @@ github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GK github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0= +github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/d5/tengo/v2 v2.17.0 h1:BWUN9NoJzw48jZKiYDXDIF3QrIVZRm1uV1gTzeZ2lqM= github.com/d5/tengo/v2 v2.17.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -160,8 +164,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= -github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.8.0+incompatible h1:1Av9pn2FyxPdvrWNQszj1g6D6YthSmvCfcN6SYclTJg= +github.com/evanphx/json-patch v5.8.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= @@ -171,6 +175,8 @@ github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBD github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.3-0.20170329110642-4da3e2cfbabc/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -181,6 +187,7 @@ github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXE github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/cors v1.7.0 h1:wZX2wuZ0o7rV2/1i7gb4Jn+gW7HBqaP91fizJkBUJOA= github.com/gin-contrib/cors v1.7.0/go.mod h1:cI+h6iOAyxKRtUtC6iF/Si1KSFvGm/gK+kshxlCi8ro= github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4= @@ -292,8 +299,8 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-containerregistry v0.13.0 h1:y1C7Z3e149OJbOPDBxLYR8ITPz8dTKqQwjErKVHJC8k= -github.com/google/go-containerregistry v0.13.0/go.mod h1:J9FQ+eSS4a1aC2GNZxvNpbWhgp0487v+cgiilB4FqDo= +github.com/google/go-containerregistry v0.16.1 h1:rUEt426sR6nyrL3gt+18ibRcvYpKYdpsa5ZW7MA08dQ= +github.com/google/go-containerregistry v0.16.1/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY1hLbf8eeGapA+vbFDCtQ= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -328,11 +335,15 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDa github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0 h1:2cz5kSrxzMYHiWOBbKj8itQm+nRykkB8aMv4ThcHYHA= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0/go.mod h1:w9Y7gY31krpLmrVU5ZPG9H7l9fZuRu5/3R3S3FMtVQ4= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v0.0.0-20170914154624-68e816d1c783/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= github.com/hashicorp/yamux v0.1.2-0.20220728231024-8f49b6f63f18 h1:IVujPV6DRIu1fYF4zUHrfhkngJzmYjelXa+iSUiFZSI= github.com/hashicorp/yamux v0.1.2-0.20220728231024-8f49b6f63f18/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= @@ -371,8 +382,6 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= @@ -465,6 +474,8 @@ github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/nexus-rpc/sdk-go v0.0.11 h1:qH3Us3spfp50t5ca775V1va2eE6z1zMQDZY4mvbw0CI= github.com/nexus-rpc/sdk-go v0.0.11/go.mod h1:TpfkM2Cw0Rlk9drGkoiSMpFqflKTiQLWUNyKJjF8mKQ= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= @@ -532,6 +543,7 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= @@ -555,8 +567,9 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spf13/afero v0.0.0-20170901052352-ee1bd8ee15a1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.1.0 h1:0Rhw4d6C8J9VPu6cjZLIhZ8+aAOHcDvGeKn+cq5Aq3k= github.com/spf13/cast v1.1.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/jwalterweatherman v0.0.0-20170901151539-12bd96e66386/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= @@ -740,6 +753,7 @@ golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -748,6 +762,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -767,11 +782,13 @@ golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20170517211232-f52d1811a629/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -899,6 +916,7 @@ google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 h1:ImUcDPHjTrAqNhlOkSocDLfG9rrNHH7w7uoKWPaWZ8s= google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7/go.mod h1:/3XmxOjePkvmKrHuBy4zNFw7IzxJXtAgdpXi8Ll990U= @@ -915,6 +933,7 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= @@ -952,6 +971,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -984,36 +1004,22 @@ knative.dev/pkg v0.0.0-20240116073220-b488e7be5902 h1:H6+JJN23fhwYWCHY1339sY6uhI knative.dev/pkg v0.0.0-20240116073220-b488e7be5902/go.mod h1:NYk8mMYoLkO7CQWnNkti4YGGnvLxN6MIDbUvtgeo0C0= knative.dev/serving v0.40.1 h1:ZAAK8KwZQYUgCgVi3ay+NqKAISnJQ1OXPYvdtXWUcBc= knative.dev/serving v0.40.1/go.mod h1:Ory3XczDB8b1lH757CSdeDeouY3LHzSamX8IjmStuoU= -lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo= -lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo= mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw= -modernc.org/cc/v3 v3.41.0 h1:QoR1Sn3YWlmA1T4vLaKZfawdVtSiGx8H+cEojbC7v1Q= -modernc.org/cc/v3 v3.41.0/go.mod h1:Ni4zjJYJ04CDOhG7dn640WGfwBzfE0ecX8TyMB0Fv0Y= -modernc.org/ccgo/v3 v3.16.15 h1:KbDR3ZAVU+wiLyMESPtbtE/Add4elztFyfsWoNTgxS0= -modernc.org/ccgo/v3 v3.16.15/go.mod h1:yT7B+/E2m43tmMOT51GMoM98/MtHIcQQSleGnddkUNI= -modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= -modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= -modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= -modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= -modernc.org/libc v1.24.1 h1:uvJSeCKL/AgzBo2yYIPPTy82v21KgGnizcGYfBHaNuM= -modernc.org/libc v1.24.1/go.mod h1:FmfO1RLrU3MHJfyi9eYYmZBfi/R+tqZ6+hQ3yQQUkak= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= +modernc.org/libc v1.41.0 h1:g9YAc6BkKlgORsUWj+JwqoB1wU3o4DE3bM3yvA3k+Gk= +modernc.org/libc v1.41.0/go.mod h1:w0eszPsiXoOnoMJgrXjglgLuDy/bt5RR4y3QzUUeodY= modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.7.1 h1:9J+2/GKTlV503mk3yv8QJ6oEpRCUrRy0ad8TXEPoV8M= -modernc.org/memory v1.7.1/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sqlite v1.25.0 h1:AFweiwPNd/b3BoKnBOfFm+Y260guGMF+0UFk0savqeA= -modernc.org/sqlite v1.25.0/go.mod h1:FL3pVXie73rg3Rii6V/u5BoHlSoyeZeIgKZEgHARyCU= +modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= +modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= +modernc.org/sqlite v1.29.1 h1:19GY2qvWB4VPw0HppFlZCPAbmxFU41r+qjKZQdQ1ryA= +modernc.org/sqlite v1.29.1/go.mod h1:hG41jCYxOAOoO6BRK66AdRlmOcDzXf7qnwlwjUIOqa0= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= -modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY= -modernc.org/tcl v1.15.2/go.mod h1:3+k/ZaEbKrC8ePv8zJWPtBSW0V7Gg9g8rkmhI1Kfs3c= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY= -modernc.org/z v1.7.3/go.mod h1:Ipv4tsdxZRbQyLq9Q1M6gdbkxYzdlrciF2Hi/lS7nWE= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/servicerunner/component/service.go b/runner/component/service.go similarity index 97% rename from servicerunner/component/service.go rename to runner/component/service.go index a249f0672..0b7276e0f 100644 --- a/servicerunner/component/service.go +++ b/runner/component/service.go @@ -49,7 +49,7 @@ func (s *ServiceComponent) GenerateService(ctx context.Context, cluster cluster. environments := []corev1.EnvVar{} appPort := 0 hardware := request.Hardware - resReq, nodeSelector := s.GenerateResources(hardware) + resReq, nodeSelector := GenerateResources(hardware) var err error if request.Env != nil { @@ -218,7 +218,7 @@ func (s *ServiceComponent) GetServicePodsWithStatus(ctx context.Context, cluster return podInstances, nil } -func (s *ServiceComponent) GenerateResources(hardware types.HardWare) (map[corev1.ResourceName]resource.Quantity, map[string]string) { +func GenerateResources(hardware types.HardWare) (map[corev1.ResourceName]resource.Quantity, map[string]string) { nodeSelector := make(map[string]string) resReq := make(map[corev1.ResourceName]resource.Quantity) diff --git a/runner/component/workflow.go b/runner/component/workflow.go new file mode 100644 index 000000000..6c81d08c6 --- /dev/null +++ b/runner/component/workflow.go @@ -0,0 +1,383 @@ +package component + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "path" + "strconv" + "strings" + "time" + + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + "github.com/argoproj/argo-workflows/v3/pkg/client/informers/externalversions" + "github.com/google/uuid" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/cache" + "k8s.io/utils/ptr" + "opencsg.com/csghub-server/builder/deploy/cluster" + "opencsg.com/csghub-server/builder/event" + "opencsg.com/csghub-server/builder/store/database" + "opencsg.com/csghub-server/common/config" + "opencsg.com/csghub-server/common/types" +) + +type workFlowComponentImpl struct { + config *config.Config + wf database.ArgoWorkFlowStore + clusterPool *cluster.ClusterPool + eventPub *event.EventPublisher +} + +type WorkFlowComponent interface { + // Create workflow + CreateWorkflow(ctx context.Context, req types.ArgoWorkFlowReq) (*database.ArgoWorkflow, error) + // Update workflow + UpdateWorkflow(ctx context.Context, update *v1alpha1.Workflow) (*database.ArgoWorkflow, error) + // find workflow by user name + FindWorkFlows(ctx context.Context, username string, per, page int) ([]database.ArgoWorkflow, int, error) + // generate workflow templates + DeleteWorkflow(ctx context.Context, id int64, username string) error + GetWorkflow(ctx context.Context, id int64, username string) (*database.ArgoWorkflow, error) + DeleteWorkflowInargo(ctx context.Context, delete *v1alpha1.Workflow) error + FindWorkFlowById(ctx context.Context, id int64) (database.ArgoWorkflow, error) + RunWorkflowsInformer(clusterPool *cluster.ClusterPool, config *config.Config) + StartAcctRequestFee(wf database.ArgoWorkflow) +} + +func NewWorkFlowComponent(config *config.Config, clusterPool *cluster.ClusterPool) WorkFlowComponent { + wf := database.NewArgoWorkFlowStore() + wc := &workFlowComponentImpl{ + config: config, + wf: wf, + clusterPool: clusterPool, + eventPub: &event.DefaultEventPublisher, + } + //watch workflows events + go wc.RunWorkflowsInformer(clusterPool, config) + return wc +} + +// Create workflow +func (wc *workFlowComponentImpl) CreateWorkflow(ctx context.Context, req types.ArgoWorkFlowReq) (*database.ArgoWorkflow, error) { + // create workflow in db + namespace := wc.config.Argo.Namespace + if req.ShareMode { + namespace = wc.config.Argo.QuotaNamespace + } + cluster, clusterId, err := GetCluster(ctx, wc.clusterPool, req.ClusterID) + if err != nil { + return nil, fmt.Errorf("failed to get cluster by id: %v", err) + } + argowf := &database.ArgoWorkflow{ + Username: req.Username, + UserUUID: req.UserUUID, + TaskName: req.TaskName, + TaskId: req.TaskId, + TaskType: req.TaskType, + RepoIds: req.RepoIds, + TaskDesc: req.TaskDesc, + Image: req.Image, + Datasets: req.Datasets, + ResourceId: req.ResourceId, + ResourceName: req.ResourceName, + ClusterID: clusterId, + RepoType: req.RepoType, + Namespace: namespace, + Status: v1alpha1.WorkflowPhase(v1alpha1.NodePending), + } + // create workflow in argo + awf := generateWorkflow(req, wc.config) + + _, err = cluster.ArgoClient.ArgoprojV1alpha1().Workflows(namespace).Create(ctx, awf, v1.CreateOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to create workflow in argo: %v", err) + } + wf, err := wc.wf.CreateWorkFlow(ctx, *argowf) + if err != nil { + return nil, fmt.Errorf("failed to create workflow in db: %v", err) + } + return wf, nil +} + +func (wc *workFlowComponentImpl) DeleteWorkflow(ctx context.Context, id int64, username string) error { + wf, err := wc.FindWorkFlowById(ctx, id) + if err != nil { + return fmt.Errorf("failed to get workflow by id: %v", err) + } + if wf.Username != username { + return fmt.Errorf("no permission to delete workflow") + } + cluster, _, err := GetCluster(ctx, wc.clusterPool, wf.ClusterID) + if err != nil { + return fmt.Errorf("failed to get cluster by id: %v", err) + } + err = cluster.ArgoClient.ArgoprojV1alpha1().Workflows(wf.Namespace).Delete(ctx, wf.TaskId, v1.DeleteOptions{}) + if err != nil { + slog.Warn("Error deleting argo workflow", slog.Any("error", err)) + } + return wc.wf.DeleteWorkFlow(ctx, wf.ID) +} + +func (wc *workFlowComponentImpl) GetWorkflow(ctx context.Context, id int64, username string) (*database.ArgoWorkflow, error) { + wf, err := wc.FindWorkFlowById(ctx, id) + if err != nil { + return nil, fmt.Errorf("failed to get workflow by id: %v", err) + } + if wf.Username != username { + return nil, fmt.Errorf("no permission to get workflow") + } + return &wf, nil +} + +// Update workflow +func (wc *workFlowComponentImpl) UpdateWorkflow(ctx context.Context, update *v1alpha1.Workflow) (*database.ArgoWorkflow, error) { + oldwf, err := wc.wf.FindByTaskID(ctx, update.Name) + if err != nil { + return nil, err + } + + oldwf.Reason = update.Status.Message + if node, ok := update.Status.Nodes[oldwf.TaskId]; ok { + oldwf.Status = v1alpha1.WorkflowPhase(node.Phase) + if node.Phase == v1alpha1.NodeRunning { + oldwf.StartTime = time.Now() + } + if _, exists := types.WorkFlowFinished[oldwf.Status]; exists { + oldwf.EndTime = time.Now() + } + if node.Outputs != nil && node.Outputs.Parameters != nil { + for _, output := range node.Outputs.Parameters { + if output.Name == "result" && output.Value != nil { + result := strings.Split(output.Value.String(), ",") + oldwf.ResultURL = result[0] + oldwf.DownloadURL = result[1] + wc.StartAcctRequestFee(oldwf) + break + } + } + } + } + return wc.wf.UpdateWorkFlow(ctx, oldwf) +} + +// DeleteWorkflowInargo +func (wc *workFlowComponentImpl) DeleteWorkflowInargo(ctx context.Context, delete *v1alpha1.Workflow) error { + wf, err := wc.wf.FindByTaskID(ctx, delete.Name) + if err != nil { + return fmt.Errorf("failed to get workflow by id: %v", err) + } + // for deleted case,check if the workflow did not finish + if wf.Status == v1alpha1.WorkflowPending || wf.Status == v1alpha1.WorkflowRunning { + wf.Status = v1alpha1.WorkflowFailed + wf.Reason = "deleted by admin" + _, err = wc.wf.UpdateWorkFlow(ctx, wf) + return err + } + return nil +} + +func (wc *workFlowComponentImpl) FindWorkFlowById(ctx context.Context, id int64) (database.ArgoWorkflow, error) { + return wc.wf.FindByID(ctx, id) +} + +// find workflow by user name +func (wc *workFlowComponentImpl) FindWorkFlows(ctx context.Context, username string, per, page int) ([]database.ArgoWorkflow, int, error) { + return wc.wf.FindByUsername(ctx, username, per, page) +} + +// create workflow in argo +func generateWorkflow(req types.ArgoWorkFlowReq, config *config.Config) *v1alpha1.Workflow { + templates := []v1alpha1.Template{} + for _, v := range req.Templates { + resReq, _ := GenerateResources(v.HardWare) + environments := []corev1.EnvVar{} + for key, value := range v.Env { + environments = append(environments, corev1.EnvVar{Name: key, Value: value}) + } + environments = append(environments, corev1.EnvVar{Name: "S3_ACCESS_ID", Value: config.S3.AccessKeyID}) + environments = append(environments, corev1.EnvVar{Name: "S3_ACCESS_SECRET", Value: config.S3.AccessKeySecret}) + environments = append(environments, corev1.EnvVar{Name: "S3_BUCKET", Value: config.Argo.S3PublicBucket}) + environments = append(environments, corev1.EnvVar{Name: "S3_ENDPOINT", Value: config.S3.Endpoint}) + environments = append(environments, corev1.EnvVar{Name: "S3_SSL_ENABLED", Value: strconv.FormatBool(config.S3.EnableSSL)}) + // fix no gpu request case + if v.HardWare.Gpu.ResourceName == "" || v.HardWare.Gpu.Num == "" { + environments = append(environments, corev1.EnvVar{Name: "NVIDIA_VISIBLE_DEVICES", Value: "none"}) + } + + resources := corev1.ResourceRequirements{ + Limits: resReq, + Requests: resReq, + } + + containerImg := v.Image + // add prefix if image is not full path + if !strings.Contains(containerImg, "/") { + if req.RepoType == string(types.ModelRepo) { + // choose registry + containerImg = path.Join(config.Model.DockerRegBase, v.Image) + } else if req.RepoType == string(types.SpaceRepo) { + // choose registry + containerImg = path.Join(config.Space.DockerRegBase, v.Image) + } + } + + templates = append(templates, v1alpha1.Template{ + Name: v.Name, + //NodeSelector: nodeSelector, + Container: &corev1.Container{ + Image: containerImg, + Command: v.Command, + Env: environments, + Args: v.Args, + Resources: resources, + }, + Outputs: v1alpha1.Outputs{ + Parameters: []v1alpha1.Parameter{ + { + Name: "result", + ValueFrom: &v1alpha1.ValueFrom{ + Path: "/tmp/output.txt", + }, + }, + }, + }, + }) + } + + workflowObject := &v1alpha1.Workflow{ + ObjectMeta: v1.ObjectMeta{ + Name: req.TaskId, + Labels: map[string]string{ + "workflow-scope": "csghub", + }, + }, + Spec: v1alpha1.WorkflowSpec{ + Priority: ptr.To(int32(3)), + ServiceAccountName: config.Argo.ServiceAccountName, + Templates: templates, + Entrypoint: req.Entrypoint, + TTLStrategy: &v1alpha1.TTLStrategy{ + // Set TTL here + SecondsAfterCompletion: ptr.To(int32(config.Argo.JobTTL)), + }, + }, + } + + return workflowObject +} + +func (wc *workFlowComponentImpl) RunWorkflowsInformer(clusterPool *cluster.ClusterPool, c *config.Config) { + clientset := clusterPool.Clusters[0].ArgoClient + f := externalversions.NewSharedInformerFactoryWithOptions(clientset, 60*time.Second, externalversions.WithTweakListOptions(func(list *v1.ListOptions) { + list.LabelSelector = "workflow-scope=csghub" + })) + informer := f.Argoproj().V1alpha1().Workflows().Informer() + + stopper := make(chan struct{}) + defer close(stopper) + + defer runtime.HandleCrash() + + eventHandler := cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + // triggered in startup + wf := obj.(*v1alpha1.Workflow) + bg, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err := wc.UpdateWorkflow(bg, wf) + if err != nil { + slog.Error("fail to update workflow", slog.Any("error", err), slog.Any("job id", wf.Name)) + } + }, + UpdateFunc: func(oldObj, newObj interface{}) { + oldWF := oldObj.(*v1alpha1.Workflow) + newWF := newObj.(*v1alpha1.Workflow) + + // compare status + bg, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if newWF.Status.Nodes == nil || oldWF.Status.Nodes == nil { + return + } + if oldWF.Status.Nodes[oldWF.Name].Phase != newWF.Status.Nodes[oldWF.Name].Phase { + _, err := wc.UpdateWorkflow(bg, newWF) + if err != nil { + slog.Error("fail to update workflow", slog.Any("error", err), slog.Any("job id", newWF.Name)) + } + } + }, + DeleteFunc: func(obj interface{}) { + //handle some special case + wf := obj.(*v1alpha1.Workflow) + bg, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + err := wc.DeleteWorkflowInargo(bg, wf) + if err != nil { + slog.Error("fail to update workflow", slog.Any("error", err), slog.Any("job id", wf.Name)) + } + }, + } + + _, _ = informer.AddEventHandler(eventHandler) + + informer.Run(stopper) + if !cache.WaitForCacheSync(stopper, informer.HasSynced) { + runtime.HandleError(fmt.Errorf("timed out waiting for caches to sync")) + return + } +} + +func (wc *workFlowComponentImpl) StartAcctRequestFee(wf database.ArgoWorkflow) { + if wf.ResourceId == 0 { + return + } + duration := wf.EndTime.Sub(wf.StartTime) + minutes := duration.Minutes() + if minutes < 1 { + return + } + slog.Info("start to acct request fee", slog.Any("mins", minutes)) + event := types.METERING_EVENT{ + Uuid: uuid.New(), + UserUUID: wf.UserUUID, + Value: int64(minutes), + ValueType: types.TimeDurationMinType, + Scene: int(types.SceneEvaluation), + OpUID: "", + ResourceID: strconv.FormatInt(wf.ResourceId, 10), + ResourceName: wf.ResourceName, + CustomerID: wf.TaskId, + CreatedAt: time.Now(), + } + str, err := json.Marshal(event) + if err != nil { + slog.Error("error marshal metering event", slog.Any("event", event), slog.Any("error", err)) + return + } + err = wc.eventPub.PublishMeteringEvent(str) + if err != nil { + slog.Error("failed to pub metering event", slog.Any("data", string(str)), slog.Any("error", err)) + } else { + slog.Info("pub metering event success", slog.Any("data", string(str))) + } +} + +// get cluster +func GetCluster(ctx context.Context, clusterPool *cluster.ClusterPool, clusterID string) (*cluster.Cluster, string, error) { + if clusterID == "" { + clusterInfo, err := clusterPool.ClusterStore.ByClusterConfig(ctx, clusterPool.Clusters[0].ID) + if err != nil { + return nil, "", fmt.Errorf("failed to get cluster info: %v", err) + } + return &clusterPool.Clusters[0], clusterInfo.ClusterID, nil + } + cluster, err := clusterPool.GetClusterByID(ctx, clusterID) + if err != nil { + return nil, clusterID, fmt.Errorf("failed to get cluster by id: %v", err) + } + return cluster, clusterID, nil +} diff --git a/servicerunner/handler/k8s.go b/runner/handler/service.go similarity index 94% rename from servicerunner/handler/k8s.go rename to runner/handler/service.go index 15f081644..1d4ec4363 100644 --- a/servicerunner/handler/k8s.go +++ b/runner/handler/service.go @@ -21,10 +21,10 @@ import ( "opencsg.com/csghub-server/builder/store/database" "opencsg.com/csghub-server/common/config" "opencsg.com/csghub-server/common/types" - "opencsg.com/csghub-server/servicerunner/component" + "opencsg.com/csghub-server/runner/component" ) -type K8sHander struct { +type K8sHandler struct { clusterPool *cluster.ClusterPool k8sNameSpace string modelDockerRegBase string @@ -32,15 +32,10 @@ type K8sHander struct { s *component.ServiceComponent } -func NewK8sHander(config *config.Config) (*K8sHander, error) { - clusterPool, err := cluster.NewClusterPool() - if err != nil { - slog.Error("falied to build kubeconfig", "error", err) - return nil, fmt.Errorf("failed to build kubeconfig,%w", err) - } +func NewK8sHandler(config *config.Config, clusterPool *cluster.ClusterPool) (*K8sHandler, error) { domainParts := strings.SplitN(config.Space.InternalRootDomain, ".", 2) serviceComponent := component.NewServiceComponent(config, domainParts[0]) - return &K8sHander{ + return &K8sHandler{ k8sNameSpace: domainParts[0], clusterPool: clusterPool, env: config, @@ -49,7 +44,7 @@ func NewK8sHander(config *config.Config) (*K8sHander, error) { }, nil } -func (s *K8sHander) RunService(c *gin.Context) { +func (s *K8sHandler) RunService(c *gin.Context) { request := &types.SVCRequest{} err := c.BindJSON(&request) if err != nil { @@ -148,7 +143,7 @@ func (s *K8sHander) RunService(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"message": "Service created successfully"}) } -func (s *K8sHander) StopService(c *gin.Context) { +func (s *K8sHandler) StopService(c *gin.Context) { var resp types.StopResponse var request = &types.StopRequest{} err := c.BindJSON(request) @@ -209,7 +204,7 @@ func (s *K8sHander) StopService(c *gin.Context) { c.JSON(http.StatusOK, resp) } -func (s *K8sHander) removeServiceForcely(c *gin.Context, cluster *cluster.Cluster, svcName string) error { +func (s *K8sHandler) removeServiceForcely(c *gin.Context, cluster *cluster.Cluster, svcName string) error { err := cluster.KnativeClient.ServingV1().Services(s.k8sNameSpace).Delete(context.Background(), svcName, *metav1.NewDeleteOptions(0)) if err != nil { return err @@ -235,7 +230,7 @@ func (s *K8sHander) removeServiceForcely(c *gin.Context, cluster *cluster.Cluste return nil } -func (s *K8sHander) UpdateService(c *gin.Context) { +func (s *K8sHandler) UpdateService(c *gin.Context) { var resp types.ModelUpdateResponse var request = &types.ModelUpdateRequest{} err := c.BindJSON(request) @@ -294,7 +289,7 @@ func (s *K8sHander) UpdateService(c *gin.Context) { } // Update CPU and Memory requests and limits hardware := request.Hardware - resReq, _ := s.s.GenerateResources(hardware) + resReq, _ := component.GenerateResources(hardware) resources := corev1.ResourceRequirements{ Limits: resReq, Requests: resReq, @@ -320,7 +315,7 @@ func (s *K8sHander) UpdateService(c *gin.Context) { c.JSON(http.StatusOK, resp) } -func (s *K8sHander) ServiceStatus(c *gin.Context) { +func (s *K8sHandler) ServiceStatus(c *gin.Context) { var resp types.StatusResponse var request = &types.StatusRequest{} @@ -422,7 +417,7 @@ func (s *K8sHander) ServiceStatus(c *gin.Context) { c.JSON(http.StatusOK, resp) } -func (s *K8sHander) ServiceLogs(c *gin.Context) { +func (s *K8sHandler) ServiceLogs(c *gin.Context) { var request = &types.LogsRequest{} err := c.BindJSON(request) @@ -452,7 +447,7 @@ func (s *K8sHander) ServiceLogs(c *gin.Context) { s.GetLogsByPod(c, *cluster, podNames[0], srvName) } -func (s *K8sHander) ServiceLogsByPod(c *gin.Context) { +func (s *K8sHandler) ServiceLogsByPod(c *gin.Context) { var request = &types.ServiceRequest{} err := c.BindJSON(request) @@ -472,7 +467,7 @@ func (s *K8sHander) ServiceLogsByPod(c *gin.Context) { s.GetLogsByPod(c, *cluster, podName, srvName) } -func (s *K8sHander) GetLogsByPod(c *gin.Context, cluster cluster.Cluster, podName string, srvName string) { +func (s *K8sHandler) GetLogsByPod(c *gin.Context, cluster cluster.Cluster, podName string, srvName string) { logs := cluster.Client.CoreV1().Pods(s.k8sNameSpace).GetLogs(podName, &corev1.PodLogOptions{ Container: "user-container", @@ -543,7 +538,7 @@ func (s *K8sHander) GetLogsByPod(c *gin.Context, cluster cluster.Cluster, podNam } } -func (s *K8sHander) ServiceStatusAll(c *gin.Context) { +func (s *K8sHandler) ServiceStatusAll(c *gin.Context) { allStatus := make(map[string]*types.StatusResponse) for index := range s.clusterPool.Clusters { cluster := s.clusterPool.Clusters[index] @@ -603,7 +598,7 @@ func (s *K8sHander) ServiceStatusAll(c *gin.Context) { c.JSON(http.StatusOK, allStatus) } -func (s *K8sHander) GetServicePods(ctx context.Context, cluster cluster.Cluster, srvName string, namespace string, limit int64) ([]string, error) { +func (s *K8sHandler) GetServicePods(ctx context.Context, cluster cluster.Cluster, srvName string, namespace string, limit int64) ([]string, error) { labelSelector := fmt.Sprintf("serving.knative.dev/service=%s", srvName) // Get the list of Pods based on the label selector opts := metav1.ListOptions{ @@ -629,7 +624,7 @@ func (s *K8sHander) GetServicePods(ctx context.Context, cluster cluster.Cluster, return podNames, nil } -func (s *K8sHander) GetClusterInfo(c *gin.Context) { +func (s *K8sHandler) GetClusterInfo(c *gin.Context) { clusterRes := []types.CluserResponse{} for index := range s.clusterPool.Clusters { cls := s.clusterPool.Clusters[index] @@ -649,7 +644,7 @@ func (s *K8sHander) GetClusterInfo(c *gin.Context) { c.JSON(http.StatusOK, clusterRes) } -func (s *K8sHander) GetClusterInfoByID(c *gin.Context) { +func (s *K8sHandler) GetClusterInfoByID(c *gin.Context) { clusterId := c.Params.ByName("id") cInfo, _ := s.clusterPool.ClusterStore.ByClusterID(c.Request.Context(), clusterId) clusterInfo := types.CluserResponse{} @@ -672,15 +667,15 @@ func (s *K8sHander) GetClusterInfoByID(c *gin.Context) { c.JSON(http.StatusOK, clusterInfo) } -func (s *K8sHander) getServiceNameFromRequest(c *gin.Context) string { +func (s *K8sHandler) getServiceNameFromRequest(c *gin.Context) string { return c.Params.ByName("service") } -func (s *K8sHander) getPodNameFromRequest(c *gin.Context) string { +func (s *K8sHandler) getPodNameFromRequest(c *gin.Context) string { return c.Params.ByName("pod_name") } -func (s *K8sHander) GetServiceByName(c *gin.Context) { +func (s *K8sHandler) GetServiceByName(c *gin.Context) { var resp types.StatusResponse var request = &types.CheckRequest{} err := c.BindJSON(request) @@ -740,7 +735,7 @@ func (s *K8sHander) GetServiceByName(c *gin.Context) { c.JSON(http.StatusOK, resp) } -func (s *K8sHander) GetReplica(c *gin.Context) { +func (s *K8sHandler) GetReplica(c *gin.Context) { var resp types.ReplicaResponse var request = &types.StatusRequest{} err := c.BindJSON(request) @@ -808,7 +803,7 @@ func (s *K8sHander) GetReplica(c *gin.Context) { c.JSON(http.StatusOK, resp) } -func (s *K8sHander) UpdateCluster(c *gin.Context) { +func (s *K8sHandler) UpdateCluster(c *gin.Context) { var resp types.UpdateClusterResponse var request = &database.ClusterInfo{} err := c.BindJSON(request) @@ -832,7 +827,7 @@ func (s *K8sHander) UpdateCluster(c *gin.Context) { c.JSON(http.StatusOK, resp) } -func (s *K8sHander) PurgeService(c *gin.Context) { +func (s *K8sHandler) PurgeService(c *gin.Context) { var resp types.PurgeResponse var request = &types.PurgeRequest{} err := c.BindJSON(request) @@ -890,7 +885,7 @@ func (s *K8sHander) PurgeService(c *gin.Context) { c.JSON(http.StatusOK, resp) } -func (s *K8sHander) GetServiceInfo(c *gin.Context) { +func (s *K8sHandler) GetServiceInfo(c *gin.Context) { var resp types.ServiceInfoResponse var request = &types.ServiceRequest{} err := c.BindJSON(request) diff --git a/runner/handler/workflow.go b/runner/handler/workflow.go new file mode 100644 index 000000000..523c301d0 --- /dev/null +++ b/runner/handler/workflow.go @@ -0,0 +1,110 @@ +package handler + +import ( + "log/slog" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "opencsg.com/csghub-server/api/httpbase" + "opencsg.com/csghub-server/builder/deploy/cluster" + "opencsg.com/csghub-server/common/config" + "opencsg.com/csghub-server/common/types" + "opencsg.com/csghub-server/common/utils/common" + "opencsg.com/csghub-server/runner/component" +) + +type ArgoHandler struct { + clusterPool *cluster.ClusterPool + workflowNameSpace string + modelDockerRegBase string + config *config.Config + wfc *component.WorkFlowComponent +} + +func NewArgoHandler(config *config.Config, clusterPool *cluster.ClusterPool) (*ArgoHandler, error) { + wfc := component.NewWorkFlowComponent(config, clusterPool) + return &ArgoHandler{ + clusterPool: clusterPool, + config: config, + wfc: wfc, + modelDockerRegBase: config.Model.DockerRegBase, + workflowNameSpace: config.Argo.Namespace, + }, nil +} + +// create workflow +func (a *ArgoHandler) CreateWorkflow(ctx *gin.Context) { + var req types.ArgoWorkFlowReq + err := ctx.ShouldBindJSON(&req) + if err != nil { + slog.Error("bad order request format", "error", err) + ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + wf, err := a.wfc.CreateWorkflow(ctx, req) + if err != nil { + slog.Error("fail to create workflow", slog.Any("error", err), slog.Any("req", req)) + ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + res := types.ArgoWorkFlowRes{ + ID: wf.ID, + Username: wf.Username, + TaskName: wf.TaskName, + TaskId: wf.TaskId, + TaskType: wf.TaskType, + TaskDesc: wf.TaskDesc, + RepoIds: wf.RepoIds, + RepoType: wf.RepoType, + SubmitTime: wf.SubmitTime, + } + + ctx.JSON(http.StatusOK, res) +} + +// list workflows +func (a *ArgoHandler) ListWorkflows(ctx *gin.Context) { + username := ctx.Query("username") + per, page, err := common.GetPerAndPageFromContext(ctx) + if err != nil { + slog.Error("Bad request format of page and per", slog.Any("error", err)) + httpbase.BadRequest(ctx, err.Error()) + return + } + wfs, total, err := a.wfc.FindWorkFlows(ctx, username, per, page) + if err != nil { + slog.Error("fail to list workflows", slog.Any("error", err)) + ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + ctx.JSON(http.StatusOK, gin.H{ + "total": total, + "list": wfs, + }) +} + +func (a *ArgoHandler) DeleteWorkflow(ctx *gin.Context) { + id := ctx.Param("id") + var req = &types.ArgoWorkFlowDeleteReq{} + err := ctx.BindJSON(req) + if err != nil { + slog.Error("bad request format", "error", err) + ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + idInt64, err := strconv.ParseInt(id, 10, 64) + if err != nil { + slog.Error("fail to convert id to int64", slog.Any("error", err), slog.Any("id", id)) + ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + err = a.wfc.DeleteWorkflow(ctx, idInt64, req.Username) + if err != nil { + slog.Error("fail to delete workflow", slog.Any("error", err)) + ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + slog.Info("Deleted argo workflow successfully", slog.String("id", id)) + httpbase.OK(ctx, nil) +} diff --git a/runner/router/api.go b/runner/router/api.go new file mode 100644 index 000000000..1fa4f7f30 --- /dev/null +++ b/runner/router/api.go @@ -0,0 +1,64 @@ +package router + +import ( + "fmt" + "log/slog" + + "github.com/gin-gonic/gin" + "opencsg.com/csghub-server/api/middleware" + "opencsg.com/csghub-server/builder/deploy/cluster" + "opencsg.com/csghub-server/common/config" + "opencsg.com/csghub-server/runner/handler" +) + +func NewHttpServer(config *config.Config) (*gin.Engine, error) { + r := gin.New() + r.Use(gin.Recovery()) + r.Use(middleware.Log()) + + clusterPool, err := cluster.NewClusterPool() + if err != nil { + slog.Error("falied to build kubeconfig", "error", err) + return nil, fmt.Errorf("failed to build kubeconfig,%w", err) + } + + k8sHandler, err := handler.NewK8sHandler(config, clusterPool) + if err != nil { + return nil, fmt.Errorf("failed to build NewHttpServer,%w", err) + } + apiGroup := r.Group("/api/v1") + service := apiGroup.Group("/service") + { + service.POST("/:service/run", k8sHandler.RunService) + service.PUT("/:service/update", k8sHandler.UpdateService) + service.POST("/:service/stop", k8sHandler.StopService) + service.GET("/:service/status", k8sHandler.ServiceStatus) + service.GET("/:service/logs", k8sHandler.ServiceLogs) + service.GET("/:service/logs/:pod_name", k8sHandler.ServiceLogsByPod) + service.GET("/:service/info", k8sHandler.GetServiceInfo) + service.GET("/status-all", k8sHandler.ServiceStatusAll) + service.GET("/:service/get", k8sHandler.GetServiceByName) + service.GET("/:service/replica", k8sHandler.GetReplica) + service.DELETE("/:service/purge", k8sHandler.PurgeService) + + } + cluster := apiGroup.Group("/cluster") + { + cluster.GET("", k8sHandler.GetClusterInfo) + cluster.GET("/:id", k8sHandler.GetClusterInfoByID) + cluster.PUT("/:id", k8sHandler.UpdateCluster) + } + argoHandler, err := handler.NewArgoHandler(config, clusterPool) + if err != nil { + return nil, fmt.Errorf("failed to build NewHttpServer,%w", err) + } + + workflows := apiGroup.Group("/workflows") + { + workflows.POST("", argoHandler.CreateWorkflow) + workflows.GET("", argoHandler.ListWorkflows) + workflows.DELETE("/:id", argoHandler.DeleteWorkflow) + } + + return r, nil +} diff --git a/servicerunner/router/api.go b/servicerunner/router/api.go deleted file mode 100644 index 2c5153e0e..000000000 --- a/servicerunner/router/api.go +++ /dev/null @@ -1,45 +0,0 @@ -package router - -import ( - "fmt" - - "github.com/gin-gonic/gin" - "opencsg.com/csghub-server/api/middleware" - "opencsg.com/csghub-server/common/config" - "opencsg.com/csghub-server/servicerunner/handler" -) - -func NewHttpServer(config *config.Config) (*gin.Engine, error) { - r := gin.New() - r.Use(gin.Recovery()) - r.Use(middleware.Log()) - - handler, err := handler.NewK8sHander(config) - if err != nil { - return nil, fmt.Errorf("failed to build NewHttpServer,%w", err) - } - apiGroup := r.Group("/api/v1") - service := apiGroup.Group("/service") - { - service.POST("/:service/run", handler.RunService) - service.PUT("/:service/update", handler.UpdateService) - service.POST("/:service/stop", handler.StopService) - service.GET("/:service/status", handler.ServiceStatus) - service.GET("/:service/logs", handler.ServiceLogs) - service.GET("/:service/logs/:pod_name", handler.ServiceLogsByPod) - service.GET("/:service/info", handler.GetServiceInfo) - service.GET("/status-all", handler.ServiceStatusAll) - service.GET("/:service/get", handler.GetServiceByName) - service.GET("/:service/replica", handler.GetReplica) - service.DELETE("/:service/purge", handler.PurgeService) - - } - cluster := apiGroup.Group("/cluster") - { - cluster.GET("", handler.GetClusterInfo) - cluster.GET("/:id", handler.GetClusterInfoByID) - cluster.PUT("/:id", handler.UpdateCluster) - } - - return r, nil -} From 4fd85766f1b478e9f3f2d9787775934bebafa4ee Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 Nov 2024 02:00:33 +0000 Subject: [PATCH 02/15] Evaluation bug fix --- api/handler/evaluation.go | 36 ++++++++++++ api/router/api.go | 5 +- builder/deploy/deployer.go | 9 +++ builder/deploy/imagerunner/local_runner.go | 4 ++ builder/deploy/imagerunner/remote_runner.go | 15 +++++ builder/deploy/imagerunner/runner.go | 1 + common/types/argo_workflow.go | 31 +++++++++- component/callback/git_callback.go | 12 ++-- component/evaluation.go | 57 +++++++++++++++++++ docker/evaluation/opencompass/upload_files.py | 2 +- runner/handler/workflow.go | 43 ++++++++++++++ runner/router/api.go | 1 + 12 files changed, 207 insertions(+), 9 deletions(-) diff --git a/api/handler/evaluation.go b/api/handler/evaluation.go index ee339dc43..c02c24563 100644 --- a/api/handler/evaluation.go +++ b/api/handler/evaluation.go @@ -75,6 +75,42 @@ func (h *EvaluationHandler) RunEvaluation(ctx *gin.Context) { httpbase.OK(ctx, evaluation) } +// get evaluation godoc +// @Security ApiKey +// @Summary get model evaluation +// @Tags Evaluation +// @Accept json +// @Produce json +// @Param id path string true "id" +// @Success 200 {object} types.Evaluation "OK" +// @Failure 400 {object} types.APIBadRequest "Bad request" +// @Failure 500 {object} types.APIInternalServerError "Internal server error" +// @Router /evaluations/{id} [get] +func (h *EvaluationHandler) GetEvaluation(ctx *gin.Context) { + currentUser := httpbase.GetCurrentUser(ctx) + if currentUser == "" { + httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first")) + return + } + id, err := strconv.ParseInt(ctx.Param("id"), 10, 64) + if err != nil { + slog.Error("Bad request format", "error", err) + httpbase.BadRequest(ctx, err.Error()) + return + } + var req = &types.EvaluationGetReq{} + req.ID = id + req.Username = currentUser + evaluation, err := h.c.GetEvaluation(ctx, *req) + if err != nil { + slog.Error("Failed to get evaluation job", slog.Any("error", err)) + httpbase.ServerError(ctx, err) + return + } + httpbase.OK(ctx, evaluation) + +} + // deleteEvaluation godoc // @Security ApiKey // @Summary delete model evaluation diff --git a/api/router/api.go b/api/router/api.go index 9adeb6d78..9b01c3e8e 100644 --- a/api/router/api.go +++ b/api/router/api.go @@ -126,7 +126,7 @@ func NewRouter(config *config.Config, enableSwagger bool) (*gin.Engine, error) { return nil, fmt.Errorf("error creatring evaluation handler: %v", err) } - createEvaluationRoutes(config, apiGroup, evaluationHandler) + createEvaluationRoutes(apiGroup, evaluationHandler) // Model routes createModelRoutes(config, apiGroup, needAPIKey, modelHandler, repoCommonHandler) @@ -388,12 +388,13 @@ func NewRouter(config *config.Config, enableSwagger bool) (*gin.Engine, error) { return r, nil } -func createEvaluationRoutes(config *config.Config, apiGroup *gin.RouterGroup, evaluationHandler *handler.EvaluationHandler) { +func createEvaluationRoutes(apiGroup *gin.RouterGroup, evaluationHandler *handler.EvaluationHandler) { // Models routes evaluationsGroup := apiGroup.Group("/evaluations") { evaluationsGroup.POST("", evaluationHandler.RunEvaluation) evaluationsGroup.DELETE("/:id", evaluationHandler.DeleteEvaluation) + evaluationsGroup.GET("/:id", evaluationHandler.GetEvaluation) } } diff --git a/builder/deploy/deployer.go b/builder/deploy/deployer.go index 518ac09b6..0087c2df7 100644 --- a/builder/deploy/deployer.go +++ b/builder/deploy/deployer.go @@ -42,6 +42,7 @@ type Deployer interface { SubmitEvaluation(ctx context.Context, req types.EvaluationReq) (*types.ArgoWorkFlowRes, error) ListEvaluations(context.Context, string, int, int) (*types.ArgoWorkFlowListRes, error) DeleteEvaluation(ctx context.Context, req types.ArgoWorkFlowDeleteReq) error + GetEvaluation(ctx context.Context, req types.EvaluationGetReq) (*types.ArgoWorkFlowRes, error) } var _ Deployer = (*deployer)(nil) @@ -834,3 +835,11 @@ func (d *deployer) DeleteEvaluation(ctx context.Context, req types.ArgoWorkFlowD } return nil } + +func (d *deployer) GetEvaluation(ctx context.Context, req types.EvaluationGetReq) (*types.ArgoWorkFlowRes, error) { + wf, err := d.ir.GetWorkFlow(ctx, req) + if err != nil { + return nil, err + } + return wf, err +} diff --git a/builder/deploy/imagerunner/local_runner.go b/builder/deploy/imagerunner/local_runner.go index 6394c9b75..9c0fd904e 100644 --- a/builder/deploy/imagerunner/local_runner.go +++ b/builder/deploy/imagerunner/local_runner.go @@ -102,3 +102,7 @@ func (h *LocalRunner) ListWorkFlows(ctx context.Context, username string, per, p func (h *LocalRunner) DeleteWorkFlow(ctx context.Context, req types.ArgoWorkFlowDeleteReq) (*httpbase.R, error) { return nil, nil } + +func (h *LocalRunner) GetWorkFlow(ctx context.Context, req types.EvaluationGetReq) (*types.ArgoWorkFlowRes, error) { + return nil, nil +} diff --git a/builder/deploy/imagerunner/remote_runner.go b/builder/deploy/imagerunner/remote_runner.go index ed00e1a8d..bb2b2f926 100644 --- a/builder/deploy/imagerunner/remote_runner.go +++ b/builder/deploy/imagerunner/remote_runner.go @@ -365,3 +365,18 @@ func (h *RemoteRunner) DeleteWorkFlow(ctx context.Context, req types.ArgoWorkFlo } return &res, nil } + +func (h *RemoteRunner) GetWorkFlow(ctx context.Context, req types.ArgoWorkFlowDeleteReq) (*types.ArgoWorkFlowRes, error) { + url := fmt.Sprintf("%s/api/v1/workflows/%d", h.remote, req.ID) + response, err := h.doRequest(http.MethodGet, url, req) + if err != nil { + return nil, err + } + defer response.Body.Close() + + var res types.ArgoWorkFlowRes + if err := json.NewDecoder(response.Body).Decode(&res); err != nil { + return nil, err + } + return &res, nil +} diff --git a/builder/deploy/imagerunner/runner.go b/builder/deploy/imagerunner/runner.go index b3e702e1e..186abf20f 100644 --- a/builder/deploy/imagerunner/runner.go +++ b/builder/deploy/imagerunner/runner.go @@ -23,4 +23,5 @@ type Runner interface { SubmitWorkFlow(context.Context, *types.ArgoWorkFlowReq) (*types.ArgoWorkFlowRes, error) ListWorkFlows(context.Context, string, int, int) (*types.ArgoWorkFlowListRes, error) DeleteWorkFlow(context.Context, types.ArgoWorkFlowDeleteReq) (*httpbase.R, error) + GetWorkFlow(context.Context, types.ArgoWorkFlowDeleteReq) (*types.ArgoWorkFlowRes, error) } diff --git a/common/types/argo_workflow.go b/common/types/argo_workflow.go index 2dda6853b..23a1dcca9 100644 --- a/common/types/argo_workflow.go +++ b/common/types/argo_workflow.go @@ -118,8 +118,37 @@ type ArgoWorkFlowRes struct { FailuresURL string `json:"failures_url"` } +type RepoTags struct { + RepoId string `json:"repo_id"` + Tags []RepoTag `json:"tags"` +} + +type EvaluationRes struct { + ID int64 `json:"id"` + RepoIds []string `json:"repo_ids"` + RepoType string `json:"repo_type,omitempty"` + Username string `json:"username"` + TaskName string `json:"task_name"` + TaskId string `json:"task_id"` + TaskType TaskType `json:"task_type"` + TaskDesc string `json:"task_desc"` + Datasets []RepoTags `json:"datasets,omitempty"` + ResourceId int64 `json:"resource_id,omitempty"` + Status v1alpha1.WorkflowPhase `json:"status"` + Reason string `json:"reason,omitempty"` + Image string `bun:",notnull" json:"image"` + SubmitTime time.Time `json:"submit_time"` + StartTime time.Time `json:"start_time,omitempty"` + EndTime time.Time `json:"end_time,omitempty"` + ResultURL string `json:"result_url"` + DownloadURL string `json:"download_url"` + FailuresURL string `json:"failures_url"` +} + type ( - EvaluationDelReq = ArgoWorkFlowDeleteReq + EvaluationDelReq = ArgoWorkFlowDeleteReq + EvaluationGetReq = ArgoWorkFlowDeleteReq + ArgoWorkFlowGetReq = ArgoWorkFlowDeleteReq ) type ArgoWorkFlowDeleteReq struct { diff --git a/component/callback/git_callback.go b/component/callback/git_callback.go index 0de8bfe5a..b9ae2ab74 100644 --- a/component/callback/git_callback.go +++ b/component/callback/git_callback.go @@ -2,6 +2,7 @@ package callback import ( "context" + "database/sql" "errors" "fmt" "log/slog" @@ -399,16 +400,17 @@ func (c *GitCallbackComponent) updateDatasetTags(ctx context.Context, namespace, // check if it's evaluation dataset evalDataset, err := c.dt.FindByRepo(ctx, string(types.EvaluationCategory), repoName, string(types.DatasetRepo)) if err != nil { - if errors.Is(err, component.ErrNotFound) { - // no need add tag - return + if errors.Is(err, sql.ErrNoRows) { + slog.Debug("not an evaluation dataset, ignore it", slog.Any("repo id", repo.Path)) + } else { + slog.Error("failed to query evaluation dataset", slog.Any("repo id", repo.Path), slog.Any("error", err)) } - slog.Warn("fail to query dataset tag in git callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("error", err)) + return } tagIds := []int64{} tagIds = append(tagIds, evalDataset.Tag.ID) if evalDataset.RuntimeFramework != "" { - rTag, _ := c.ts.FindTag(ctx, evalDataset.RuntimeFramework, string(types.DatasetRepo), evalDataset.Category) + rTag, _ := c.ts.FindTag(ctx, evalDataset.RuntimeFramework, string(types.DatasetRepo), "runtime_framework") if rTag != nil { tagIds = append(tagIds, rTag.ID) } diff --git a/component/evaluation.go b/component/evaluation.go index 70f6d8b77..61fb3f012 100644 --- a/component/evaluation.go +++ b/component/evaluation.go @@ -16,6 +16,7 @@ type evaluationComponentImpl struct { deployer deploy.Deployer us database.UserStore ms database.ModelStore + ds database.DatasetStore ss database.SpaceResourceStore tokenStore database.AccessTokenStore rtfm database.RuntimeFrameworksStore @@ -26,6 +27,7 @@ type evaluationComponentImpl struct { type EvaluationComponent interface { // Create argo workflow CreateEvaluation(ctx context.Context, req *types.EvaluationReq) (*types.ArgoWorkFlowRes, error) + GetEvaluation(ctx context.Context, req types.EvaluationGetReq) (*types.EvaluationRes, error) DeleteEvaluation(ctx context.Context, req types.ArgoWorkFlowDeleteReq) error } @@ -35,6 +37,7 @@ func NewEvaluationComponent(config *config.Config) (EvaluationComponent, error) c.us = database.NewUserStore() c.ms = database.NewModelStore() c.ss = database.NewSpaceResourceStore() + c.ds = database.NewDatasetStore() c.tokenStore = database.NewAccessTokenStore() c.rtfm = database.NewRuntimeFrameworksStore() c.config = config @@ -103,3 +106,57 @@ func (c *evaluationComponentImpl) CreateEvaluation(ctx context.Context, req *typ func (c *evaluationComponentImpl) DeleteEvaluation(ctx context.Context, req types.ArgoWorkFlowDeleteReq) error { return c.deployer.DeleteEvaluation(ctx, req) } + +// get evaluation result +func (c *argoWFSComponentImpl) GetEvaluation(ctx context.Context, req types.EvaluationGetReq) (*types.EvaluationRes, error) { + wf, err := c.deployer.GetEvaluation(ctx, req) + if err != nil { + return nil, fmt.Errorf("fail to get evaluation result, %w", err) + } + datasets, err := c.ds.ListByPath(ctx, wf.Datasets) + if err != nil { + return nil, fmt.Errorf("fail to get datasets for evaluation, %w", err) + } + var repoTags []types.RepoTags + for _, ds := range datasets { + var tags []types.RepoTag + for _, tag := range ds.Repository.Tags { + tags = append(tags, types.RepoTag{ + Name: tag.Name, + Category: tag.Category, + Group: tag.Group, + BuiltIn: tag.BuiltIn, + ShowName: tag.ShowName, + CreatedAt: tag.CreatedAt, + UpdatedAt: tag.UpdatedAt, + }) + } + var dsRepoTags = types.RepoTags{ + RepoId: ds.Repository.Path, + Tags: tags, + } + repoTags = append(repoTags, dsRepoTags) + } + var res = &types.EvaluationRes{ + ID: wf.ID, + RepoIds: wf.RepoIds, + RepoType: wf.RepoType, + Username: wf.Username, + TaskName: wf.TaskName, + TaskId: wf.TaskId, + TaskType: wf.TaskType, + TaskDesc: wf.TaskDesc, + ResourceId: wf.ResourceId, + Status: wf.Status, + Reason: wf.Reason, + Datasets: repoTags, + Image: wf.Image, + SubmitTime: wf.SubmitTime, + StartTime: wf.StartTime, + EndTime: wf.EndTime, + ResultURL: wf.ResultURL, + DownloadURL: wf.DownloadURL, + FailuresURL: wf.FailuresURL, + } + return res, nil +} diff --git a/docker/evaluation/opencompass/upload_files.py b/docker/evaluation/opencompass/upload_files.py index 3e72ccd30..9c9bff302 100644 --- a/docker/evaluation/opencompass/upload_files.py +++ b/docker/evaluation/opencompass/upload_files.py @@ -133,7 +133,7 @@ def json_to_summary(jsonPath, tasks): item_new = item.copy() if item_new["dataset"] in keywords: keys = list(item_new.keys()) - model_name = item_new[keys[-1]] + model_name = keys[-1] item_new['id'] = len(summary_data) + 1 summary_data.append(item_new) column.append( diff --git a/runner/handler/workflow.go b/runner/handler/workflow.go index 523c301d0..16fd59219 100644 --- a/runner/handler/workflow.go +++ b/runner/handler/workflow.go @@ -108,3 +108,46 @@ func (a *ArgoHandler) DeleteWorkflow(ctx *gin.Context) { slog.Info("Deleted argo workflow successfully", slog.String("id", id)) httpbase.OK(ctx, nil) } + +func (a *ArgoHandler) GetWorkflow(ctx *gin.Context) { + id := ctx.Param("id") + var req = &types.ArgoWorkFlowGetReq{} + err := ctx.BindJSON(req) + if err != nil { + slog.Error("bad request format", "error", err) + ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + idInt64, err := strconv.ParseInt(id, 10, 64) + if err != nil { + slog.Error("fail to convert id to int64", slog.Any("error", err), slog.Any("id", id)) + ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + wf, err := a.wfc.GetWorkflow(ctx, idInt64, req.Username) + if err != nil { + slog.Error("fail to get workflow", slog.Any("error", err)) + ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + res := types.ArgoWorkFlowRes{ + ID: wf.ID, + Username: wf.Username, + TaskName: wf.TaskName, + Image: wf.Image, + TaskId: wf.TaskId, + TaskType: wf.TaskType, + TaskDesc: wf.TaskDesc, + RepoIds: wf.RepoIds, + RepoType: wf.RepoType, + SubmitTime: wf.SubmitTime, + StartTime: wf.StartTime, + EndTime: wf.EndTime, + Datasets: wf.Datasets, + ResultURL: wf.ResultURL, + DownloadURL: wf.DownloadURL, + FailuresURL: wf.FailuresURL, + Status: wf.Status, + } + ctx.JSON(http.StatusOK, res) +} diff --git a/runner/router/api.go b/runner/router/api.go index 1fa4f7f30..9a532d28e 100644 --- a/runner/router/api.go +++ b/runner/router/api.go @@ -58,6 +58,7 @@ func NewHttpServer(config *config.Config) (*gin.Engine, error) { workflows.POST("", argoHandler.CreateWorkflow) workflows.GET("", argoHandler.ListWorkflows) workflows.DELETE("/:id", argoHandler.DeleteWorkflow) + workflows.GET("/:id", argoHandler.GetWorkflow) } return r, nil From 8259e55af36f6dd5b529af4413b21bf2eb4008a8 Mon Sep 17 00:00:00 2001 From: James Date: Sun, 24 Nov 2024 03:56:43 +0000 Subject: [PATCH 03/15] fix bugs --- api/handler/evaluation.go | 2 +- builder/store/database/repository.go | 11 +- builder/store/database/repository_test.go | 733 +--------------------- builder/store/database/tag.go | 7 +- common/types/argo_workflow.go | 38 +- component/callback/git_callback.go | 1 - component/evaluation.go | 2 +- docs/docs.go | 143 +++++ docs/swagger.json | 143 +++++ docs/swagger.yaml | 95 +++ 10 files changed, 417 insertions(+), 758 deletions(-) diff --git a/api/handler/evaluation.go b/api/handler/evaluation.go index c02c24563..d7192be07 100644 --- a/api/handler/evaluation.go +++ b/api/handler/evaluation.go @@ -82,7 +82,7 @@ func (h *EvaluationHandler) RunEvaluation(ctx *gin.Context) { // @Accept json // @Produce json // @Param id path string true "id" -// @Success 200 {object} types.Evaluation "OK" +// @Success 200 {object} types.EvaluationRes "OK" // @Failure 400 {object} types.APIBadRequest "Bad request" // @Failure 500 {object} types.APIInternalServerError "Internal server error" // @Router /evaluations/{id} [get] diff --git a/builder/store/database/repository.go b/builder/store/database/repository.go index e929d22c4..084b6773f 100644 --- a/builder/store/database/repository.go +++ b/builder/store/database/repository.go @@ -430,11 +430,14 @@ func (s *repoStoreImpl) PublicToUser(ctx context.Context, repoType types.Reposit ) } if len(filter.Tags) > 0 { - q.Join("JOIN repository_tags ON repository.id = repository_tags.repository_id"). - Join("JOIN tags ON repository_tags.tag_id = tags.id") - for _, tag := range filter.Tags { - q.Where("tags.category = ? AND tags.name = ?", tag.Category, tag.Name) + for i, tag := range filter.Tags { + var asRepoTag = fmt.Sprintf("%s%d", "rt", i) + var asTag = fmt.Sprintf("%s%d", "ts", i) + q.Join(fmt.Sprintf("JOIN repository_tags AS %s ON repository.id = %s.repository_id", asRepoTag, asRepoTag)). + Join(fmt.Sprintf("JOIN tags AS %s ON %s.tag_id = %s.id", asTag, asRepoTag, asTag)) + q.Where(fmt.Sprintf("%s.category = ? AND %s.name = ?", asTag, asTag), tag.Category, tag.Name) } + } count, err = q.Count(ctx) diff --git a/builder/store/database/repository_test.go b/builder/store/database/repository_test.go index 45d8db5c8..1cb209edb 100644 --- a/builder/store/database/repository_test.go +++ b/builder/store/database/repository_test.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "testing" - "time" "github.com/google/uuid" "github.com/stretchr/testify/require" @@ -13,262 +12,7 @@ import ( "opencsg.com/csghub-server/common/types" ) -func TestRepoStore_CRUD(t *testing.T) { - db := tests.InitTestDB() - defer db.Close() - ctx := context.TODO() - - store := database.NewRepoStoreWithDB(db) - - _, err := store.CreateRepo(ctx, database.Repository{ - Name: "repo1", - UserID: 123, - GitPath: "foos_u/bar", - }) - require.Nil(t, err) - - rp := &database.Repository{} - err = db.Core.NewSelect().Model(rp).Where("user_id=?", 123).Scan(ctx) - require.Nil(t, err) - require.Equal(t, "repo1", rp.Name) - - rp, err = store.FindById(ctx, rp.ID) - require.Nil(t, err) - require.Equal(t, "repo1", rp.Name) - - rps, err := store.FindByIds(ctx, []int64{rp.ID}) - require.Nil(t, err) - require.Equal(t, 1, len(rps)) - require.Equal(t, "repo1", rps[0].Name) - - rps, err = store.All(ctx) - require.Nil(t, err) - require.Equal(t, 1, len(rps)) - require.Equal(t, "repo1", rps[0].Name) - - rp, err = store.Find(ctx, "u", "foO", "bAr") - require.Nil(t, err) - require.Equal(t, "repo1", rp.Name) - - exist, err := store.Exists(ctx, "foO", "u", "bar") - require.Nil(t, err) - require.True(t, exist) - - rp, err = store.FindByPath(ctx, "foO", "u", "bAr") - require.Nil(t, err) - require.Equal(t, "repo1", rp.Name) - - rp, err = store.FindByGitPath(ctx, "foos_u/bAr") - require.Nil(t, err) - require.Equal(t, "repo1", rp.Name) - - rps, err = store.FindByGitPaths(ctx, []string{"foos_u/bAr"}) - require.Nil(t, err) - require.Equal(t, 1, len(rps)) - require.Equal(t, "repo1", rps[0].Name) - - rpsp, err := store.ByUser(ctx, 123) - require.Nil(t, err) - require.Equal(t, 1, len(rpsp)) - require.Equal(t, "repo1", rpsp[0].Name) - rpsp, err = store.ByUser(ctx, 125) - require.Nil(t, err) - require.Equal(t, 0, len(rpsp)) - - rpn := *rp - rpn.Name = "repo1-new" - _, err = store.UpdateRepo(ctx, rpn) - require.Nil(t, err) - err = db.Core.NewSelect().Model(rp).Where("user_id=?", 123).Scan(ctx) - require.Nil(t, err) - require.Equal(t, "repo1-new", rp.Name) - - err = store.DeleteRepo(ctx, database.Repository{ - ID: rp.ID, - }) - require.Nil(t, err) - err = db.Core.NewSelect().Model(rp).Where("user_id=?", 123).Scan(ctx) - require.NotNil(t, err) - - _, err = store.UpdateOrCreateRepo(ctx, database.Repository{ - Name: "repo3", - UserID: 231, - Path: "bars_u/bar", - RepositoryType: types.CodeRepo, - }) - require.Nil(t, err) - rp = &database.Repository{} - err = db.Core.NewSelect().Model(rp).Where("user_id=?", 231).Scan(ctx) - require.Nil(t, err) - require.Equal(t, "repo3", rp.Name) - - _, err = store.UpdateOrCreateRepo(ctx, database.Repository{ - Name: "repo3n", - UserID: 231, - Path: "bars_u/bar", - RepositoryType: types.CodeRepo, - }) - require.Nil(t, err) - rp = &database.Repository{} - err = db.Core.NewSelect().Model(rp).Where("user_id=?", 231).Scan(ctx) - require.Nil(t, err) - require.Equal(t, "repo3n", rp.Name) - - cnt, err := store.CountByRepoType(ctx, types.CodeRepo) - require.Nil(t, err) - require.Equal(t, 1, cnt) - -} - -func TestRepoStore_UpdateRepoFileDownloads(t *testing.T) { - db := tests.InitTestDB() - defer db.Close() - ctx := context.TODO() - - store := database.NewRepoStoreWithDB(db) - - repo, err := store.CreateRepo(ctx, database.Repository{ - Name: "repo1", - UserID: 123, - GitPath: "foos_u/bar", - }) - require.Nil(t, err) - - dt := time.Date(2022, 12, 11, 0, 0, 0, 0, time.UTC) - dw := &database.RepositoryDownload{} - // create - err = store.UpdateRepoFileDownloads(ctx, repo, dt, 111) - require.Nil(t, err) - err = db.Core.NewSelect().Model(dw).Where("repository_id = ?", repo.ID).Scan(ctx) - require.Nil(t, err) - require.Equal(t, 111, int(dw.ClickDownloadCount)) - err = db.Core.NewSelect().Model(repo).Where("user_id=?", 123).Scan(ctx) - require.Nil(t, err) - require.Equal(t, 111, int(repo.DownloadCount)) - - // update - err = store.UpdateRepoFileDownloads(ctx, repo, dt, 5) - require.Nil(t, err) - err = db.Core.NewSelect().Model(dw).Where("repository_id = ?", repo.ID).Scan(ctx) - require.Nil(t, err) - require.Equal(t, 116, int(dw.ClickDownloadCount)) - err = db.Core.NewSelect().Model(repo).Where("user_id=?", 123).Scan(ctx) - require.Nil(t, err) - require.Equal(t, 116, int(repo.DownloadCount)) - -} - -func TestRepoStore_UpdateRepoCloneDownloads(t *testing.T) { - db := tests.InitTestDB() - defer db.Close() - ctx := context.TODO() - - store := database.NewRepoStoreWithDB(db) - - repo, err := store.CreateRepo(ctx, database.Repository{ - Name: "repo1", - UserID: 123, - GitPath: "foos_u/bar", - }) - require.Nil(t, err) - - dt := time.Date(2022, 12, 11, 0, 0, 0, 0, time.UTC) - dw := &database.RepositoryDownload{} - // create - err = store.UpdateRepoCloneDownloads(ctx, repo, dt, 111) - require.Nil(t, err) - err = db.Core.NewSelect().Model(dw).Where("repository_id = ?", repo.ID).Scan(ctx) - require.Nil(t, err) - require.Equal(t, 111, int(dw.CloneCount)) - err = db.Core.NewSelect().Model(repo).Where("user_id=?", 123).Scan(ctx) - require.Nil(t, err) - require.Equal(t, 111, int(repo.DownloadCount)) - - // update - err = store.UpdateRepoCloneDownloads(ctx, repo, dt, 5) - require.Nil(t, err) - err = db.Core.NewSelect().Model(dw).Where("repository_id = ?", repo.ID).Scan(ctx) - require.Nil(t, err) - // clone count will be override, not add to previous - require.Equal(t, 5, int(dw.CloneCount)) - err = db.Core.NewSelect().Model(repo).Where("user_id=?", 123).Scan(ctx) - require.Nil(t, err) - require.Equal(t, 5, int(repo.DownloadCount)) - -} - -func TestRepoStore_Tags(t *testing.T) { - db := tests.InitTestDB() - defer db.Close() - ctx := context.TODO() - - store := database.NewRepoStoreWithDB(db) - repo, err := store.CreateRepo(ctx, database.Repository{ - Name: "repo1", - UserID: 123, - }) - require.Nil(t, err) - - tag := &database.Tag{ - Name: "tg", - Category: "foo", - } - _, err = db.Core.NewInsert().Model(tag).Exec(ctx, tag) - require.Nil(t, err) - tag2 := &database.Tag{ - Name: "tg2", - Category: "bar", - } - _, err = db.Core.NewInsert().Model(tag2).Exec(ctx, tag2) - require.Nil(t, err) - - rtags := []database.RepositoryTag{ - {TagID: tag.ID, RepositoryID: repo.ID, Count: 1}, - {TagID: tag2.ID, RepositoryID: repo.ID, Count: 1}, - } - err = store.BatchCreateRepoTags(ctx, rtags) - require.Nil(t, err) - - tags, err := store.Tags(ctx, repo.ID) - require.Nil(t, err) - require.Equal(t, 2, len(tags)) - require.Equal(t, tags[0].Name, "tg") - require.Equal(t, tags[1].Name, "tg2") - - tags, err = store.TagsWithCategory(ctx, repo.ID, "foo") - require.Nil(t, err) - require.Equal(t, 1, len(tags)) - require.Equal(t, tags[0].Name, "tg") - - ids, err := store.TagIDs(ctx, repo.ID, "foo") - require.Nil(t, err) - require.Equal(t, []int64{tag.ID}, ids) -} - -func TestRepoStore_SetUpdateTimeByPath(t *testing.T) { - db := tests.InitTestDB() - defer db.Close() - ctx := context.TODO() - - store := database.NewRepoStoreWithDB(db) - repo, err := store.CreateRepo(ctx, database.Repository{ - Name: "repo1", - UserID: 123, - GitPath: "foos_u/bar", - }) - require.Nil(t, err) - - dt := time.Date(2022, 12, 6, 1, 2, 0, 0, time.UTC) - err = store.SetUpdateTimeByPath(ctx, "foo", "u", "bar", dt) - require.Nil(t, err) - - err = db.Core.NewSelect().Model(repo).WherePK().Scan(ctx) - require.Nil(t, err) - require.Equal(t, dt, repo.UpdatedAt) - -} - -func TestRepoStore_PublicToUserSimple(t *testing.T) { +func TestRepoStore_PublicToUser(t *testing.T) { db := tests.InitTestDB() defer db.Close() ctx := context.TODO() @@ -340,478 +84,5 @@ func TestRepoStore_PublicToUserSimple(t *testing.T) { // case 2: two tag repos, _, err = rs.PublicToUser(ctx, repo.RepositoryType, []int64{1}, filter, 20, 1) require.Nil(t, err) - require.Nil(t, repos) -} - -func TestRepoStore_PublicToUser(t *testing.T) { - - cases := []struct { - admin bool - repoType types.RepositoryType - source string - search string - tags []types.TagReq - sort string - expected []string - }{ - { - admin: false, repoType: types.CodeRepo, - expected: []string{"rp1", "rp2", "rp4", "rp5", "rp6"}, - }, - { - admin: false, repoType: types.CodeRepo, source: string(types.HuggingfaceSource), - expected: []string{"rp2"}, - }, - { - admin: true, repoType: types.CodeRepo, - expected: []string{"rp1", "rp2", "rp4", "rp5", "rp6"}, - }, - { - admin: false, repoType: types.CodeRepo, search: "rp4", - expected: []string{"rp4", "rp6"}, - }, - { - admin: false, repoType: types.CodeRepo, sort: "most_download", - expected: []string{"rp1", "rp2", "rp4", "rp5", "rp6"}, - }, - { - admin: false, repoType: types.CodeRepo, sort: "trending", - expected: []string{"rp1", "rp2", "rp4", "rp5", "rp6"}, - }, - { - admin: false, repoType: types.CodeRepo, tags: []types.TagReq{{Name: "foo"}}, - expected: []string{"rp4"}, - }, - } - - for _, c := range cases { - t.Run(fmt.Sprintf("%+v", c), func(t *testing.T) { - db := tests.InitTestDB() - defer db.Close() - ctx := context.TODO() - - store := database.NewRepoStoreWithDB(db) - - repos := []*database.Repository{ - { - Name: "rp1", Path: "rp1", UserID: 123, RepositoryType: types.CodeRepo, - DownloadCount: 10, - }, - { - Name: "rp2", Path: "rp2", UserID: 123, RepositoryType: types.CodeRepo, - Private: true, Source: types.HuggingfaceSource, - DownloadCount: 10, - }, - { - Name: "rp3", Path: "rp3", UserID: 456, RepositoryType: types.CodeRepo, - Private: true, - DownloadCount: 15, - }, - { - Name: "rp4", Path: "rp4", UserID: 456, RepositoryType: types.CodeRepo, - Private: false, Tags: []database.Tag{{Name: "foo"}}, - DownloadCount: 10, - }, - { - Name: "rp5", Path: "rp5", UserID: 789, RepositoryType: types.CodeRepo, - Private: false, Tags: []database.Tag{{Name: "bar"}}, - DownloadCount: 10, - }, - { - Name: "rp6", Path: "rp6", UserID: 789, RepositoryType: types.CodeRepo, - Private: false, Description: "rp4desc", - DownloadCount: 10, - }, - } - - for _, repo := range repos { - repo.GitPath = repo.Path - rn, err := store.CreateRepo(ctx, *repo) - require.Nil(t, err) - for _, tag := range repo.Tags { - _, err = db.Core.NewInsert().Model(&tag).Exec(ctx, &tag) - require.Nil(t, err) - rtags := []database.RepositoryTag{ - {TagID: tag.ID, RepositoryID: rn.ID, Count: 1}, - } - err = store.BatchCreateRepoTags(ctx, rtags) - require.Nil(t, err) - } - } - - rs, count, err := store.PublicToUser(ctx, c.repoType, []int64{123}, &types.RepoFilter{ - Tags: c.tags, - Sort: c.sort, - Search: c.search, - Source: c.source, - }, 10, 1) - require.Nil(t, err) - names := []string{} - for _, r := range rs { - names = append(names, r.Name) - } - require.Equal(t, len(c.expected), count) - require.Equal(t, c.expected, names) - - }) - } -} - -func TestRepoStore_IsMirrorRepo(t *testing.T) { - db := tests.InitTestDB() - defer db.Close() - ctx := context.TODO() - - store := database.NewRepoStoreWithDB(db) - rn, err := store.CreateRepo(ctx, database.Repository{ - GitPath: "codes_ns/n", - }) - require.Nil(t, err) - mi := &database.Mirror{RepositoryID: rn.ID} - _, err = db.Core.NewInsert().Model(mi).Exec(ctx) - require.Nil(t, err) - m, err := store.IsMirrorRepo(ctx, types.CodeRepo, "ns", "n") - require.Nil(t, err) - require.True(t, m) - m, err = store.IsMirrorRepo(ctx, types.CodeRepo, "ns", "n2") - require.NotNil(t, err) - require.False(t, m) - -} - -func TestRepoStore_WithMirror(t *testing.T) { - db := tests.InitTestDB() - defer db.Close() - ctx := context.TODO() - - store := database.NewRepoStoreWithDB(db) - rn, err := store.CreateRepo(ctx, database.Repository{ - GitPath: "codes_ns/n", - }) - require.Nil(t, err) - mi := &database.Mirror{RepositoryID: rn.ID} - _, err = db.Core.NewInsert().Model(mi).Exec(ctx) - require.Nil(t, err) - _, err = store.CreateRepo(ctx, database.Repository{ - GitPath: "codes_ns2/n", - Path: "zzz", - }) - require.Nil(t, err) - rs, count, err := store.WithMirror(ctx, 10, 1) - require.Nil(t, err) - require.Equal(t, 1, count) - require.Equal(t, "codes_ns/n", rs[0].GitPath) - -} - -func TestRepoStore_ListRepoPublicToUserByRepoIDs(t *testing.T) { - - cases := []struct { - admin bool - repoType types.RepositoryType - search string - sort string - expected []string - }{ - { - admin: false, repoType: types.CodeRepo, - expected: []string{"rp1", "rp2", "rp4", "rp5", "rp6"}, - }, - { - admin: true, repoType: types.CodeRepo, - expected: []string{"rp1", "rp2", "rp4", "rp5", "rp6"}, - }, - { - admin: false, repoType: types.CodeRepo, search: "rp4", - expected: []string{"rp4", "rp6"}, - }, - { - admin: false, repoType: types.CodeRepo, sort: "most_download", - expected: []string{"rp1", "rp2", "rp4", "rp5", "rp6"}, - }, - { - admin: false, repoType: types.CodeRepo, sort: "trending", - expected: []string{"rp1", "rp2", "rp4", "rp5", "rp6"}, - }, - } - - for _, c := range cases { - t.Run(fmt.Sprintf("%+v", c), func(t *testing.T) { - db := tests.InitTestDB() - defer db.Close() - ctx := context.TODO() - - store := database.NewRepoStoreWithDB(db) - - repos := []*database.Repository{ - { - Name: "rp1", Path: "rp1", UserID: 123, RepositoryType: types.CodeRepo, - DownloadCount: 10, - }, - { - Name: "rp2", Path: "rp2", UserID: 123, RepositoryType: types.CodeRepo, - Private: true, - DownloadCount: 10, - }, - { - Name: "rp3", Path: "rp3", UserID: 456, RepositoryType: types.CodeRepo, - Private: true, - DownloadCount: 15, - }, - { - Name: "rp4", Path: "rp4", UserID: 456, RepositoryType: types.CodeRepo, - Private: false, Tags: []database.Tag{{Name: "foo"}}, - DownloadCount: 10, - }, - { - Name: "rp5", Path: "rp5", UserID: 789, RepositoryType: types.CodeRepo, - Private: false, Tags: []database.Tag{{Name: "bar"}}, - DownloadCount: 10, - }, - { - Name: "rp6", Path: "rp6", UserID: 789, RepositoryType: types.CodeRepo, - Private: false, Description: "rp4desc", - DownloadCount: 10, - }, - } - - rids := []int64{} - for _, repo := range repos { - repo.GitPath = repo.Path - rn, err := store.CreateRepo(ctx, *repo) - require.Nil(t, err) - rids = append(rids, rn.ID) - for _, tag := range repo.Tags { - _, err = db.Core.NewInsert().Model(&tag).Exec(ctx, &tag) - require.Nil(t, err) - rtags := []database.RepositoryTag{ - {TagID: tag.ID, RepositoryID: rn.ID, Count: 1}, - } - err = store.BatchCreateRepoTags(ctx, rtags) - require.Nil(t, err) - } - } - - rs, count, err := store.ListRepoPublicToUserByRepoIDs(ctx, c.repoType, 123, c.search, c.sort, 10, 1, rids) - require.Nil(t, err) - names := []string{} - for _, r := range rs { - names = append(names, r.Name) - } - require.Equal(t, len(c.expected), count) - require.Equal(t, c.expected, names) - - }) - } -} - -func TestRepoStore_CleanRelationsByRepoID(t *testing.T) { - db := tests.InitTestDB() - defer db.Close() - ctx := context.TODO() - - store := database.NewRepoStoreWithDB(db) - rn, err := store.CreateRepo(ctx, database.Repository{ - GitPath: "codes_ns/n", - }) - require.Nil(t, err) - - _, err = db.Core.NewInsert().Model(&database.RepositoriesRuntimeFramework{ - RepoID: rn.ID, - }).Exec(ctx) - require.Nil(t, err) - _, err = db.Core.NewInsert().Model(&database.UserLike{ - RepoID: rn.ID, - }).Exec(ctx) - require.Nil(t, err) - count, err := db.Core.NewSelect().Model(&database.RepositoriesRuntimeFramework{}).Where("repo_id=?", rn.ID).Count(ctx) - require.Nil(t, err) - require.Equal(t, 1, count) - count, err = db.Core.NewSelect().Model(&database.UserLike{}).Where("repo_id=?", rn.ID).Count(ctx) - require.Nil(t, err) - require.Equal(t, 1, count) - - err = store.CleanRelationsByRepoID(ctx, rn.ID) - require.Nil(t, err) - - count, err = db.Core.NewSelect().Model(&database.RepositoriesRuntimeFramework{}).Where("repo_id=?", rn.ID).Count(ctx) - require.Nil(t, err) - require.Equal(t, 0, count) - count, err = db.Core.NewSelect().Model(&database.UserLike{}).Where("repo_id=?", rn.ID).Count(ctx) - require.Nil(t, err) - require.Equal(t, 0, count) - -} - -func TestRepoStore_DeleteAllFiles(t *testing.T) { - db := tests.InitTestDB() - defer db.Close() - ctx := context.TODO() - - store := database.NewRepoStoreWithDB(db) - - _, err := db.Core.NewInsert().Model(&database.File{RepositoryID: 123}).Exec(ctx) - require.Nil(t, err) - err = store.DeleteAllFiles(ctx, 123) - require.Nil(t, err) - count, err := db.Core.NewSelect().Model(&database.File{}).Where("repository_id = ?", 123).Count(ctx) - require.Nil(t, err) - require.Equal(t, 0, count) -} - -func TestRepoStore_DeleteAllTags(t *testing.T) { - db := tests.InitTestDB() - defer db.Close() - ctx := context.TODO() - - store := database.NewRepoStoreWithDB(db) - - _, err := db.Core.NewInsert().Model(&database.RepositoryTag{RepositoryID: 123}).Exec(ctx) - require.Nil(t, err) - err = store.DeleteAllTags(ctx, 123) - require.Nil(t, err) - count, err := db.Core.NewSelect().Model(&database.RepositoryTag{}).Where("repository_id = ?", 123).Count(ctx) - require.Nil(t, err) - require.Equal(t, 0, count) -} - -func TestRepoStore_UpdateLicenseByTag(t *testing.T) { - db := tests.InitTestDB() - defer db.Close() - ctx := context.TODO() - - store := database.NewRepoStoreWithDB(db) - rn, err := store.CreateRepo(ctx, database.Repository{ - GitPath: "codes_ns/n", - License: "foo", - }) - require.Nil(t, err) - tag := &database.Tag{ - Category: "license", - Name: "MIT", - } - _, err = db.Core.NewInsert().Model(tag).Exec(ctx, tag) - require.Nil(t, err) - rtags := []database.RepositoryTag{ - {TagID: tag.ID, RepositoryID: rn.ID, Count: 1}, - } - err = store.BatchCreateRepoTags(ctx, rtags) - require.Nil(t, err) - - err = store.UpdateLicenseByTag(ctx, rn.ID) - require.Nil(t, err) - - rn, err = store.FindById(ctx, rn.ID) - require.Nil(t, err) - require.Equal(t, "MIT", rn.License) -} - -func TestRepoStore_GetRepoRuntimeByID(t *testing.T) { - db := tests.InitTestDB() - defer db.Close() - ctx := context.TODO() - - store := database.NewRepoStoreWithDB(db) - repo, err := store.CreateRepo(ctx, database.Repository{ - Path: "codes_ns/n", - License: "foo", - RepositoryType: types.ModelRepo, - }) - require.Nil(t, err) - - rf := &database.RepositoriesRuntimeFramework{ - RepoID: repo.ID, - RuntimeFrameworkID: 999, - } - _, err = db.Core.NewInsert().Model(rf).Exec(ctx, rf) - require.Nil(t, err) - - rs, err := store.GetRepoWithRuntimeByID(ctx, rf.RuntimeFrameworkID, []string{"codes_ns/n"}) - require.Nil(t, err) - require.Equal(t, 1, len(rs)) - - rs, err = store.GetRepoWithoutRuntimeByID(ctx, rf.RuntimeFrameworkID, []string{"codes_ns/n"}) - require.Nil(t, err) - require.Equal(t, 0, len(rs)) - -} - -func TestRepoStore_BatchMethods(t *testing.T) { - db := tests.InitTestDB() - defer db.Close() - ctx := context.TODO() - - store := database.NewRepoStoreWithDB(db) - - repos := []*database.Repository{ - { - Name: "rp1", Path: "rp1", UserID: 123, RepositoryType: types.CodeRepo, - SensitiveCheckStatus: types.SensitiveCheckPending, - Source: types.HuggingfaceSource, - }, - { - Name: "rp2", Path: "rp2", UserID: 123, RepositoryType: types.CodeRepo, - SensitiveCheckStatus: types.SensitiveCheckPending, - Source: types.HuggingfaceSource, - }, - { - Name: "rp3", Path: "rp3", UserID: 123, RepositoryType: types.CodeRepo, - SensitiveCheckStatus: types.SensitiveCheckPending, - Source: types.HuggingfaceSource, - }, - { - Name: "rp4", Path: "rp4", UserID: 456, RepositoryType: types.CodeRepo, - SensitiveCheckStatus: types.SensitiveCheckPass, - Source: types.HuggingfaceSource, - }, - { - Name: "rp5", Path: "rp5", UserID: 456, RepositoryType: types.DatasetRepo, - SensitiveCheckStatus: types.SensitiveCheckPending, - }, - } - - names := func(rs []database.Repository) []string { - names := []string{} - for _, r := range rs { - names = append(names, r.Name) - } - return names - } - - rids := []int64{} - for _, repo := range repos { - repo.GitPath = repo.Path - rn, err := store.CreateRepo(ctx, *repo) - require.Nil(t, err) - rids = append(rids, rn.ID) - } - rs, err := store.BatchGet(ctx, types.CodeRepo, 0, 10) - require.Nil(t, err) - require.Equal(t, len(rs), 3) - require.Equal(t, []string{"rp1", "rp2", "rp3"}, names(rs)) - - rs, err = store.BatchGet(ctx, types.CodeRepo, rids[1], 10) - require.Nil(t, err) - require.Equal(t, len(rs), 1) - require.Equal(t, []string{"rp3"}, names(rs)) - - rs, err = store.BatchGet(ctx, types.CodeRepo, 0, 1) - require.Nil(t, err) - require.Equal(t, len(rs), 1) - require.Equal(t, []string{"rp1"}, names(rs)) - - rs, err = store.FindWithBatch(ctx, 2, 1) - require.Nil(t, err) - require.Equal(t, len(rs), 2) - require.Equal(t, []string{"rp3", "rp2"}, names(rs)) - - rs, err = store.ByUser(ctx, 123) - require.Nil(t, err) - require.Equal(t, len(rs), 3) - require.ElementsMatch(t, []string{"rp1", "rp2", "rp3"}, names(rs)) - - rs, err = store.FindByRepoSourceWithBatch(ctx, types.HuggingfaceSource, 2, 1) - require.Nil(t, err) - require.Equal(t, len(rs), 2) - require.Equal(t, []string{"rp2", "rp1"}, names(rs)) + require.NotNil(t, repos) } diff --git a/builder/store/database/tag.go b/builder/store/database/tag.go index b9c339ce9..e0ced7b65 100644 --- a/builder/store/database/tag.go +++ b/builder/store/database/tag.go @@ -220,8 +220,13 @@ func (ts *tagStoreImpl) SetMetaTags(ctx context.Context, repoType types.Reposito } var metaTagIds []int64 + exCategories := map[string]bool{ + "framework": true, + "runtime_framework": true, + "evaluation": true, + } for _, tag := range repo.Tags { - if tag.Category != "framework" { + if !exCategories[tag.Category] { metaTagIds = append(metaTagIds, tag.ID) } } diff --git a/common/types/argo_workflow.go b/common/types/argo_workflow.go index 23a1dcca9..87931438b 100644 --- a/common/types/argo_workflow.go +++ b/common/types/argo_workflow.go @@ -124,25 +124,25 @@ type RepoTags struct { } type EvaluationRes struct { - ID int64 `json:"id"` - RepoIds []string `json:"repo_ids"` - RepoType string `json:"repo_type,omitempty"` - Username string `json:"username"` - TaskName string `json:"task_name"` - TaskId string `json:"task_id"` - TaskType TaskType `json:"task_type"` - TaskDesc string `json:"task_desc"` - Datasets []RepoTags `json:"datasets,omitempty"` - ResourceId int64 `json:"resource_id,omitempty"` - Status v1alpha1.WorkflowPhase `json:"status"` - Reason string `json:"reason,omitempty"` - Image string `bun:",notnull" json:"image"` - SubmitTime time.Time `json:"submit_time"` - StartTime time.Time `json:"start_time,omitempty"` - EndTime time.Time `json:"end_time,omitempty"` - ResultURL string `json:"result_url"` - DownloadURL string `json:"download_url"` - FailuresURL string `json:"failures_url"` + ID int64 `json:"id"` + RepoIds []string `json:"repo_ids"` + RepoType string `json:"repo_type,omitempty"` + Username string `json:"username"` + TaskName string `json:"task_name"` + TaskId string `json:"task_id"` + TaskType TaskType `json:"task_type"` + TaskDesc string `json:"task_desc"` + Datasets []RepoTags `json:"datasets,omitempty"` + ResourceId int64 `json:"resource_id,omitempty"` + Status string `json:"status"` + Reason string `json:"reason,omitempty"` + Image string `bun:",notnull" json:"image"` + SubmitTime time.Time `json:"submit_time"` + StartTime time.Time `json:"start_time,omitempty"` + EndTime time.Time `json:"end_time,omitempty"` + ResultURL string `json:"result_url"` + DownloadURL string `json:"download_url"` + FailuresURL string `json:"failures_url"` } type ( diff --git a/component/callback/git_callback.go b/component/callback/git_callback.go index b9ae2ab74..64e0acd10 100644 --- a/component/callback/git_callback.go +++ b/component/callback/git_callback.go @@ -418,7 +418,6 @@ func (c *GitCallbackComponent) updateDatasetTags(ctx context.Context, namespace, err = c.ts.UpsertRepoTags(ctx, repo.ID, []int64{}, tagIds) if err != nil { - slog.Warn("fail to add dataset tag to model repo", slog.Any("repoId", repo.ID), slog.Any("tag id", evalDataset.Tag.ID), slog.Any("error", err)) slog.Warn("fail to add dataset tag", slog.Any("repoId", repo.ID), slog.Any("tag id", tagIds), slog.Any("error", err)) } diff --git a/component/evaluation.go b/component/evaluation.go index 61fb3f012..82efc946d 100644 --- a/component/evaluation.go +++ b/component/evaluation.go @@ -147,7 +147,7 @@ func (c *argoWFSComponentImpl) GetEvaluation(ctx context.Context, req types.Eval TaskType: wf.TaskType, TaskDesc: wf.TaskDesc, ResourceId: wf.ResourceId, - Status: wf.Status, + Status: string(wf.Status), Reason: wf.Reason, Datasets: repoTags, Image: wf.Image, diff --git a/docs/docs.go b/docs/docs.go index 39d1ac09c..96566d3e2 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2247,6 +2247,52 @@ const docTemplate = `{ } }, "/evaluations/{id}": { + "get": { + "security": [ + { + "ApiKey": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Evaluation" + ], + "summary": "get model evaluation", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/types.EvaluationRes" + } + }, + "400": { + "description": "Bad request", + "schema": { + "$ref": "#/definitions/types.APIBadRequest" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/types.APIInternalServerError" + } + } + } + }, "delete": { "security": [ { @@ -17119,6 +17165,74 @@ const docTemplate = `{ } } }, + "types.EvaluationRes": { + "type": "object", + "properties": { + "datasets": { + "type": "array", + "items": { + "$ref": "#/definitions/types.RepoTags" + } + }, + "download_url": { + "type": "string" + }, + "end_time": { + "type": "string" + }, + "failures_url": { + "type": "string" + }, + "id": { + "type": "integer" + }, + "image": { + "type": "string" + }, + "reason": { + "type": "string" + }, + "repo_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "repo_type": { + "type": "string" + }, + "resource_id": { + "type": "integer" + }, + "result_url": { + "type": "string" + }, + "start_time": { + "type": "string" + }, + "status": { + "type": "string" + }, + "submit_time": { + "type": "string" + }, + "task_desc": { + "type": "string" + }, + "task_id": { + "type": "string" + }, + "task_name": { + "type": "string" + }, + "task_type": { + "$ref": "#/definitions/types.TaskType" + }, + "username": { + "type": "string" + } + } + }, "types.Event": { "type": "object", "properties": { @@ -17790,6 +17904,20 @@ const docTemplate = `{ } } }, + "types.RepoTags": { + "type": "object", + "properties": { + "repo_id": { + "type": "string" + }, + "tags": { + "type": "array", + "items": { + "$ref": "#/definitions/types.RepoTag" + } + } + } + }, "types.Repository": { "type": "object", "properties": { @@ -18180,6 +18308,21 @@ const docTemplate = `{ } } }, + "types.TaskType": { + "type": "string", + "enum": [ + "evaluation", + "training", + "comparison", + "leaderboard" + ], + "x-enum-varnames": [ + "TaskTypeEvaluation", + "TaskTypeTraining", + "TaskTypeComparison", + "TaskTypeLeaderBoard" + ] + }, "types.UpdateCodeReq": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index a6ea36bb7..a588ae3f9 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2236,6 +2236,52 @@ } }, "/evaluations/{id}": { + "get": { + "security": [ + { + "ApiKey": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Evaluation" + ], + "summary": "get model evaluation", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/types.EvaluationRes" + } + }, + "400": { + "description": "Bad request", + "schema": { + "$ref": "#/definitions/types.APIBadRequest" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/types.APIInternalServerError" + } + } + } + }, "delete": { "security": [ { @@ -17108,6 +17154,74 @@ } } }, + "types.EvaluationRes": { + "type": "object", + "properties": { + "datasets": { + "type": "array", + "items": { + "$ref": "#/definitions/types.RepoTags" + } + }, + "download_url": { + "type": "string" + }, + "end_time": { + "type": "string" + }, + "failures_url": { + "type": "string" + }, + "id": { + "type": "integer" + }, + "image": { + "type": "string" + }, + "reason": { + "type": "string" + }, + "repo_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "repo_type": { + "type": "string" + }, + "resource_id": { + "type": "integer" + }, + "result_url": { + "type": "string" + }, + "start_time": { + "type": "string" + }, + "status": { + "type": "string" + }, + "submit_time": { + "type": "string" + }, + "task_desc": { + "type": "string" + }, + "task_id": { + "type": "string" + }, + "task_name": { + "type": "string" + }, + "task_type": { + "$ref": "#/definitions/types.TaskType" + }, + "username": { + "type": "string" + } + } + }, "types.Event": { "type": "object", "properties": { @@ -17779,6 +17893,20 @@ } } }, + "types.RepoTags": { + "type": "object", + "properties": { + "repo_id": { + "type": "string" + }, + "tags": { + "type": "array", + "items": { + "$ref": "#/definitions/types.RepoTag" + } + } + } + }, "types.Repository": { "type": "object", "properties": { @@ -18169,6 +18297,21 @@ } } }, + "types.TaskType": { + "type": "string", + "enum": [ + "evaluation", + "training", + "comparison", + "leaderboard" + ], + "x-enum-varnames": [ + "TaskTypeEvaluation", + "TaskTypeTraining", + "TaskTypeComparison", + "TaskTypeLeaderBoard" + ] + }, "types.UpdateCodeReq": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index f92b2a9f9..6ba7c8aea 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1399,6 +1399,51 @@ definitions: task_name: type: string type: object + types.EvaluationRes: + properties: + datasets: + items: + $ref: '#/definitions/types.RepoTags' + type: array + download_url: + type: string + end_time: + type: string + failures_url: + type: string + id: + type: integer + image: + type: string + reason: + type: string + repo_ids: + items: + type: string + type: array + repo_type: + type: string + resource_id: + type: integer + result_url: + type: string + start_time: + type: string + status: + type: string + submit_time: + type: string + task_desc: + type: string + task_id: + type: string + task_name: + type: string + task_type: + $ref: '#/definitions/types.TaskType' + username: + type: string + type: object types.Event: properties: c_id: @@ -1849,6 +1894,15 @@ definitions: updated_at: type: string type: object + types.RepoTags: + properties: + repo_id: + type: string + tags: + items: + $ref: '#/definitions/types.RepoTag' + type: array + type: object types.Repository: properties: http_clone_url: @@ -2121,6 +2175,18 @@ definitions: msg: type: string type: object + types.TaskType: + enum: + - evaluation + - training + - comparison + - leaderboard + type: string + x-enum-varnames: + - TaskTypeEvaluation + - TaskTypeTraining + - TaskTypeComparison + - TaskTypeLeaderBoard types.UpdateCodeReq: properties: description: @@ -5538,6 +5604,35 @@ paths: summary: delete model evaluation tags: - Evaluation + get: + consumes: + - application/json + parameters: + - description: id + in: path + name: id + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/types.EvaluationRes' + "400": + description: Bad request + schema: + $ref: '#/definitions/types.APIBadRequest' + "500": + description: Internal server error + schema: + $ref: '#/definitions/types.APIInternalServerError' + security: + - ApiKey: [] + summary: get model evaluation + tags: + - Evaluation /events: post: consumes: From d05383ded7ee702d753d8a79265fa429060a0aa8 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 3 Dec 2024 10:27:08 +0000 Subject: [PATCH 04/15] support master hosts support harness evaluation --- api/handler/evaluation.go | 8 +- builder/deploy/common/deploy_config.go | 3 +- builder/deploy/deployer.go | 17 +- ...0241126071538_add_lm_harness_data.down.sql | 10 + .../20241126071538_add_lm_harness_data.up.sql | 160 +++++++++++++ builder/store/database/tag_rule.go | 21 +- builder/store/database/tag_rule_test.go | 34 +++ cmd/csghub-server/cmd/start/server.go | 3 +- common/config/config.go | 9 +- common/types/mirror.go | 7 +- component/callback/git_callback.go | 39 ++- component/evaluation.go | 75 ++++-- component/evaluation_test.go | 210 +++++++++++++++++ component/mirror.go | 1 + .../Dockerfile.lm-evaluation-harness | 17 ++ docker/evaluation/Dockerfile.opencompass | 3 +- .../lm-evaluation-harness/download.py | 27 +++ .../lm-evaluation-harness/get_task.py | 112 +++++++++ .../evaluation/lm-evaluation-harness/start.sh | 123 ++++++++++ .../lm-evaluation-harness/upload_files.py | 222 ++++++++++++++++++ docker/evaluation/opencompass/start.sh | 18 +- docker/inference/Dockerfile.tgi | 10 +- runner/component/workflow.go | 3 + runner/handler/service.go | 10 +- 24 files changed, 1069 insertions(+), 73 deletions(-) create mode 100644 builder/store/database/migrations/20241126071538_add_lm_harness_data.down.sql create mode 100644 builder/store/database/migrations/20241126071538_add_lm_harness_data.up.sql create mode 100644 builder/store/database/tag_rule_test.go create mode 100644 component/evaluation_test.go create mode 100644 docker/evaluation/Dockerfile.lm-evaluation-harness create mode 100644 docker/evaluation/lm-evaluation-harness/download.py create mode 100644 docker/evaluation/lm-evaluation-harness/get_task.py create mode 100644 docker/evaluation/lm-evaluation-harness/start.sh create mode 100644 docker/evaluation/lm-evaluation-harness/upload_files.py diff --git a/api/handler/evaluation.go b/api/handler/evaluation.go index d7192be07..74522220d 100644 --- a/api/handler/evaluation.go +++ b/api/handler/evaluation.go @@ -14,7 +14,7 @@ import ( ) func NewEvaluationHandler(config *config.Config) (*EvaluationHandler, error) { - wkf, err := component.NewArgoWFSComponent(config) + wkf, err := component.NewEvaluationComponent(config) if err != nil { return nil, err } @@ -29,8 +29,8 @@ func NewEvaluationHandler(config *config.Config) (*EvaluationHandler, error) { } type EvaluationHandler struct { - c *component.ArgoWFSComponent - sc *component.SensitiveComponent + c component.EvaluationComponent + sc component.SensitiveComponent } // create evaluation godoc @@ -66,7 +66,7 @@ func (h *EvaluationHandler) RunEvaluation(ctx *gin.Context) { return } req.Username = currentUser - evaluation, err := h.c.CreateEvaluation(ctx, &req) + evaluation, err := h.c.CreateEvaluation(ctx, req) if err != nil { slog.Error("Failed to create evaluation job", slog.Any("error", err)) httpbase.ServerError(ctx, err) diff --git a/builder/deploy/common/deploy_config.go b/builder/deploy/common/deploy_config.go index 53d67417b..4ea02f815 100644 --- a/builder/deploy/common/deploy_config.go +++ b/builder/deploy/common/deploy_config.go @@ -13,5 +13,6 @@ type DeployConfig struct { PublicRootDomain string SSHDomain string //download lfs object from internal s3 address - S3Internal bool + S3Internal bool + IsMasterHost bool } diff --git a/builder/deploy/deployer.go b/builder/deploy/deployer.go index 0087c2df7..47c9d16ea 100644 --- a/builder/deploy/deployer.go +++ b/builder/deploy/deployer.go @@ -83,14 +83,15 @@ func newDeployer(s scheduler.Scheduler, ib imagebuilder.Builder, ir imagerunner. } go d.refreshStatus() - go func() { - err = d.s.Run() - if err != nil { - slog.Error("run scheduler failed", slog.Any("error", err)) - } - }() - go d.startAccounting() - + if d.c.IsMasterHost { + go func() { + err = d.s.Run() + if err != nil { + slog.Error("run scheduler failed", slog.Any("error", err)) + } + }() + go d.startAccounting() + } return d, nil } diff --git a/builder/store/database/migrations/20241126071538_add_lm_harness_data.down.sql b/builder/store/database/migrations/20241126071538_add_lm_harness_data.down.sql new file mode 100644 index 000000000..c3b2a0dd8 --- /dev/null +++ b/builder/store/database/migrations/20241126071538_add_lm_harness_data.down.sql @@ -0,0 +1,10 @@ +SET statement_timeout = 0; + +--bun:split +Delete from public.tag_rules where runtime_framework='lm-evaluation-harness'; +ALTER TABLE public.tag_rules DROP COLUMN IF EXISTS namespace; +ALTER TABLE public.tag_rules DROP COLUMN IF EXISTS source; +ALTER TABLE public.tag_rules DROP CONSTRAINT unique_tag_rules; +ALTER TABLE public.tag_rules ADD CONSTRAINT unique_tag_rules UNIQUE (repo_name, category); +Delete from public.tags where name='Benchmark'; +Delete from public.tags where name='lm-evaluation-harness'; diff --git a/builder/store/database/migrations/20241126071538_add_lm_harness_data.up.sql b/builder/store/database/migrations/20241126071538_add_lm_harness_data.up.sql new file mode 100644 index 000000000..a574bafc8 --- /dev/null +++ b/builder/store/database/migrations/20241126071538_add_lm_harness_data.up.sql @@ -0,0 +1,160 @@ +SET statement_timeout = 0; + +--bun:split +--add opencompass as namespace for all tag_rules created in 20241111095847_init_tag_rule.up.sql +--then drop the default the value +ALTER TABLE public.tag_rules ADD COLUMN IF NOT EXISTS namespace VARCHAR DEFAULT 'opencompass'; +ALTER TABLE public.tag_rules ALTER COLUMN namespace DROP DEFAULT; +ALTER TABLE public.tag_rules ADD COLUMN IF NOT EXISTS source VARCHAR DEFAULT 'ms'; +ALTER TABLE public.tag_rules ALTER COLUMN source DROP DEFAULT; +ALTER TABLE public.tag_rules DROP CONSTRAINT unique_tag_rules; +ALTER TABLE public.tag_rules ADD CONSTRAINT unique_tag_rules UNIQUE (namespace,repo_name, category); + +--bun:split +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name) VALUES('lm-evaluation-harness', 'runtime_framework', 'evaluation', 'model', true, 'lm-evaluation-harness') ON CONFLICT ("name", category, scope) DO NOTHING; +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name) VALUES('lm-evaluation-harness', 'runtime_framework', 'evaluation', 'dataset', true, 'lm-evaluation-harness') ON CONFLICT ("name", category, scope) DO NOTHING; +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name, created_at, updated_at) VALUES('Benchmark', 'evaluation', '', 'dataset', true, 'Benchmark', '2024-11-26 10:42:12.939', '2024-11-26 10:42:12.939') ON CONFLICT ("name", category, scope) DO NOTHING; + +--bun:split +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HCAI', 'metabench', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'bigbench', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('lighteval', 'MATH-Hard', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('ofir408', 'MedConceptsQA', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('pbevan11', 'EQ-Bench', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('tinyBenchmarks', 'tinyAI2_arc', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('tinyBenchmarks', 'tinyGSM8k', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('tinyBenchmarks', 'tinyHellaswag', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('tinyBenchmarks', 'tinyMMLU', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('tinyBenchmarks', 'tinyTruthfulQA', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('tinyBenchmarks', 'tinyWinogrande', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('wis-k', 'instruction-following-eval', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('SaylorTwift', 'bbh', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('TAUR-Lab', 'MuSR', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('ambean', 'lingOly', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('facebook', 'anli', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('openai', 'gsm8k', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('inverse-scaling', 'NeQA', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('inverse-scaling', 'hindsight-neglect-10shot', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('inverse-scaling', 'quote-repetition', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('inverse-scaling', 'redefine-math', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('mathemakitten', 'winobias_antistereotype_test_v5', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('qintongli', 'GSM-Plus', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('allenai', 'swag', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('cambridgeltl', 'xcopa', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('facebook', 'anli', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Rowan', 'hellaswag', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('ybisk', 'piqa', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('ErnestSDavis', 'winograd_wsc', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; + +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('AYueksel', 'TurkishMMLU', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Hennara', 'copa_ar', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Hennara', 'pica_ar', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('LumiOpen', 'arc_challenge_mt', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('TIGER-Lab', 'MMLU-Pro', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('alexandrainst', 'm_arc', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('alexandrainst', 'm_hellaswag', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('alexandrainst', 'm_mmlu', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('alexandrainst', 'm_truthfulqa', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('allenai', 'ai2_arc', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('allenai', 'paloma', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('google', 'IFEval', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-aqua-rat', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-gaokao-biology', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-gaokao-chemistry', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-gaokao-chinese', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-gaokao-english', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-gaokao-geography', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-gaokao-history', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-gaokao-mathcloze', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-gaokao-mathqa', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-gaokao-physics', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-jec-qa-ca', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-jec-qa-kd', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-logiqa-en', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-logiqa-zh', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-lsat-ar', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-lsat-lr', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-lsat-rc', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-math', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-sat-en', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-sat-en-without-passage', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-sat-math', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('mideind', 'icelandic-arc-challenge', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('allenai', 'math_qa', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('openlifescienceai', 'medmcqa', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('allenai', 'sciq', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; + +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HAERAE-HUB', 'HAE_RAE_BENCH', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HAERAE-HUB', 'KMMLU', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HAERAE-HUB', 'KMMLU-HARD', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Idavidrein', 'gpqa', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Muennighoff', 'babi', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('masakhane', 'afrimgsm', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('masakhane', 'afrimgsm-translate-test', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('masakhane', 'afrimmlu', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('masakhane', 'afrimmlu-translate-test', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('masakhane', 'afrixnli', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('masakhane', 'afrixnli-translate-test', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('google-research-datasets', 'nq_open', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('allenai', 'openbookqa', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('tau', 'commonsense_qa', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('mandarjoshi', 'trivia_qa', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('truthfulqa', 'truthful_qa', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Stanford', 'web_questions', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('community-datasets', 'qa4mre', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; + + +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('GBaker', 'MedQA-USMLE-4-options-hf', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HiTZ', 'BertaQA', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HiTZ', 'EusExams', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HiTZ', 'EusProficiency', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HiTZ', 'EusTrivia', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HiTZ', 'MGSM-eu', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HiTZ', 'XCOPA-eu', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HiTZ', 'wnli-eu', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HiTZ', 'xnli-eu', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('facebook', 'belebele', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('nyu-mll', 'glue', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hitachi-nlp', 'FLD.v2', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('maximegmd', 'glianorex', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('google-research-datasets', 'paws-x', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('facebook', 'xnli', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('google', 'xquad', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('cimec', 'lambada', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('CogComp', 'mc_taco', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('allenai', 'social_i_qa', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('LSDSem', 'story_cloze', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; + +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('CM', 'codexglue_code2text_go', 'evaluation', 'Code', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('CM', 'codexglue_code2text_java', 'evaluation', 'Code', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('CM', 'codexglue_code2text_javascript', 'evaluation', 'Code', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('CM', 'codexglue_code2text_php', 'evaluation', 'Code', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('CM', 'codexglue_code2text_python', 'evaluation', 'Code', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('CM', 'codexglue_code2text_ruby', 'evaluation', 'Code', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('allenai', 'real-toxicity-prompts', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Albertmade', 'into-the-unknown', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Albertmade', 'memo-trap', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Albertmade', 'modus-tollens', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Albertmade', 'pattern-matching-suppression', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Albertmade', 'repetitive-algebra', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Albertmade', 'sig-figs', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('BSC-LT', 'COPA-es', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('BSC-LT', 'openbookqa-es', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('EleutherAI', 'headqa', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('EleutherAI', 'winogenerated', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Geralt-Targaryen', 'MELA', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('NeelNanda', 'pile-10k', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('nyu-mll', 'blimp', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('cais', 'wmdp', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('skg', 'toxigen-data', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('wmt', 'wmt14', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('wmt', 'wmt16', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('allenai', 'winogrande', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('aps', 'super_glue', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('nyu-mll', 'blimp', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('facebook', 'xnli', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('nyu-mll', 'glue', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('IWSLT', 'iwslt2017', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; + + + diff --git a/builder/store/database/tag_rule.go b/builder/store/database/tag_rule.go index cbb49c68f..e4b905978 100644 --- a/builder/store/database/tag_rule.go +++ b/builder/store/database/tag_rule.go @@ -5,33 +5,44 @@ import ( "time" ) -type TagRuleStore struct { +type tagRuleStoreImpl struct { db *DB } -func NewTagRuleStore() *TagRuleStore { - return &TagRuleStore{db: defaultDB} +type TagRuleStore interface { + // find dataset tag by name and related tag with tag name + FindByRepo(ctx context.Context, category, namespace, repoName, repoType string) (*TagRule, error) +} + +func NewTagRuleStore() TagRuleStore { + return &tagRuleStoreImpl{db: defaultDB} +} + +func NewTagRuleStoreWithDB(db *DB) TagRuleStore { + return &tagRuleStoreImpl{db: db} } // internal use for the relation between dataset and tag type TagRule struct { ID int64 `bun:",pk,autoincrement" json:"id"` + Namespace string `bun:",notnull" json:"namespace"` RepoName string `bun:",notnull" json:"repo_name"` RepoType string `bun:",notnull" json:"repo_type"` Category string `bun:",notnull" json:"category"` TagName string `bun:",notnull" json:"tag_name"` RuntimeFramework string `bun:"," json:"runtime_framework"` + Source string `bun:"," json:"source"` Tag Tag `bun:",rel:has-one,join:tag_name=name"` CreatedAt time.Time `bun:",nullzero,notnull,skipupdate,default:current_timestamp" json:"created_at"` } // find dataset tag by name and related tag with tag name -func (s *TagRuleStore) FindByRepo(ctx context.Context, category string, repoName string, repoType string) (*TagRule, error) { +func (s *tagRuleStoreImpl) FindByRepo(ctx context.Context, category, namespace, repoName, repoType string) (*TagRule, error) { var TagRule TagRule err := s.db.Core.NewSelect(). Model(&TagRule). Relation("Tag"). - Where("LOWER(tag_rule.repo_name) = LOWER(?) AND tag_rule.repo_type = ?", repoName, repoType). + Where("LOWER(tag_rule.repo_name) = LOWER(?) AND tag_rule.namespace =? AND tag_rule.repo_type = ? AND tag_rule.category = ?", repoName, namespace, repoType, category). Limit(1). Scan(ctx) if err != nil { diff --git a/builder/store/database/tag_rule_test.go b/builder/store/database/tag_rule_test.go new file mode 100644 index 000000000..23db79829 --- /dev/null +++ b/builder/store/database/tag_rule_test.go @@ -0,0 +1,34 @@ +package database_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "opencsg.com/csghub-server/builder/store/database" + "opencsg.com/csghub-server/common/tests" + "opencsg.com/csghub-server/common/types" +) + +func TestTagRuleStore_FindByRepo(t *testing.T) { + db := tests.InitTestDB() + defer db.Close() + ctx := context.TODO() + + store := database.NewTagRuleStoreWithDB(db) + + _, err := db.Core.NewInsert().Model(&database.TagRule{ + Category: "foo", + Namespace: "test", + RepoName: "bar", + RepoType: string(types.ModelRepo), + TagName: "t1", + }).Exec(ctx) + require.Nil(t, err) + tr, err := store.FindByRepo(ctx, "foo", "test", "bar", string(types.ModelRepo)) + require.Nil(t, err) + require.Equal(t, "t1", tr.TagName) + + _, err = store.FindByRepo(ctx, "foo", "test", "foo", string(types.ModelRepo)) + require.NotNil(t, err) +} diff --git a/cmd/csghub-server/cmd/start/server.go b/cmd/csghub-server/cmd/start/server.go index a684674c5..1f3cf55ba 100644 --- a/cmd/csghub-server/cmd/start/server.go +++ b/cmd/csghub-server/cmd/start/server.go @@ -67,7 +67,6 @@ var serverCmd = &cobra.Command{ if err != nil { return fmt.Errorf("fail to initialize message queue, %w", err) } - s3Internal := len(cfg.S3.InternalEndpoint) > 0 err = deploy.Init(common.DeployConfig{ ImageBuilderURL: cfg.Space.BuilderEndpoint, ImageRunnerURL: cfg.Space.RunnerEndpoint, @@ -77,7 +76,7 @@ var serverCmd = &cobra.Command{ ModelDeployTimeoutInMin: cfg.Model.DeployTimeoutInMin, ModelDownloadEndpoint: cfg.Model.DownloadEndpoint, PublicRootDomain: cfg.Space.PublicRootDomain, - S3Internal: s3Internal, + IsMasterHost: cfg.IsMasterHost, }) if err != nil { return fmt.Errorf("failed to init deploy: %w", err) diff --git a/common/config/config.go b/common/config/config.go index f93318094..e631e2d58 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -16,8 +16,13 @@ type Config struct { EnableSwagger bool `env:"STARHUB_SERVER_ENABLE_SWAGGER, default=false"` APIToken string `env:"STARHUB_SERVER_API_TOKEN, default=0c11e6e4f2054444374ba3f0b70de4145935a7312289d404814cd5907c6aa93cc65cd35dbf94e04c13a3dedbf51f1694de84240c8acb7238b54a2c3ac8e87c59"` // enable if you want to acess csghub through https, especially for space rproxy - EnableHTTPS bool `env:"STARHUB_SERVER_ENABLE_HTTPS, default=false"` - DocsHost string `env:"STARHUB_SERVER_SERVER_DOCS_HOST, default=http://localhost:6636"` + EnableHTTPS bool `env:"STARHUB_SERVER_ENABLE_HTTPS, default=false"` + //the api key to call lbs service, like tencent map or gaode map + LBSServiceKey string `env:"STARHUB_SERVER_LBS_SERVICE_KEY, default=123456"` + //the cdn domain for different city + CityToCdnDomain map[string]string `env:"STARHUB_SERVER_CITY_TO_CDN_DOMAIN, default="` + //the master host + IsMasterHost bool `env:"STARHUB_SERVER_IS_MASTER_HOST, default=true"` APIServer struct { Port int `env:"STARHUB_SERVER_SERVER_PORT, default=8080"` diff --git a/common/types/mirror.go b/common/types/mirror.go index f9e6cecce..294bed0ca 100644 --- a/common/types/mirror.go +++ b/common/types/mirror.go @@ -99,9 +99,10 @@ const ( type Mapping string const ( - AutoMapping Mapping = "auto" - HFMapping Mapping = "hf" - CSGHubMapping Mapping = "csghub" + AutoMapping Mapping = "auto" + HFMapping Mapping = "hf" + CSGHubMapping Mapping = "csghub" + ModelScopeMapping Mapping = "ms" ) type MirrorPriority int diff --git a/component/callback/git_callback.go b/component/callback/git_callback.go index 64e0acd10..b1bddcd4c 100644 --- a/component/callback/git_callback.go +++ b/component/callback/git_callback.go @@ -7,6 +7,7 @@ import ( "fmt" "log/slog" "path" + "slices" "strconv" "strings" "time" @@ -215,7 +216,7 @@ func (c *GitCallbackComponent) modifyFiles(ctx context.Context, repoType, namesp for _, fileName := range fileNames { slog.Debug("modify file", slog.String("file", fileName)) // update model runtime - c.updateRepoRelations(ctx, repoType, namespace, repoName, ref, fileName, false) + c.updateRepoRelations(ctx, repoType, namespace, repoName, ref, fileName, false, fileNames) // only care about readme file under root directory if fileName != types.ReadmeFileName { continue @@ -237,7 +238,7 @@ func (c *GitCallbackComponent) removeFiles(ctx context.Context, repoType, namesp for _, fileName := range fileNames { slog.Debug("remove file", slog.String("file", fileName)) // update model runtime - c.updateRepoRelations(ctx, repoType, namespace, repoName, ref, fileName, true) + c.updateRepoRelations(ctx, repoType, namespace, repoName, ref, fileName, true, fileNames) // only care about readme file under root directory if fileName == types.ReadmeFileName { // use empty content to clear all the meta tags @@ -282,7 +283,7 @@ func (c *GitCallbackComponent) addFiles(ctx context.Context, repoType, namespace for _, fileName := range fileNames { slog.Debug("add file", slog.String("file", fileName)) // update model runtime - c.updateRepoRelations(ctx, repoType, namespace, repoName, ref, fileName, false) + c.updateRepoRelations(ctx, repoType, namespace, repoName, ref, fileName, false, fileNames) // only care about readme file under root directory if fileName == types.ReadmeFileName { content, err := c.getFileRaw(repoType, namespace, repoName, ref, fileName) @@ -379,33 +380,51 @@ func (c *GitCallbackComponent) getFileRaw(repoType, namespace, repoName, ref, fi } // update repo relations -func (c *GitCallbackComponent) updateRepoRelations(ctx context.Context, repoType, namespace, repoName, ref, fileName string, deleteAction bool) { +func (c *GitCallbackComponent) updateRepoRelations(ctx context.Context, repoType, namespace, repoName, ref, fileName string, deleteAction bool, fileNames []string) { slog.Debug("update model relation for git callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("repoType", repoType), slog.Any("fileName", fileName), slog.Any("branch", ref)) if repoType == fmt.Sprintf("%ss", types.ModelRepo) { c.updateModelRuntimeFrameworks(ctx, repoType, namespace, repoName, ref, fileName, deleteAction) } if repoType == fmt.Sprintf("%ss", types.DatasetRepo) { - c.updateDatasetTags(ctx, namespace, repoName) + c.updateDatasetTags(ctx, namespace, repoName, fileNames) } } // update dataset tags for evaluation -func (c *GitCallbackComponent) updateDatasetTags(ctx context.Context, namespace, repoName string) { - +func (c *GitCallbackComponent) updateDatasetTags(ctx context.Context, namespace, repoName string, fileNames []string) { + // script dataset repo was not supported so far + scriptName := fmt.Sprintf("%s.py", repoName) + if slices.Contains(fileNames, scriptName) { + return + } repo, err := c.rs.FindByPath(ctx, types.DatasetRepo, namespace, repoName) if err != nil || repo == nil { slog.Warn("fail to query repo for in callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("error", err)) return } // check if it's evaluation dataset - evalDataset, err := c.dt.FindByRepo(ctx, string(types.EvaluationCategory), repoName, string(types.DatasetRepo)) + evalDataset, err := c.dt.FindByRepo(ctx, string(types.EvaluationCategory), namespace, repoName, string(types.DatasetRepo)) if err != nil { if errors.Is(err, sql.ErrNoRows) { - slog.Debug("not an evaluation dataset, ignore it", slog.Any("repo id", repo.Path)) + // check if it's a mirror repo + mirror, err := c.mirrorStore.FindByRepoPath(ctx, types.DatasetRepo, namespace, repoName) + if err != nil || mirror == nil { + slog.Debug("fail to query mirror dataset for in callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("error", err)) + return + } + namespace := strings.Split(mirror.SourceRepoPath, "/")[0] + name := strings.Split(mirror.SourceRepoPath, "/")[1] + // use mirror namespace and name to find dataset + evalDataset, err = c.dt.FindByRepo(ctx, string(types.EvaluationCategory), namespace, name, string(types.DatasetRepo)) + if err != nil { + slog.Debug("not an evaluation dataset, ignore it", slog.Any("repo id", repo.Path)) + return + } } else { slog.Error("failed to query evaluation dataset", slog.Any("repo id", repo.Path), slog.Any("error", err)) + return } - return + } tagIds := []int64{} tagIds = append(tagIds, evalDataset.Tag.ID) diff --git a/component/evaluation.go b/component/evaluation.go index 82efc946d..683261cba 100644 --- a/component/evaluation.go +++ b/component/evaluation.go @@ -2,7 +2,9 @@ package component import ( "context" + "database/sql" "encoding/json" + "errors" "fmt" "strings" @@ -13,20 +15,21 @@ import ( ) type evaluationComponentImpl struct { - deployer deploy.Deployer - us database.UserStore - ms database.ModelStore - ds database.DatasetStore - ss database.SpaceResourceStore - tokenStore database.AccessTokenStore - rtfm database.RuntimeFrameworksStore - config *config.Config - ac AccountingComponent + deployer deploy.Deployer + userStore database.UserStore + modelStore database.ModelStore + datasetStore database.DatasetStore + mirrorStore database.MirrorStore + spaceResourceStore database.SpaceResourceStore + tokenStore database.AccessTokenStore + rtfm database.RuntimeFrameworksStore + config *config.Config + ac AccountingComponent } type EvaluationComponent interface { // Create argo workflow - CreateEvaluation(ctx context.Context, req *types.EvaluationReq) (*types.ArgoWorkFlowRes, error) + CreateEvaluation(ctx context.Context, req types.EvaluationReq) (*types.ArgoWorkFlowRes, error) GetEvaluation(ctx context.Context, req types.EvaluationGetReq) (*types.EvaluationRes, error) DeleteEvaluation(ctx context.Context, req types.ArgoWorkFlowDeleteReq) error } @@ -34,10 +37,11 @@ type EvaluationComponent interface { func NewEvaluationComponent(config *config.Config) (EvaluationComponent, error) { c := &evaluationComponentImpl{} c.deployer = deploy.NewDeployer() - c.us = database.NewUserStore() - c.ms = database.NewModelStore() - c.ss = database.NewSpaceResourceStore() - c.ds = database.NewDatasetStore() + c.userStore = database.NewUserStore() + c.modelStore = database.NewModelStore() + c.spaceResourceStore = database.NewSpaceResourceStore() + c.datasetStore = database.NewDatasetStore() + c.mirrorStore = database.NewMirrorStore() c.tokenStore = database.NewAccessTokenStore() c.rtfm = database.NewRuntimeFrameworksStore() c.config = config @@ -50,25 +54,30 @@ func NewEvaluationComponent(config *config.Config) (EvaluationComponent, error) } // Create argo workflow -func (c *evaluationComponentImpl) CreateEvaluation(ctx context.Context, req *types.EvaluationReq) (*types.ArgoWorkFlowRes, error) { - user, err := c.us.FindByUsername(ctx, req.Username) +func (c *evaluationComponentImpl) CreateEvaluation(ctx context.Context, req types.EvaluationReq) (*types.ArgoWorkFlowRes, error) { + user, err := c.userStore.FindByUsername(ctx, req.Username) if err != nil { return nil, fmt.Errorf("failed to get current user %s, error:%w", req.Username, err) } result := strings.Split(req.ModelId, "/") - _, err = c.ms.FindByPath(ctx, result[0], result[1]) + _, err = c.modelStore.FindByPath(ctx, result[0], result[1]) if err != nil { return nil, fmt.Errorf("cannot find model, %w", err) } - token, err := c.tokenStore.FindByUID(context.Background(), user.ID) + token, err := c.tokenStore.FindByUID(ctx, user.ID) if err != nil { return nil, fmt.Errorf("cant get git access token:%w", err) } + mirrorRepos, err := c.GenerateMirrorRepoIds(ctx, req.Datasets) + if err != nil { + return nil, fmt.Errorf("failed to generate mirror repo ids, %w", err) + } + req.Datasets = mirrorRepos req.Token = token.Token var hardware types.HardWare if req.ResourceId != 0 { - resource, err := c.ss.FindByID(ctx, req.ResourceId) + resource, err := c.spaceResourceStore.FindByID(ctx, req.ResourceId) if err != nil { return nil, fmt.Errorf("cannot find resource, %w", err) } @@ -85,7 +94,7 @@ func (c *evaluationComponentImpl) CreateEvaluation(ctx context.Context, req *typ // for share mode hardware.Gpu.Num = c.config.Argo.QuotaGPUNumber hardware.Gpu.ResourceName = "nvidia.com/gpu" - hardware.Cpu.Num = "12" + hardware.Cpu.Num = "8" hardware.Memory = "32Gi" } frame, err := c.rtfm.FindEnabledByID(ctx, req.RuntimeFrameworkId) @@ -100,7 +109,27 @@ func (c *evaluationComponentImpl) CreateEvaluation(ctx context.Context, req *typ req.RepoType = string(types.ModelRepo) req.TaskType = types.TaskTypeEvaluation req.DownloadEndpoint = c.config.Model.DownloadEndpoint - return c.deployer.SubmitEvaluation(ctx, *req) + return c.deployer.SubmitEvaluation(ctx, req) +} + +// generate mirror repo ids +func (c *evaluationComponentImpl) GenerateMirrorRepoIds(ctx context.Context, datasets []string) ([]string, error) { + var mirrorRepos []string + for _, ds := range datasets { + namespace := strings.Split(ds, "/")[0] + name := strings.Split(ds, "/")[1] + mirrorRepo, err := c.mirrorStore.FindByRepoPath(ctx, types.DatasetRepo, namespace, name) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + //no mirror, will use csghub repo + mirrorRepos = append(mirrorRepos, ds) + continue + } + return nil, fmt.Errorf("fail to get mirror repo, %w", err) + } + mirrorRepos = append(mirrorRepos, mirrorRepo.SourceRepoPath) + } + return mirrorRepos, nil } func (c *evaluationComponentImpl) DeleteEvaluation(ctx context.Context, req types.ArgoWorkFlowDeleteReq) error { @@ -108,12 +137,12 @@ func (c *evaluationComponentImpl) DeleteEvaluation(ctx context.Context, req type } // get evaluation result -func (c *argoWFSComponentImpl) GetEvaluation(ctx context.Context, req types.EvaluationGetReq) (*types.EvaluationRes, error) { +func (c *evaluationComponentImpl) GetEvaluation(ctx context.Context, req types.EvaluationGetReq) (*types.EvaluationRes, error) { wf, err := c.deployer.GetEvaluation(ctx, req) if err != nil { return nil, fmt.Errorf("fail to get evaluation result, %w", err) } - datasets, err := c.ds.ListByPath(ctx, wf.Datasets) + datasets, err := c.datasetStore.ListByPath(ctx, wf.Datasets) if err != nil { return nil, fmt.Errorf("fail to get datasets for evaluation, %w", err) } diff --git a/component/evaluation_test.go b/component/evaluation_test.go new file mode 100644 index 000000000..83cafb223 --- /dev/null +++ b/component/evaluation_test.go @@ -0,0 +1,210 @@ +package component + +import ( + "context" + "encoding/json" + "strconv" + "testing" + + "github.com/stretchr/testify/require" + mock_deploy "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/builder/deploy" + mock_component "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/component" + "opencsg.com/csghub-server/builder/deploy" + "opencsg.com/csghub-server/builder/store/database" + "opencsg.com/csghub-server/common/config" + "opencsg.com/csghub-server/common/tests" + "opencsg.com/csghub-server/common/types" +) + +func NewTestEvaluationComponent(deployer deploy.Deployer, stores *tests.MockStores, ac AccountingComponent) EvaluationComponent { + cfg := &config.Config{} + cfg.Argo.QuotaGPUNumber = "1" + return &evaluationComponentImpl{ + deployer: deployer, + config: cfg, + userStore: stores.User, + modelStore: stores.Model, + datasetStore: stores.Dataset, + mirrorStore: stores.Mirror, + spaceResourceStore: stores.SpaceResource, + tokenStore: stores.AccessToken, + rtfm: stores.RuntimeFramework, + ac: ac, + } +} + +func TestEvaluationComponent_CreateEvaluation(t *testing.T) { + req := types.EvaluationReq{ + TaskName: "test", + ModelId: "opencsg/wukong", + Username: "test", + ResourceId: 0, + Datasets: []string{"opencsg/hellaswag"}, + RuntimeFrameworkId: 1, + } + ctx := context.TODO() + req2 := types.EvaluationReq{ + UserUUID: "test", + TaskName: "test", + ModelId: "opencsg/wukong", + Username: "test", + ResourceId: 0, + Datasets: []string{"Rowan/hellaswag"}, + RuntimeFrameworkId: 1, + Hardware: types.HardWare{ + Gpu: types.GPU{ + Num: "1", + ResourceName: "nvidia.com/gpu", + }, + Cpu: types.CPU{ + Num: "8", + }, + Memory: "32Gi", + }, + Image: "lm-evaluation-harness:0.4.6", + RepoType: "model", + TaskType: "evaluation", + Token: "foo", + } + t.Run("create evaluation without resource id", func(t *testing.T) { + deployerMock := &mock_deploy.MockDeployer{} + stores := tests.NewMockStores(t) + ac := &mock_component.MockAccountingComponent{} + c := NewTestEvaluationComponent(deployerMock, stores, ac) + stores.UserMock().EXPECT().FindByUsername(ctx, req.Username).Return(database.User{ + RoleMask: "admin", + Username: req.Username, + UUID: req.Username, + ID: 1, + }, nil).Once() + stores.ModelMock().EXPECT().FindByPath(ctx, "opencsg", "wukong").Return( + &database.Model{ + ID: 1, + }, nil, + ).Maybe() + stores.MirrorMock().EXPECT().FindByRepoPath(ctx, types.DatasetRepo, "opencsg", "hellaswag").Return(&database.Mirror{ + SourceRepoPath: "Rowan/hellaswag", + }, nil) + stores.AccessTokenMock().EXPECT().FindByUID(ctx, int64(1)).Return(&database.AccessToken{Token: "foo"}, nil) + stores.RuntimeFrameworkMock().EXPECT().FindEnabledByID(ctx, int64(1)).Return(&database.RuntimeFramework{ + ID: 1, + FrameImage: "lm-evaluation-harness:0.4.6", + }, nil) + deployerMock.EXPECT().SubmitEvaluation(ctx, req2).Return(&types.ArgoWorkFlowRes{ + ID: 1, + TaskName: "test", + }, nil) + e, err := c.CreateEvaluation(ctx, req) + require.NotNil(t, e) + require.Equal(t, "test", e.TaskName) + require.Nil(t, err) + }) + t.Run("create evaluation with resource id", func(t *testing.T) { + req.ResourceId = 1 + req2.ResourceId = 1 + deployerMock := &mock_deploy.MockDeployer{} + stores := tests.NewMockStores(t) + ac := &mock_component.MockAccountingComponent{} + c := NewTestEvaluationComponent(deployerMock, stores, ac) + stores.UserMock().EXPECT().FindByUsername(ctx, req.Username).Return(database.User{ + RoleMask: "admin", + Username: req.Username, + UUID: req.Username, + ID: 1, + }, nil).Once() + stores.ModelMock().EXPECT().FindByPath(ctx, "opencsg", "wukong").Return( + &database.Model{ + ID: 1, + }, nil, + ).Maybe() + stores.MirrorMock().EXPECT().FindByRepoPath(ctx, types.DatasetRepo, "opencsg", "hellaswag").Return(&database.Mirror{ + SourceRepoPath: "Rowan/hellaswag", + }, nil) + stores.AccessTokenMock().EXPECT().FindByUID(ctx, int64(1)).Return(&database.AccessToken{Token: "foo"}, nil) + stores.RuntimeFrameworkMock().EXPECT().FindEnabledByID(ctx, int64(1)).Return(&database.RuntimeFramework{ + ID: 1, + FrameImage: "lm-evaluation-harness:0.4.6", + }, nil) + ac.EXPECT().QueryBalanceByUserIDInternal(ctx, "test").Return(&database.AccountUser{ + Balance: 123.4, + }, nil) + resource, err := json.Marshal(req2.Hardware) + require.Nil(t, err) + stores.SpaceResourceMock().EXPECT().FindByID(ctx, int64(1)).Return(&database.SpaceResource{ + ID: 1, + Resources: string(resource), + }, nil) + ac.EXPECT().QueryPricesBySKUType("", types.AcctPriceListReq{ + SkuType: types.SKUCSGHub, + SkuKind: strconv.Itoa(int(types.SKUPayAsYouGo)), + ResourceID: strconv.FormatInt(int64(1), 10), + }).Return(&database.PriceResp{ + Total: 100, + Prices: []database.AccountPrice{{SkuPrice: 10}}, + }, nil) + deployerMock.EXPECT().SubmitEvaluation(ctx, req2).Return(&types.ArgoWorkFlowRes{ + ID: 1, + TaskName: "test", + }, nil) + e, err := c.CreateEvaluation(ctx, req) + require.NotNil(t, e) + require.Equal(t, "test", e.TaskName) + require.Nil(t, err) + }) +} + +func TestEvaluationComponent_GetEvaluation(t *testing.T) { + deployerMock := &mock_deploy.MockDeployer{} + stores := tests.NewMockStores(t) + ac := &mock_component.MockAccountingComponent{} + c := NewTestEvaluationComponent(deployerMock, stores, ac) + req := types.EvaluationGetReq{ + Username: "test", + } + ctx := context.TODO() + deployerMock.EXPECT().GetEvaluation(ctx, req).Return(&types.ArgoWorkFlowRes{ + ID: 1, + RepoIds: []string{"Rowan/hellaswag"}, + Datasets: []string{"Rowan/hellaswag"}, + RepoType: "model", + Username: "test", + TaskName: "test", + TaskId: "test", + TaskType: "evaluation", + Status: "Succeed", + }, nil) + stores.DatasetMock().EXPECT().ListByPath(ctx, []string{"Rowan/hellaswag"}).Return([]database.Dataset{ + { + Repository: &database.Repository{ + Path: "Rowan/hellaswag", + Tags: []database.Tag{ + { + Name: "test", + Category: "test", + Group: "test", + Scope: "test", + BuiltIn: true, + }, + }, + }, + }, + }, nil) + e, err := c.GetEvaluation(ctx, req) + require.NotNil(t, e) + require.Equal(t, "test", e.TaskName) + require.Nil(t, err) +} + +func TestEvaluationComponent_DeleteEvaluation(t *testing.T) { + deployerMock := &mock_deploy.MockDeployer{} + stores := tests.NewMockStores(t) + ac := &mock_component.MockAccountingComponent{} + c := NewTestEvaluationComponent(deployerMock, stores, ac) + req := types.EvaluationDelReq{ + Username: "test", + } + ctx := context.TODO() + deployerMock.EXPECT().DeleteEvaluation(ctx, req).Return(nil) + err := c.DeleteEvaluation(ctx, req) + require.Nil(t, err) +} diff --git a/component/mirror.go b/component/mirror.go index 04f1cec6b..050ae92e4 100644 --- a/component/mirror.go +++ b/component/mirror.go @@ -332,6 +332,7 @@ var mirrorOrganizationMap = map[string]string{ "openbmb": "OpenBMB", "netease-youdao": "Netease-youdao", "ByteDance": "ByteDance", + "opencompass": "opencompass", } var mirrorStatusAndRepoSyncStatusMapping = map[types.MirrorTaskStatus]types.RepositorySyncStatus{ diff --git a/docker/evaluation/Dockerfile.lm-evaluation-harness b/docker/evaluation/Dockerfile.lm-evaluation-harness new file mode 100644 index 000000000..fe32dc6a8 --- /dev/null +++ b/docker/evaluation/Dockerfile.lm-evaluation-harness @@ -0,0 +1,17 @@ +FROM nvidia/cuda:12.4.1-devel-ubuntu22.04 +RUN apt-get update && apt-get -y install git python3.10 python3-pip dumb-init \ + && apt-get clean && rm -rf /var/lib/apt/lists/* +RUN pip install --upgrade pip && pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple \ + && pip install --no-cache-dir accelerate transformers==4.46.3 \ + minio oss2 langdetect huggingface_hub openpyxl +WORKDIR /workspace/ +RUN git clone --depth 1 https://gitee.com/xzgan/lm-evaluation-harness.git --branch v0.4.6 --single-branch && \ + cd lm-evaluation-harness && pip install setuptools --upgrade --no-cache-dir -e \ + ".[ifeval,math,multilingual,sentencepiece]" +COPY ./lm-evaluation-harness/ /etc/csghub/ +RUN ln -s /usr/bin/python3 /usr/bin/python &&\ + chmod +x /etc/csghub/*.sh +ENV HUGGINGFACE_HUB_CACHE=/workspace/ \ + HF_HUB_ENABLE_HF_TRANSFER=0 +ENTRYPOINT [ "/usr/bin/dumb-init", "--" ] +CMD ["/etc/csghub/start.sh"] diff --git a/docker/evaluation/Dockerfile.opencompass b/docker/evaluation/Dockerfile.opencompass index 8efdb702d..e4a595511 100644 --- a/docker/evaluation/Dockerfile.opencompass +++ b/docker/evaluation/Dockerfile.opencompass @@ -6,8 +6,7 @@ RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple \ csghub-sdk==0.4.4 minio oss2 RUN pip install --no-cache-dir vllm==0.6.3.post1 openpyxl COPY ./opencompass/ /etc/csghub/ -RUN mkdir -p /var/log/supervisord && \ - ln -s /usr/bin/python3 /usr/bin/python &&\ +RUN ln -s /usr/bin/python3 /usr/bin/python &&\ chmod +x /etc/csghub/*.sh WORKDIR /workspace/ ENV HUGGINGFACE_HUB_CACHE=/workspace/ \ diff --git a/docker/evaluation/lm-evaluation-harness/download.py b/docker/evaluation/lm-evaluation-harness/download.py new file mode 100644 index 000000000..46501c03f --- /dev/null +++ b/docker/evaluation/lm-evaluation-harness/download.py @@ -0,0 +1,27 @@ +import os +import argparse +import datasets +# Load model directly +from huggingface_hub import snapshot_download + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Download repo.') + subparsers = parser.add_subparsers(dest='command', required=True) + + parser_a = subparsers.add_parser('datasets', help='download datasets') + parser_a.add_argument('--dataset_ids', type=str, help='repo id') + parser_b = subparsers.add_parser('models', help='download model') + parser_b.add_argument('--model_ids', type=str, help='repo id') + + args = parser.parse_args() + + # split repo ids + + if args.command == 'models': + repo_ids = args.model_ids.split(',') + for repo_id in repo_ids: + snapshot_download(repo_id=repo_id) + elif args.command == 'datasets': + repo_ids = args.dataset_ids.split(',') + for repo_id in repo_ids: + snapshot_download(repo_id=repo_id, repo_type="dataset") diff --git a/docker/evaluation/lm-evaluation-harness/get_task.py b/docker/evaluation/lm-evaluation-harness/get_task.py new file mode 100644 index 000000000..ebca88650 --- /dev/null +++ b/docker/evaluation/lm-evaluation-harness/get_task.py @@ -0,0 +1,112 @@ +import collections +import argparse +import os + +from lm_eval import utils + + +def _config_is_task(config) -> bool: + if ("task" in config) and isinstance(config["task"], str): + return True + return False + + +def _config_is_group(config) -> bool: + if ("task" in config) and isinstance(config["task"], list): + return True + return False + + +def _config_is_python_task(config) -> bool: + if "class" in config: + return True + return False + + +def get_task_and_group(task_dir: str): + def _populate_tags_and_groups(config, task, tasks_and_groups, print_info): + # TODO: remove group in next release + if "tag" in config: + attr_list = config["tag"] + if isinstance(attr_list, str): + attr_list = [attr_list] + + for tag in attr_list: + if tag not in tasks_and_groups: + tasks_and_groups[tag] = { + "type": "tag", + "task": [task], + "yaml_path": -1, + } + elif tasks_and_groups[tag]["type"] != "tag": + break + else: + tasks_and_groups[tag]["task"].append(task) + + # TODO: remove group in next release + print_info = True + ignore_dirs = [ + "__pycache__", + ".ipynb_checkpoints", + ] + tasks_and_groups = collections.defaultdict() + for root, dirs, file_list in os.walk(task_dir): + dirs[:] = [d for d in dirs if d not in ignore_dirs] + for f in file_list: + if f.endswith(".yaml"): + yaml_path = os.path.join(root, f) + config = utils.load_yaml_config(yaml_path, mode="simple") + if _config_is_python_task(config): + # This is a python class config + task = config["task"] + tasks_and_groups[task] = { + "type": "python_task", + "yaml_path": yaml_path, + } + _populate_tags_and_groups( + config, task, tasks_and_groups, print_info + ) + elif _config_is_group(config): + tasks_and_groups[config["group"]] = { + "type": "group", + "task": -1, + "yaml_path": yaml_path, + } + + elif _config_is_task(config): + # This is a task config + task = config["task"] + tasks_and_groups[task] = { + "type": "task", + "yaml_path": yaml_path, + } + _populate_tags_and_groups( + config, task, tasks_and_groups, print_info + ) + + return tasks_and_groups + + +def get_related_task(task_dir): + tasks_and_groups = get_task_and_group(task_dir) + # loop group + all_groups = [] + all_tasks = [] + for task in tasks_and_groups: + if tasks_and_groups[task]["type"] == "group": + all_groups.append(task) + if tasks_and_groups[task]["type"] == "task": + all_tasks.append(task) + if len(all_groups) > 0: + sorted_tasks = sorted(all_groups, key=len) + return sorted_tasks[0] + else: + return ", ".join(all_tasks) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Get dataset task by dir.') + parser.add_argument('task_dir', type=str, help='The task dir') + args = parser.parse_args() + + print(get_related_task(args.task_dir)) diff --git a/docker/evaluation/lm-evaluation-harness/start.sh b/docker/evaluation/lm-evaluation-harness/start.sh new file mode 100644 index 000000000..7d687d160 --- /dev/null +++ b/docker/evaluation/lm-evaluation-harness/start.sh @@ -0,0 +1,123 @@ +#!/bin/bash + +search_path_with_most_term() { + search_term=$1 + paths=("${@:2}") + + declare -A count_dict + for path in "${paths[@]}"; do + count_dict["$path"]=$(grep -o "$search_term" "$path" | wc -l) + done + + max_count_path="" + max_count=0 + for path in "${!count_dict[@]}"; do + count=${count_dict[$path]} + if [[ $count -gt $max_count ]]; then + max_count=$count + max_count_path=$path + fi + done + if [ -z "$max_count_path" ]; then + echo $paths[0] + return 0 + fi + echo $max_count_path + return 0 +} +#download datasets +if [ ! -z "$DATASET_IDS" ]; then + echo "Downloading datasets..." + python /etc/csghub/download.py datasets --dataset_ids $DATASET_IDS +fi +if [ $? -ne 0 ]; then + echo "Failed to download datasets" + exit 1 +fi +#download models +if [ ! -z "$MODEL_ID" ]; then + echo "Downloading models..." + python /etc/csghub/download.py models --model_ids $MODEL_ID +fi +if [ $? -ne 0 ]; then + echo "Failed to download models" + exit 1 +fi + +export HF_ENDPOINT="$HF_ENDPOINT/hf" + +tasks="" +task_dir="/workspace/lm-evaluation-harness/lm_eval/tasks" +IFS=',' read -r -a dataset_repos <<< "$DATASET_IDS" +if [ -z "$NUM_FEW_SHOT" ]; then + NUM_FEW_SHOT=0 +fi +script_dts_array=("allenai/winogrande" "facebook/anli" "aps/super_glue" "Rowan/hellaswag" "nyu-mll/blimp" "EdinburghNLP/orange_sum" "facebook/xnli" "nyu-mll/glue" "openai/gsm8k" "cimec/lambada" "allenai/math_qa" "openlifescienceai/medmcqa" "google-research-datasets/nq_open" "allenai/openbookqa" "google-research-datasets/paws-x" "ybisk/piqa" "community-datasets/qa4mre" "allenai/sciq" "allenai/social_i_qa" "LSDSem/story_cloze" "allenai/swag" "IWSLT/iwslt2017" "wmt/wmt14" "wmt/wmt16","mandarjoshi/trivia_qa" "truthfulqa/truthful_qa" "Stanford/web_questions" "ErnestSDavis/winograd_wsc" "cambridgeltl/xcopa" "google/xquad") +for repo in "${dataset_repos[@]}"; do + repo_name="${repo#*/}" + if [[ " ${script_dts_array[@]} " =~ " ${repo} " ]]; then + #need replace with real path + echo "replace script repo with namespace repo" + find . -type f -exec sed -i "s|dataset_path: $repo_name|dataset_path: $repo|g" {} + + fi + # search full id to cover mirror repo id + mapfile -t yaml_files < <(grep -Rl -E "(dataset_path: ${repo}($|\s))" $task_dir) + file_count=${#yaml_files[@]} + if [ "$file_count" -eq 0 ]; then + # search short id to cover csghub repo id + mapfile -t yaml_files < <(grep -Rl -E "(dataset_path: .*/${repo_name}($|\s))|(dataset_path: ${repo_name}($|\s))" $task_dir) + fi + file_count=${#yaml_files[@]} + if [ "$file_count" -eq 0 ]; then + echo "no yaml file found for repo $repo" + continue + fi + # check yaml_files size + common_path="${yaml_files[0]}" + if [ "$file_count" -gt 1 ]; then + for path in "${yaml_files[@]}"; do + while [[ "$path" != "${common_path}"* ]]; do + common_path="${common_path%/*}" + done + done + if [ "x$common_path" == "x$task_dir" ]; then + echo "no common path found for repo $repo, will pick one of the yaml_files" + matched_path=$(search_path_with_most_term "$repo_name" "${yaml_files[@]}") + common_path=$(dirname "$matched_path") + fi + else + common_path=$(dirname "$common_path") + fi + echo "common path found for repo $repo: $common_path" + repo_task=`python /etc/csghub/get_task.py $common_path` + if [ ! -z "$repo_task" ]; then + tasks="$tasks,$repo_task" + fi +done +tasks=$(echo "$tasks" | sed 's/^,//; s/,$//') +tasks=$(echo "$tasks" | tr -d ' ' | tr ',' ',') +echo "will start tasks: $tasks" + + +accelerate launch -m lm_eval \ + --model hf \ + --model_args pretrained=${MODEL_ID},dtype=auto,trust_remote_code=True \ + --tasks "$tasks" \ + --batch_size auto \ + --output_path /workspace/output/ + +if [ $? -eq 0 ]; then + echo "Evaluation completed successfully." +else + echo "Evaluation failed." + exit 1 +fi + +# upload result to mino server +json_file=`ls -dt /workspace/output/*/*.json |head -n 1` +model_name=`basename $MODEL_ID` +python /etc/csghub/upload_files.py summary --file $json_file --model $model_name +upload_json_file=`ls -d /workspace/output/*${model_name}*/*upload.json` +upload_xlsx_file=`ls -d /workspace/output/*${model_name}*/*upload.xlsx` +python /etc/csghub/upload_files.py upload "$upload_json_file,$upload_xlsx_file" +echo "finish evaluation for $MODEL_ID" \ No newline at end of file diff --git a/docker/evaluation/lm-evaluation-harness/upload_files.py b/docker/evaluation/lm-evaluation-harness/upload_files.py new file mode 100644 index 000000000..668a1cd5a --- /dev/null +++ b/docker/evaluation/lm-evaluation-harness/upload_files.py @@ -0,0 +1,222 @@ +import os +import argparse +from datetime import datetime +from minio import Minio +from minio.error import S3Error +from pathlib import Path +import pandas as pd +import json +import oss2 +import csv + +access_key_id = os.environ['S3_ACCESS_ID'] +access_key_secret = os.environ['S3_ACCESS_SECRET'] +bucket_name = os.environ['S3_BUCKET'] +endpoint = os.environ['S3_ENDPOINT'] +s3_ssl_enabled = json.loads(os.environ['S3_SSL_ENABLED']) +if endpoint.find("aliyuncs.com") == -1: + client = Minio(endpoint, access_key=access_key_id, secret_key=access_key_secret, secure=s3_ssl_enabled) +else: + auth = oss2.Auth(access_key_id, access_key_secret) + bucket = oss2.Bucket(auth, endpoint, bucket_name) + + +def generate_file_name(name): + # Get the current date and time + now = datetime.now() + + # Format the string as YYYYMMDD_HHMMSS + formatted_uuid = now.strftime("%Y%m%d_%H%M%S") + + return f"{name}_{formatted_uuid}" + + +def upload_to_minio(object_name, location_file): + # Make the bucket if it doesn't exist. + found = client.bucket_exists(bucket_name) + if not found: + client.make_bucket(bucket_name) + print("Created bucket", bucket_name) + else: + print("Bucket", bucket_name, "already exists") + + # Upload the file, renaming it in the process + client.fput_object( + bucket_name, object_name, location_file, + ) + + +def upload_to_ali(object_name, location_file): + # Upload the file, renaming it in the process + bucket.put_object_from_file(object_name, location_file) + + +def upload(files): + output = [] + # get schema based on s3_ssl_enabled + schema = "https" if s3_ssl_enabled else "http" + fileName = generate_file_name("result") + for file in files.split(','): + suffix = Path(file).suffix + object_name = f"evaluation/{fileName}{suffix}" + # check if the endpoint is aliyun oss + if endpoint.find("aliyuncs.com") != -1: + upload_to_ali(object_name, file) + file_url = f"https://{bucket_name}.{endpoint}/{object_name}" + else: + upload_to_minio(object_name, file) + file_url = f"{schema}://{endpoint}/{bucket_name}/{object_name}" + output.append(file_url) + try: + with open('/tmp/output.txt', 'w') as file: + file.write(",".join(output)) + print("Output written to /tmp/output.txt") + print(f'Successfully uploaded to {file_url}') + except Exception as e: + print(f"Error writing to file: {e}") + + +column = [ + { + "title": { + "zh-CN": "数据集", + "en-US": "Dataset" + }, + "width": 220, + "key": "dataset", + "fixed": "left" + }, + { + "title": { + "zh-CN": "指标", + "en-US": "Metric" + }, + "width": 130, + "key": "metric", + "fixed": "left" + }, + { + "title": { + "zh-CN": "模式", + "en-US": "Mode" + }, + "width": 100, + "key": "mode", + "fixed": "left" + } +] + + +def get_metric_value(resultObj, key, metric): + for item, value in resultObj[key].items(): + if metric in item: + return round(value*100, 2) + + +def get_metric_items(jsonObj, item, model_name): + metric_items = [] + metrics = jsonObj['higher_is_better'][item].keys() + for metric in metrics: + value = get_metric_value(jsonObj['results'], item, metric) + item_new = { + "dataset": item, + "version": jsonObj['versions'][item], + "metric": metric, + model_name: value, + } + metric_items.append(item_new) + return metric_items + + +def is_a_sub_group(jsonObj, key): + for k, v in jsonObj["group_subtasks"].items(): + if key in v: + return True + return False + + +def json_to_summary(jsonPath, model_name): + with open(jsonPath, 'r', encoding='utf-8') as f: + jsonObj = json.load(f) + summary_data = [] + xlsx_json = {} + # generate summary data + for k in jsonObj["group_subtasks"].keys(): + m_items = get_metric_items(jsonObj, k, model_name) + summary_data.extend(m_items) + column.append( + { + "title": { + "zh-CN": model_name, + "en-US": model_name + }, + "width": 200, + "key": model_name, + "customizeRender": { + "sorter": "descend" + } + } + ) + summary = { + "column": column, + "data": summary_data + } + final_json = {"summary": summary} + xlsx_json['summary'] = summary_data + # generate detail data + for key, value in jsonObj["group_subtasks"].items(): + sub_data = [] + if len(value) == 0: + m_items = get_metric_items(jsonObj, key, model_name) + sub_data.extend(m_items) + xlsx_json[key] = sub_data + final_json[key] = {"column": column, "data": sub_data} + else: + sub_g = is_a_sub_group(jsonObj, key) + if not sub_g: + # loop root group + for sub_key in value: + m_items = get_metric_items(jsonObj, sub_key, model_name) + sub_data.extend(m_items) + if sub_key not in jsonObj["group_subtasks"]: + continue + # level 2 group + value2 = jsonObj["group_subtasks"][sub_key] + if len(value2) == 0: + m_items = get_metric_items(jsonObj, key, model_name) + sub_data.extend(m_items) + else: + for sub_sub_key in value2: + m_items = get_metric_items(jsonObj, sub_sub_key, model_name) + sub_data.extend(m_items) + xlsx_json[key] = sub_data + final_json[key] = {"column": column, "data": sub_data} + + json_file_path = os.path.splitext(jsonPath)[0] + '_upload.json' + with open(json_file_path, 'w', encoding='utf-8') as f: + json.dump(final_json, f, ensure_ascii=False, indent=4) + + xlsx_file = os.path.splitext(jsonPath)[0] + '_upload.xlsx' + with pd.ExcelWriter(xlsx_file) as writer: + for sheet_name, records in xlsx_json.items(): + df = pd.DataFrame(records) + df.to_excel(writer, sheet_name=sheet_name, index=False) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Get upload files.') + subparsers = parser.add_subparsers(dest='command', required=True) + + parser_a = subparsers.add_parser('upload', help='upload files') + parser_a.add_argument('files', type=str, help='Name to greet') + + parser_c = subparsers.add_parser('summary', help='Convert json to json summary') + parser_c.add_argument('--file', type=str, help='Convert json to json summary') + parser_c.add_argument('--model', type=str, help='model name') + + args = parser.parse_args() + + if args.command == 'upload': + upload(args.files) + elif args.command == 'summary': + json_to_summary(args.file, args.model) diff --git a/docker/evaluation/opencompass/start.sh b/docker/evaluation/opencompass/start.sh index 08f1d0b74..8ee5ef041 100644 --- a/docker/evaluation/opencompass/start.sh +++ b/docker/evaluation/opencompass/start.sh @@ -8,6 +8,12 @@ insert_string=$(cat << 'EOF' EOF ) repo_tokenizer_config="/workspace/$MODEL_ID/tokenizer_config.json" +# check if repo_tokenizer_config exists +if [ ! -f "$repo_tokenizer_config" ]; then + echo "the model is invalid." + exit 1 +fi + # fix some model does not contain chat_template if ! grep -q "chat_template" "$repo_tokenizer_config"; then filename="/tmp/tokenizer_config.json" @@ -60,8 +66,18 @@ if [ -z "$GPU_NUM" ]; then fi #LimitedMaxToken is gpu_num multiplied by 4096 LimitedMaxToken=$(($GPU_NUM * 4096)) +# avoid GPU OOM +repo_config="/workspace/$MODEL_ID/config.json" +sed -i "s/\"max_position_embeddings\": [0-9]*/\"max_position_embeddings\": $LimitedMaxToken/" $repo_config -opencompass --datasets $dataset_tasks --work-dir /workspace/output --hf-type chat --hf-path /workspace/$MODEL_ID -a vllm --max-out-len 100 --max-seq-len $LimitedMaxToken --batch-size 8 --hf-num-gpus $GPU_NUM --max-num-workers $GPU_NUM +opencompass --datasets $dataset_tasks --work-dir /workspace/output --hf-type chat --hf-path /workspace/$MODEL_ID -a vllm --max-out-len 100 --max-seq-len $LimitedMaxToken --batch-size 8 --hf-num-gpus $GPU_NUM --max-num-workers $GPU_NUM + +if [ $? -eq 0 ]; then + echo "Evaluation completed successfully." +else + echo "Evaluation failed." + exit 1 +fi # upload result to mino server output_dir=`ls -dt /workspace/output/* |head -n 1` diff --git a/docker/inference/Dockerfile.tgi b/docker/inference/Dockerfile.tgi index fe99daf21..562a47139 100644 --- a/docker/inference/Dockerfile.tgi +++ b/docker/inference/Dockerfile.tgi @@ -1,9 +1,7 @@ FROM ghcr.io/huggingface/text-generation-inference:2.4.0 RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple -RUN pip install --no-cache-dir csghub-sdk==0.4.3 supervisor -RUN apt-get update && apt-get install -y supervisor -RUN mkdir -p /var/log/supervisord -COPY ./supervisord.conf /etc/supervisor/conf.d/supervisord.conf +RUN pip install --no-cache-dir csghub-sdk==0.4.5 +RUN apt-get update && apt-get install -y dumb-init && apt-get clean && rm -rf /var/lib/apt/lists/* COPY ./tgi/ /etc/csghub/ RUN chmod +x /etc/csghub/*.sh @@ -12,5 +10,5 @@ ENV HUGGINGFACE_HUB_CACHE=/data \ HF_HUB_ENABLE_HF_TRANSFER=0 ENV PORT=8000 EXPOSE 8000 - -ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] +ENTRYPOINT [ "/usr/bin/dumb-init", "--" ] +CMD ["/etc/csghub/serve.sh"] diff --git a/runner/component/workflow.go b/runner/component/workflow.go index 6c81d08c6..9f9fe9702 100644 --- a/runner/component/workflow.go +++ b/runner/component/workflow.go @@ -332,6 +332,9 @@ func (wc *workFlowComponentImpl) RunWorkflowsInformer(clusterPool *cluster.Clust } func (wc *workFlowComponentImpl) StartAcctRequestFee(wf database.ArgoWorkflow) { + if !wc.config.IsMasterHost { + return + } if wf.ResourceId == 0 { return } diff --git a/runner/handler/service.go b/runner/handler/service.go index 1d4ec4363..863e4c281 100644 --- a/runner/handler/service.go +++ b/runner/handler/service.go @@ -531,7 +531,7 @@ func (s *K8sHandler) GetLogsByPod(c *gin.Context, cluster cluster.Cluster, podNa slog.Error("write data failed", "error", err) } c.Writer.Flush() - slog.Info("send pod logs", slog.String("srv_name", srvName), slog.String("srv_name", srvName), slog.Int("len", n), slog.String("log", string(buf[:n]))) + slog.Info("send pod logs", slog.String("srv_name", srvName), slog.String("srv_name", srvName), slog.Int("len", n)) } } @@ -628,8 +628,9 @@ func (s *K8sHandler) GetClusterInfo(c *gin.Context) { clusterRes := []types.CluserResponse{} for index := range s.clusterPool.Clusters { cls := s.clusterPool.Clusters[index] - cInfo, _ := s.clusterPool.ClusterStore.ByClusterConfig(c.Request.Context(), cls.ID) - if !cInfo.Enable { + cInfo, err := s.clusterPool.ClusterStore.ByClusterConfig(c.Request.Context(), cls.ID) + if err != nil { + slog.Error("get cluster info failed", slog.Any("error", err)) continue } clusterInfo := types.CluserResponse{} @@ -792,9 +793,6 @@ func (s *K8sHandler) GetReplica(c *gin.Context) { } // revision exist - deployIDStr := srv.Annotations[types.ResDeployID] - deployID, _ := strconv.ParseInt(deployIDStr, 10, 64) - resp.DeployID = deployID resp.Code = 1 resp.Message = srvName resp.ActualReplica = int(*revision.Status.ActualReplicas) From 4bf69cd683a9a2e5a3470d6a5a99df5e68148fc7 Mon Sep 17 00:00:00 2001 From: Xiang Zhen Gan Date: Thu, 5 Dec 2024 20:43:03 +0800 Subject: [PATCH 05/15] fix --- builder/deploy/deployer.go | 18 ++++++------- builder/store/database/argo_workflow.go | 35 ++++++++++++++++++------- component/tag.go | 1 + component/user.go | 3 ++- runner/component/workflow.go | 1 - runner/handler/workflow.go | 2 +- 6 files changed, 38 insertions(+), 22 deletions(-) diff --git a/builder/deploy/deployer.go b/builder/deploy/deployer.go index 47c9d16ea..8e05d81a3 100644 --- a/builder/deploy/deployer.go +++ b/builder/deploy/deployer.go @@ -38,7 +38,7 @@ type Deployer interface { UpdateCluster(ctx context.Context, data types.ClusterRequest) (*types.UpdateClusterResponse, error) UpdateDeploy(ctx context.Context, dur *types.DeployUpdateReq, deploy *database.Deploy) error StartDeploy(ctx context.Context, deploy *database.Deploy) error - CheckResourceAvailable(ctx context.Context, clusterId string, orderDetailID int64, hardWare *types.HardWare) (bool, error) + CheckResourceAvailable(ctx context.Context, clusterId string, hardWare *types.HardWare) (bool, error) SubmitEvaluation(ctx context.Context, req types.EvaluationReq) (*types.ArgoWorkFlowRes, error) ListEvaluations(context.Context, string, int, int) (*types.ArgoWorkFlowListRes, error) DeleteEvaluation(ctx context.Context, req types.ArgoWorkFlowDeleteReq) error @@ -83,15 +83,13 @@ func newDeployer(s scheduler.Scheduler, ib imagebuilder.Builder, ir imagerunner. } go d.refreshStatus() - if d.c.IsMasterHost { - go func() { - err = d.s.Run() - if err != nil { - slog.Error("run scheduler failed", slog.Any("error", err)) - } - }() - go d.startAccounting() - } + go func() { + err = d.s.Run() + if err != nil { + slog.Error("run scheduler failed", slog.Any("error", err)) + } + }() + go d.startAccounting() return d, nil } diff --git a/builder/store/database/argo_workflow.go b/builder/store/database/argo_workflow.go index 2ddb1b0da..4378f5b77 100644 --- a/builder/store/database/argo_workflow.go +++ b/builder/store/database/argo_workflow.go @@ -9,16 +9,33 @@ import ( "opencsg.com/csghub-server/common/types" ) -type ArgoWorkFlowStore struct { +type argoWorkFlowStoreImpl struct { db *DB } -func NewArgoWorkFlowStore() *ArgoWorkFlowStore { - return &ArgoWorkFlowStore{ +type ArgoWorkFlowStore interface { + FindByID(ctx context.Context, id int64) (WorkFlow ArgoWorkflow, err error) + FindByTaskID(ctx context.Context, id string) (WorkFlow ArgoWorkflow, err error) + FindByUsername(ctx context.Context, username string, per, page int) (WorkFlows []ArgoWorkflow, total int, err error) + CreateWorkFlow(ctx context.Context, workFlow ArgoWorkflow) (*ArgoWorkflow, error) + // mainly for update status + UpdateWorkFlow(ctx context.Context, workFlow ArgoWorkflow) (*ArgoWorkflow, error) + // delete workflow by id + DeleteWorkFlow(ctx context.Context, id int64) error +} + +func NewArgoWorkFlowStore() ArgoWorkFlowStore { + return &argoWorkFlowStoreImpl{ db: defaultDB, } } +func NewArgoWorkFlowStoreWithDB(db *DB) ArgoWorkFlowStore { + return &argoWorkFlowStoreImpl{ + db: db, + } +} + type ArgoWorkflow struct { ID int64 `bun:",pk,autoincrement" json:"id"` Username string `bun:",notnull" json:"username"` @@ -45,7 +62,7 @@ type ArgoWorkflow struct { FailuresURL string `bun:"," json:"failures_url"` } -func (s *ArgoWorkFlowStore) FindByID(ctx context.Context, id int64) (WorkFlow ArgoWorkflow, err error) { +func (s *argoWorkFlowStoreImpl) FindByID(ctx context.Context, id int64) (WorkFlow ArgoWorkflow, err error) { err = s.db.Operator.Core.NewSelect().Model(&WorkFlow).Where("id = ?", id).Scan(ctx, &WorkFlow) if err != nil { return @@ -53,7 +70,7 @@ func (s *ArgoWorkFlowStore) FindByID(ctx context.Context, id int64) (WorkFlow Ar return } -func (s *ArgoWorkFlowStore) FindByTaskID(ctx context.Context, id string) (WorkFlow ArgoWorkflow, err error) { +func (s *argoWorkFlowStoreImpl) FindByTaskID(ctx context.Context, id string) (WorkFlow ArgoWorkflow, err error) { err = s.db.Operator.Core.NewSelect().Model(&WorkFlow).Where("task_id = ?", id).Scan(ctx, &WorkFlow) if err != nil { return @@ -61,7 +78,7 @@ func (s *ArgoWorkFlowStore) FindByTaskID(ctx context.Context, id string) (WorkFl return } -func (s *ArgoWorkFlowStore) FindByUsername(ctx context.Context, username string, per, page int) (WorkFlows []ArgoWorkflow, total int, err error) { +func (s *argoWorkFlowStoreImpl) FindByUsername(ctx context.Context, username string, per, page int) (WorkFlows []ArgoWorkflow, total int, err error) { query := s.db.Operator.Core. NewSelect(). Model(&WorkFlows). @@ -82,7 +99,7 @@ func (s *ArgoWorkFlowStore) FindByUsername(ctx context.Context, username string, return } -func (s *ArgoWorkFlowStore) CreateWorkFlow(ctx context.Context, workFlow ArgoWorkflow) (*ArgoWorkflow, error) { +func (s *argoWorkFlowStoreImpl) CreateWorkFlow(ctx context.Context, workFlow ArgoWorkflow) (*ArgoWorkflow, error) { res, err := s.db.Core.NewInsert().Model(&workFlow).Exec(ctx, &workFlow) if err := assertAffectedOneRow(res, err); err != nil { return nil, fmt.Errorf("failed to save WorkFlow in db, error:%w", err) @@ -92,13 +109,13 @@ func (s *ArgoWorkFlowStore) CreateWorkFlow(ctx context.Context, workFlow ArgoWor } // mainly for update status -func (s *ArgoWorkFlowStore) UpdateWorkFlow(ctx context.Context, workFlow ArgoWorkflow) (*ArgoWorkflow, error) { +func (s *argoWorkFlowStoreImpl) UpdateWorkFlow(ctx context.Context, workFlow ArgoWorkflow) (*ArgoWorkflow, error) { _, err := s.db.Core.NewUpdate().Model(&workFlow).WherePK().Exec(ctx) return &workFlow, err } // delete workflow by id -func (s *ArgoWorkFlowStore) DeleteWorkFlow(ctx context.Context, id int64) error { +func (s *argoWorkFlowStoreImpl) DeleteWorkFlow(ctx context.Context, id int64) error { _, err := s.db.Core.NewDelete().Model(&ArgoWorkflow{}).Where("id = ?", id).Exec(ctx) return err } diff --git a/component/tag.go b/component/tag.go index dd9579a98..d428b8952 100644 --- a/component/tag.go +++ b/component/tag.go @@ -15,6 +15,7 @@ import ( ) type TagComponent interface { + AllTagsByScopeAndCategory(ctx context.Context, scope string, category string) ([]*database.Tag, error) AllTags(ctx context.Context) ([]database.Tag, error) ClearMetaTags(ctx context.Context, repoType types.RepositoryType, namespace, name string) error UpdateMetaTags(ctx context.Context, tagScope database.TagScope, namespace, name, content string) ([]*database.RepositoryTag, error) diff --git a/component/user.go b/component/user.go index 1dbc8f06c..86cc31e91 100644 --- a/component/user.go +++ b/component/user.go @@ -38,6 +38,7 @@ type UserComponent interface { ListServerless(ctx context.Context, req types.DeployReq) ([]types.DeployRepo, int, error) GetUserByName(ctx context.Context, userName string) (*database.User, error) Prompts(ctx context.Context, req *types.UserPromptsReq) ([]types.PromptRes, int, error) + Evaluations(ctx context.Context, req *types.UserEvaluationReq) ([]types.ArgoWorkFlowRes, int, error) } func NewUserComponent(config *config.Config) (UserComponent, error) { @@ -98,7 +99,7 @@ type userComponentImpl struct { // srs database.SpaceResourceStore // urs *database.UserResourcesStore promptStore database.PromptStore - wfs *database.ArgoWorkFlowStore + wfs database.ArgoWorkFlowStore } func (c *userComponentImpl) Datasets(ctx context.Context, req *types.UserDatasetsReq) ([]types.Dataset, int, error) { diff --git a/runner/component/workflow.go b/runner/component/workflow.go index 9f9fe9702..3b1642563 100644 --- a/runner/component/workflow.go +++ b/runner/component/workflow.go @@ -45,7 +45,6 @@ type WorkFlowComponent interface { DeleteWorkflowInargo(ctx context.Context, delete *v1alpha1.Workflow) error FindWorkFlowById(ctx context.Context, id int64) (database.ArgoWorkflow, error) RunWorkflowsInformer(clusterPool *cluster.ClusterPool, config *config.Config) - StartAcctRequestFee(wf database.ArgoWorkflow) } func NewWorkFlowComponent(config *config.Config, clusterPool *cluster.ClusterPool) WorkFlowComponent { diff --git a/runner/handler/workflow.go b/runner/handler/workflow.go index 16fd59219..b29f7b411 100644 --- a/runner/handler/workflow.go +++ b/runner/handler/workflow.go @@ -19,7 +19,7 @@ type ArgoHandler struct { workflowNameSpace string modelDockerRegBase string config *config.Config - wfc *component.WorkFlowComponent + wfc component.WorkFlowComponent } func NewArgoHandler(config *config.Config, clusterPool *cluster.ClusterPool) (*ArgoHandler, error) { From 0de6abab62ae334e7a338a0e786d9c2b443fda58 Mon Sep 17 00:00:00 2001 From: Xiang Zhen Gan Date: Thu, 5 Dec 2024 21:35:34 +0800 Subject: [PATCH 06/15] fix --- component/model.go | 2 +- component/user.go | 4 ---- go.mod | 19 +++++++++++-------- go.sum | 30 ++++++++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 13 deletions(-) diff --git a/component/model.go b/component/model.go index 2b5afe17f..c06c6e806 100644 --- a/component/model.go +++ b/component/model.go @@ -476,7 +476,7 @@ func (c *modelComponentImpl) Show(ctx context.Context, namespace, name, currentU if len(finetunes) > 0 { resModel.EnableFinetune = true } - evaluations, _ := c.rrtfms.GetByRepoIDsAndType(ctx, model.Repository.ID, types.EvaluationType) + evaluations, _ := c.repoRuntimeFrameworkStore.GetByRepoIDsAndType(ctx, model.Repository.ID, types.EvaluationType) if len(evaluations) > 0 { resModel.EnableEvaluation = true } diff --git a/component/user.go b/component/user.go index 86cc31e91..16a8aab28 100644 --- a/component/user.go +++ b/component/user.go @@ -88,12 +88,8 @@ type userComponentImpl struct { spaceStore database.SpaceStore namespaceStore database.NamespaceStore gitServer gitserver.GitServer - spaceComponent SpaceComponent - repoComponent RepoComponent - deployer deploy.Deployer userLikeStore database.UserLikesStore repoStore database.RepoStore - deploy database.DeployTaskStore collectionStore database.CollectionStore accountingComponent AccountingComponent // srs database.SpaceResourceStore diff --git a/go.mod b/go.mod index 7fbef3f2f..b19433b53 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/d5/tengo/v2 v2.17.0 github.com/gin-contrib/cors v1.7.0 github.com/gin-contrib/sessions v0.0.5 - github.com/gin-gonic/gin v1.9.1 + github.com/gin-gonic/gin v1.10.0 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f github.com/google/wire v0.6.0 @@ -46,9 +46,9 @@ require ( google.golang.org/grpc v1.66.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.31.0 - k8s.io/apimachinery v0.31.0 - k8s.io/client-go v0.31.0 + k8s.io/api v0.31.3 + k8s.io/apimachinery v0.31.3 + k8s.io/client-go v0.31.3 knative.dev/serving v0.40.1 ) @@ -65,8 +65,11 @@ require ( github.com/aws/aws-sdk-go v1.50.36 // indirect github.com/beevik/ntp v1.3.1 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect github.com/containerd/containerd v1.7.18 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect @@ -161,7 +164,7 @@ require ( github.com/argoproj/argo-workflows/v3 v3.5.12 github.com/blendle/zapdriver v1.3.1 // indirect github.com/bwmarrin/snowflake v0.3.0 - github.com/bytedance/sonic v1.11.2 // indirect + github.com/bytedance/sonic v1.11.6 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.1 // indirect @@ -183,7 +186,7 @@ require ( github.com/go-openapi/swag v0.22.7 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.19.0 + github.com/go-playground/validator/v10 v10.20.0 github.com/go-redis/redis/v8 v8.11.5 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -220,7 +223,7 @@ require ( github.com/nats-io/nats.go v1.35.0 github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect - github.com/pelletier/go-toml/v2 v2.2.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rs/xid v1.5.0 // indirect @@ -235,7 +238,7 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect - golang.org/x/arch v0.7.0 // indirect + golang.org/x/arch v0.8.0 // indirect golang.org/x/crypto v0.27.0 golang.org/x/net v0.28.0 // indirect golang.org/x/oauth2 v0.22.0 // indirect diff --git a/go.sum b/go.sum index 70e140cb3..bcaaecbf1 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,10 @@ github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1 github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= github.com/bytedance/sonic v1.11.2 h1:ywfwo0a/3j9HR8wsYGWsIWl2mvRsI950HyoxiBERw5A= github.com/bytedance/sonic v1.11.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/casdoor/casdoor-go-sdk v0.41.0 h1:mqqoc1Jub34/OkAQqjeASRAaiy7x/5ZWtGObI08cfEk= github.com/casdoor/casdoor-go-sdk v0.41.0/go.mod h1:cMnkCQJgMYpgAlgEx8reSt1AVaDIQLcJ1zk5pzBaz+4= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= @@ -125,6 +129,10 @@ github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/tableflip v1.2.3 h1:8I+B99QnnEWPHOY3fWipwVKxS70LGgUsslG7CSfmHMw= github.com/cloudflare/tableflip v1.2.3/go.mod h1:P4gRehmV6Z2bY5ao5ml9Pd8u6kuEnlB37pUFMmv7j2E= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= github.com/containerd/containerd v1.7.18/go.mod h1:IYEk9/IO6wAPUz2bCMVUbsfXjzw5UNP5fLz4PsUygQ4= @@ -199,6 +207,8 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -236,6 +246,8 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4= github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= @@ -514,6 +526,8 @@ github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/pelletier/go-toml v1.0.1-0.20170904195809-1d6b12b7cb29/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.2.0 h1:QLgLl2yMN7N+ruc31VynXs1vhMZa7CeHHejIeBAsoHo= github.com/pelletier/go-toml/v2 v2.2.0/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ= github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -706,6 +720,8 @@ go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -722,6 +738,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -843,6 +861,8 @@ golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -853,6 +873,8 @@ golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -865,6 +887,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20170424234030-8be79e1e0910/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= @@ -988,10 +1012,16 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo= k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE= +k8s.io/api v0.31.3 h1:umzm5o8lFbdN/hIXbrK9oRpOproJO62CV1zqxXrLgk8= +k8s.io/api v0.31.3/go.mod h1:UJrkIp9pnMOI9K2nlL6vwpxRzzEX5sWgn8kGQe92kCE= k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc= k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/apimachinery v0.31.3 h1:6l0WhcYgasZ/wk9ktLq5vLaoXJJr5ts6lkaQzgeYPq4= +k8s.io/apimachinery v0.31.3/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8= k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU= +k8s.io/client-go v0.31.3 h1:CAlZuM+PH2cm+86LOBemaJI/lQ5linJ6UFxKX/SoG+4= +k8s.io/client-go v0.31.3/go.mod h1:2CgjPUTpv3fE5dNygAr2NcM8nhHzXvxB8KL5gYc3kJs= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= From 5934e34388cfd7374a34cc66530c38ebf4cb5e93 Mon Sep 17 00:00:00 2001 From: Xiang Zhen Gan Date: Thu, 5 Dec 2024 22:51:38 +0800 Subject: [PATCH 07/15] fix bug --- .../builder/deploy/mock_Deployer.go | 226 +++++++++++ .../store/database/mock_ArgoWorkFlowStore.go | 384 ++++++++++++++++++ .../store/database/mock_TagRuleStore.go | 99 +++++ .../builder/store/database/mock_TagStore.go | 61 +++ .../component/mock_TagComponent.go | 60 +++ cmd/csghub-server/cmd/start/server.go | 2 + common/config/config.go | 7 +- common/tests/stores.go | 14 +- component/evaluation_test.go | 12 - component/model_test.go | 6 +- component/user.go | 4 + 11 files changed, 848 insertions(+), 27 deletions(-) create mode 100644 _mocks/opencsg.com/csghub-server/builder/store/database/mock_ArgoWorkFlowStore.go create mode 100644 _mocks/opencsg.com/csghub-server/builder/store/database/mock_TagRuleStore.go diff --git a/_mocks/opencsg.com/csghub-server/builder/deploy/mock_Deployer.go b/_mocks/opencsg.com/csghub-server/builder/deploy/mock_Deployer.go index 2f067af46..954cb181f 100644 --- a/_mocks/opencsg.com/csghub-server/builder/deploy/mock_Deployer.go +++ b/_mocks/opencsg.com/csghub-server/builder/deploy/mock_Deployer.go @@ -84,6 +84,53 @@ func (_c *MockDeployer_CheckResourceAvailable_Call) RunAndReturn(run func(contex return _c } +// DeleteEvaluation provides a mock function with given fields: ctx, req +func (_m *MockDeployer) DeleteEvaluation(ctx context.Context, req types.ArgoWorkFlowDeleteReq) error { + ret := _m.Called(ctx, req) + + if len(ret) == 0 { + panic("no return value specified for DeleteEvaluation") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, types.ArgoWorkFlowDeleteReq) error); ok { + r0 = rf(ctx, req) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockDeployer_DeleteEvaluation_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteEvaluation' +type MockDeployer_DeleteEvaluation_Call struct { + *mock.Call +} + +// DeleteEvaluation is a helper method to define mock.On call +// - ctx context.Context +// - req types.ArgoWorkFlowDeleteReq +func (_e *MockDeployer_Expecter) DeleteEvaluation(ctx interface{}, req interface{}) *MockDeployer_DeleteEvaluation_Call { + return &MockDeployer_DeleteEvaluation_Call{Call: _e.mock.On("DeleteEvaluation", ctx, req)} +} + +func (_c *MockDeployer_DeleteEvaluation_Call) Run(run func(ctx context.Context, req types.ArgoWorkFlowDeleteReq)) *MockDeployer_DeleteEvaluation_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(types.ArgoWorkFlowDeleteReq)) + }) + return _c +} + +func (_c *MockDeployer_DeleteEvaluation_Call) Return(_a0 error) *MockDeployer_DeleteEvaluation_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockDeployer_DeleteEvaluation_Call) RunAndReturn(run func(context.Context, types.ArgoWorkFlowDeleteReq) error) *MockDeployer_DeleteEvaluation_Call { + _c.Call.Return(run) + return _c +} + // Deploy provides a mock function with given fields: ctx, dr func (_m *MockDeployer) Deploy(ctx context.Context, dr types.DeployRepo) (int64, error) { ret := _m.Called(ctx, dr) @@ -257,6 +304,65 @@ func (_c *MockDeployer_GetClusterById_Call) RunAndReturn(run func(context.Contex return _c } +// GetEvaluation provides a mock function with given fields: ctx, req +func (_m *MockDeployer) GetEvaluation(ctx context.Context, req types.EvaluationGetReq) (*types.ArgoWorkFlowRes, error) { + ret := _m.Called(ctx, req) + + if len(ret) == 0 { + panic("no return value specified for GetEvaluation") + } + + var r0 *types.ArgoWorkFlowRes + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, types.EvaluationGetReq) (*types.ArgoWorkFlowRes, error)); ok { + return rf(ctx, req) + } + if rf, ok := ret.Get(0).(func(context.Context, types.EvaluationGetReq) *types.ArgoWorkFlowRes); ok { + r0 = rf(ctx, req) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.ArgoWorkFlowRes) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, types.EvaluationGetReq) error); ok { + r1 = rf(ctx, req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockDeployer_GetEvaluation_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetEvaluation' +type MockDeployer_GetEvaluation_Call struct { + *mock.Call +} + +// GetEvaluation is a helper method to define mock.On call +// - ctx context.Context +// - req types.EvaluationGetReq +func (_e *MockDeployer_Expecter) GetEvaluation(ctx interface{}, req interface{}) *MockDeployer_GetEvaluation_Call { + return &MockDeployer_GetEvaluation_Call{Call: _e.mock.On("GetEvaluation", ctx, req)} +} + +func (_c *MockDeployer_GetEvaluation_Call) Run(run func(ctx context.Context, req types.EvaluationGetReq)) *MockDeployer_GetEvaluation_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(types.EvaluationGetReq)) + }) + return _c +} + +func (_c *MockDeployer_GetEvaluation_Call) Return(_a0 *types.ArgoWorkFlowRes, _a1 error) *MockDeployer_GetEvaluation_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockDeployer_GetEvaluation_Call) RunAndReturn(run func(context.Context, types.EvaluationGetReq) (*types.ArgoWorkFlowRes, error)) *MockDeployer_GetEvaluation_Call { + _c.Call.Return(run) + return _c +} + // GetReplica provides a mock function with given fields: ctx, dr func (_m *MockDeployer) GetReplica(ctx context.Context, dr types.DeployRepo) (int, int, []types.Instance, error) { ret := _m.Called(ctx, dr) @@ -447,6 +553,67 @@ func (_c *MockDeployer_ListCluster_Call) RunAndReturn(run func(context.Context) return _c } +// ListEvaluations provides a mock function with given fields: _a0, _a1, _a2, _a3 +func (_m *MockDeployer) ListEvaluations(_a0 context.Context, _a1 string, _a2 int, _a3 int) (*types.ArgoWorkFlowListRes, error) { + ret := _m.Called(_a0, _a1, _a2, _a3) + + if len(ret) == 0 { + panic("no return value specified for ListEvaluations") + } + + var r0 *types.ArgoWorkFlowListRes + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, int, int) (*types.ArgoWorkFlowListRes, error)); ok { + return rf(_a0, _a1, _a2, _a3) + } + if rf, ok := ret.Get(0).(func(context.Context, string, int, int) *types.ArgoWorkFlowListRes); ok { + r0 = rf(_a0, _a1, _a2, _a3) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.ArgoWorkFlowListRes) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, int, int) error); ok { + r1 = rf(_a0, _a1, _a2, _a3) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockDeployer_ListEvaluations_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListEvaluations' +type MockDeployer_ListEvaluations_Call struct { + *mock.Call +} + +// ListEvaluations is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 string +// - _a2 int +// - _a3 int +func (_e *MockDeployer_Expecter) ListEvaluations(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}) *MockDeployer_ListEvaluations_Call { + return &MockDeployer_ListEvaluations_Call{Call: _e.mock.On("ListEvaluations", _a0, _a1, _a2, _a3)} +} + +func (_c *MockDeployer_ListEvaluations_Call) Run(run func(_a0 context.Context, _a1 string, _a2 int, _a3 int)) *MockDeployer_ListEvaluations_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(int), args[3].(int)) + }) + return _c +} + +func (_c *MockDeployer_ListEvaluations_Call) Return(_a0 *types.ArgoWorkFlowListRes, _a1 error) *MockDeployer_ListEvaluations_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockDeployer_ListEvaluations_Call) RunAndReturn(run func(context.Context, string, int, int) (*types.ArgoWorkFlowListRes, error)) *MockDeployer_ListEvaluations_Call { + _c.Call.Return(run) + return _c +} + // Logs provides a mock function with given fields: ctx, dr func (_m *MockDeployer) Logs(ctx context.Context, dr types.DeployRepo) (*deploy.MultiLogReader, error) { ret := _m.Called(ctx, dr) @@ -721,6 +888,65 @@ func (_c *MockDeployer_Stop_Call) RunAndReturn(run func(context.Context, types.D return _c } +// SubmitEvaluation provides a mock function with given fields: ctx, req +func (_m *MockDeployer) SubmitEvaluation(ctx context.Context, req types.EvaluationReq) (*types.ArgoWorkFlowRes, error) { + ret := _m.Called(ctx, req) + + if len(ret) == 0 { + panic("no return value specified for SubmitEvaluation") + } + + var r0 *types.ArgoWorkFlowRes + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, types.EvaluationReq) (*types.ArgoWorkFlowRes, error)); ok { + return rf(ctx, req) + } + if rf, ok := ret.Get(0).(func(context.Context, types.EvaluationReq) *types.ArgoWorkFlowRes); ok { + r0 = rf(ctx, req) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.ArgoWorkFlowRes) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, types.EvaluationReq) error); ok { + r1 = rf(ctx, req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockDeployer_SubmitEvaluation_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SubmitEvaluation' +type MockDeployer_SubmitEvaluation_Call struct { + *mock.Call +} + +// SubmitEvaluation is a helper method to define mock.On call +// - ctx context.Context +// - req types.EvaluationReq +func (_e *MockDeployer_Expecter) SubmitEvaluation(ctx interface{}, req interface{}) *MockDeployer_SubmitEvaluation_Call { + return &MockDeployer_SubmitEvaluation_Call{Call: _e.mock.On("SubmitEvaluation", ctx, req)} +} + +func (_c *MockDeployer_SubmitEvaluation_Call) Run(run func(ctx context.Context, req types.EvaluationReq)) *MockDeployer_SubmitEvaluation_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(types.EvaluationReq)) + }) + return _c +} + +func (_c *MockDeployer_SubmitEvaluation_Call) Return(_a0 *types.ArgoWorkFlowRes, _a1 error) *MockDeployer_SubmitEvaluation_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockDeployer_SubmitEvaluation_Call) RunAndReturn(run func(context.Context, types.EvaluationReq) (*types.ArgoWorkFlowRes, error)) *MockDeployer_SubmitEvaluation_Call { + _c.Call.Return(run) + return _c +} + // UpdateCluster provides a mock function with given fields: ctx, data func (_m *MockDeployer) UpdateCluster(ctx context.Context, data types.ClusterRequest) (*types.UpdateClusterResponse, error) { ret := _m.Called(ctx, data) diff --git a/_mocks/opencsg.com/csghub-server/builder/store/database/mock_ArgoWorkFlowStore.go b/_mocks/opencsg.com/csghub-server/builder/store/database/mock_ArgoWorkFlowStore.go new file mode 100644 index 000000000..a392d894a --- /dev/null +++ b/_mocks/opencsg.com/csghub-server/builder/store/database/mock_ArgoWorkFlowStore.go @@ -0,0 +1,384 @@ +// Code generated by mockery v2.49.1. DO NOT EDIT. + +package database + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + database "opencsg.com/csghub-server/builder/store/database" +) + +// MockArgoWorkFlowStore is an autogenerated mock type for the ArgoWorkFlowStore type +type MockArgoWorkFlowStore struct { + mock.Mock +} + +type MockArgoWorkFlowStore_Expecter struct { + mock *mock.Mock +} + +func (_m *MockArgoWorkFlowStore) EXPECT() *MockArgoWorkFlowStore_Expecter { + return &MockArgoWorkFlowStore_Expecter{mock: &_m.Mock} +} + +// CreateWorkFlow provides a mock function with given fields: ctx, workFlow +func (_m *MockArgoWorkFlowStore) CreateWorkFlow(ctx context.Context, workFlow database.ArgoWorkflow) (*database.ArgoWorkflow, error) { + ret := _m.Called(ctx, workFlow) + + if len(ret) == 0 { + panic("no return value specified for CreateWorkFlow") + } + + var r0 *database.ArgoWorkflow + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, database.ArgoWorkflow) (*database.ArgoWorkflow, error)); ok { + return rf(ctx, workFlow) + } + if rf, ok := ret.Get(0).(func(context.Context, database.ArgoWorkflow) *database.ArgoWorkflow); ok { + r0 = rf(ctx, workFlow) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*database.ArgoWorkflow) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, database.ArgoWorkflow) error); ok { + r1 = rf(ctx, workFlow) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockArgoWorkFlowStore_CreateWorkFlow_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateWorkFlow' +type MockArgoWorkFlowStore_CreateWorkFlow_Call struct { + *mock.Call +} + +// CreateWorkFlow is a helper method to define mock.On call +// - ctx context.Context +// - workFlow database.ArgoWorkflow +func (_e *MockArgoWorkFlowStore_Expecter) CreateWorkFlow(ctx interface{}, workFlow interface{}) *MockArgoWorkFlowStore_CreateWorkFlow_Call { + return &MockArgoWorkFlowStore_CreateWorkFlow_Call{Call: _e.mock.On("CreateWorkFlow", ctx, workFlow)} +} + +func (_c *MockArgoWorkFlowStore_CreateWorkFlow_Call) Run(run func(ctx context.Context, workFlow database.ArgoWorkflow)) *MockArgoWorkFlowStore_CreateWorkFlow_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(database.ArgoWorkflow)) + }) + return _c +} + +func (_c *MockArgoWorkFlowStore_CreateWorkFlow_Call) Return(_a0 *database.ArgoWorkflow, _a1 error) *MockArgoWorkFlowStore_CreateWorkFlow_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockArgoWorkFlowStore_CreateWorkFlow_Call) RunAndReturn(run func(context.Context, database.ArgoWorkflow) (*database.ArgoWorkflow, error)) *MockArgoWorkFlowStore_CreateWorkFlow_Call { + _c.Call.Return(run) + return _c +} + +// DeleteWorkFlow provides a mock function with given fields: ctx, id +func (_m *MockArgoWorkFlowStore) DeleteWorkFlow(ctx context.Context, id int64) error { + ret := _m.Called(ctx, id) + + if len(ret) == 0 { + panic("no return value specified for DeleteWorkFlow") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok { + r0 = rf(ctx, id) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockArgoWorkFlowStore_DeleteWorkFlow_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteWorkFlow' +type MockArgoWorkFlowStore_DeleteWorkFlow_Call struct { + *mock.Call +} + +// DeleteWorkFlow is a helper method to define mock.On call +// - ctx context.Context +// - id int64 +func (_e *MockArgoWorkFlowStore_Expecter) DeleteWorkFlow(ctx interface{}, id interface{}) *MockArgoWorkFlowStore_DeleteWorkFlow_Call { + return &MockArgoWorkFlowStore_DeleteWorkFlow_Call{Call: _e.mock.On("DeleteWorkFlow", ctx, id)} +} + +func (_c *MockArgoWorkFlowStore_DeleteWorkFlow_Call) Run(run func(ctx context.Context, id int64)) *MockArgoWorkFlowStore_DeleteWorkFlow_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(int64)) + }) + return _c +} + +func (_c *MockArgoWorkFlowStore_DeleteWorkFlow_Call) Return(_a0 error) *MockArgoWorkFlowStore_DeleteWorkFlow_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockArgoWorkFlowStore_DeleteWorkFlow_Call) RunAndReturn(run func(context.Context, int64) error) *MockArgoWorkFlowStore_DeleteWorkFlow_Call { + _c.Call.Return(run) + return _c +} + +// FindByID provides a mock function with given fields: ctx, id +func (_m *MockArgoWorkFlowStore) FindByID(ctx context.Context, id int64) (database.ArgoWorkflow, error) { + ret := _m.Called(ctx, id) + + if len(ret) == 0 { + panic("no return value specified for FindByID") + } + + var r0 database.ArgoWorkflow + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, int64) (database.ArgoWorkflow, error)); ok { + return rf(ctx, id) + } + if rf, ok := ret.Get(0).(func(context.Context, int64) database.ArgoWorkflow); ok { + r0 = rf(ctx, id) + } else { + r0 = ret.Get(0).(database.ArgoWorkflow) + } + + if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockArgoWorkFlowStore_FindByID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FindByID' +type MockArgoWorkFlowStore_FindByID_Call struct { + *mock.Call +} + +// FindByID is a helper method to define mock.On call +// - ctx context.Context +// - id int64 +func (_e *MockArgoWorkFlowStore_Expecter) FindByID(ctx interface{}, id interface{}) *MockArgoWorkFlowStore_FindByID_Call { + return &MockArgoWorkFlowStore_FindByID_Call{Call: _e.mock.On("FindByID", ctx, id)} +} + +func (_c *MockArgoWorkFlowStore_FindByID_Call) Run(run func(ctx context.Context, id int64)) *MockArgoWorkFlowStore_FindByID_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(int64)) + }) + return _c +} + +func (_c *MockArgoWorkFlowStore_FindByID_Call) Return(WorkFlow database.ArgoWorkflow, err error) *MockArgoWorkFlowStore_FindByID_Call { + _c.Call.Return(WorkFlow, err) + return _c +} + +func (_c *MockArgoWorkFlowStore_FindByID_Call) RunAndReturn(run func(context.Context, int64) (database.ArgoWorkflow, error)) *MockArgoWorkFlowStore_FindByID_Call { + _c.Call.Return(run) + return _c +} + +// FindByTaskID provides a mock function with given fields: ctx, id +func (_m *MockArgoWorkFlowStore) FindByTaskID(ctx context.Context, id string) (database.ArgoWorkflow, error) { + ret := _m.Called(ctx, id) + + if len(ret) == 0 { + panic("no return value specified for FindByTaskID") + } + + var r0 database.ArgoWorkflow + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (database.ArgoWorkflow, error)); ok { + return rf(ctx, id) + } + if rf, ok := ret.Get(0).(func(context.Context, string) database.ArgoWorkflow); ok { + r0 = rf(ctx, id) + } else { + r0 = ret.Get(0).(database.ArgoWorkflow) + } + + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockArgoWorkFlowStore_FindByTaskID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FindByTaskID' +type MockArgoWorkFlowStore_FindByTaskID_Call struct { + *mock.Call +} + +// FindByTaskID is a helper method to define mock.On call +// - ctx context.Context +// - id string +func (_e *MockArgoWorkFlowStore_Expecter) FindByTaskID(ctx interface{}, id interface{}) *MockArgoWorkFlowStore_FindByTaskID_Call { + return &MockArgoWorkFlowStore_FindByTaskID_Call{Call: _e.mock.On("FindByTaskID", ctx, id)} +} + +func (_c *MockArgoWorkFlowStore_FindByTaskID_Call) Run(run func(ctx context.Context, id string)) *MockArgoWorkFlowStore_FindByTaskID_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *MockArgoWorkFlowStore_FindByTaskID_Call) Return(WorkFlow database.ArgoWorkflow, err error) *MockArgoWorkFlowStore_FindByTaskID_Call { + _c.Call.Return(WorkFlow, err) + return _c +} + +func (_c *MockArgoWorkFlowStore_FindByTaskID_Call) RunAndReturn(run func(context.Context, string) (database.ArgoWorkflow, error)) *MockArgoWorkFlowStore_FindByTaskID_Call { + _c.Call.Return(run) + return _c +} + +// FindByUsername provides a mock function with given fields: ctx, username, per, page +func (_m *MockArgoWorkFlowStore) FindByUsername(ctx context.Context, username string, per int, page int) ([]database.ArgoWorkflow, int, error) { + ret := _m.Called(ctx, username, per, page) + + if len(ret) == 0 { + panic("no return value specified for FindByUsername") + } + + var r0 []database.ArgoWorkflow + var r1 int + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, string, int, int) ([]database.ArgoWorkflow, int, error)); ok { + return rf(ctx, username, per, page) + } + if rf, ok := ret.Get(0).(func(context.Context, string, int, int) []database.ArgoWorkflow); ok { + r0 = rf(ctx, username, per, page) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]database.ArgoWorkflow) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, int, int) int); ok { + r1 = rf(ctx, username, per, page) + } else { + r1 = ret.Get(1).(int) + } + + if rf, ok := ret.Get(2).(func(context.Context, string, int, int) error); ok { + r2 = rf(ctx, username, per, page) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// MockArgoWorkFlowStore_FindByUsername_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FindByUsername' +type MockArgoWorkFlowStore_FindByUsername_Call struct { + *mock.Call +} + +// FindByUsername is a helper method to define mock.On call +// - ctx context.Context +// - username string +// - per int +// - page int +func (_e *MockArgoWorkFlowStore_Expecter) FindByUsername(ctx interface{}, username interface{}, per interface{}, page interface{}) *MockArgoWorkFlowStore_FindByUsername_Call { + return &MockArgoWorkFlowStore_FindByUsername_Call{Call: _e.mock.On("FindByUsername", ctx, username, per, page)} +} + +func (_c *MockArgoWorkFlowStore_FindByUsername_Call) Run(run func(ctx context.Context, username string, per int, page int)) *MockArgoWorkFlowStore_FindByUsername_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(int), args[3].(int)) + }) + return _c +} + +func (_c *MockArgoWorkFlowStore_FindByUsername_Call) Return(WorkFlows []database.ArgoWorkflow, total int, err error) *MockArgoWorkFlowStore_FindByUsername_Call { + _c.Call.Return(WorkFlows, total, err) + return _c +} + +func (_c *MockArgoWorkFlowStore_FindByUsername_Call) RunAndReturn(run func(context.Context, string, int, int) ([]database.ArgoWorkflow, int, error)) *MockArgoWorkFlowStore_FindByUsername_Call { + _c.Call.Return(run) + return _c +} + +// UpdateWorkFlow provides a mock function with given fields: ctx, workFlow +func (_m *MockArgoWorkFlowStore) UpdateWorkFlow(ctx context.Context, workFlow database.ArgoWorkflow) (*database.ArgoWorkflow, error) { + ret := _m.Called(ctx, workFlow) + + if len(ret) == 0 { + panic("no return value specified for UpdateWorkFlow") + } + + var r0 *database.ArgoWorkflow + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, database.ArgoWorkflow) (*database.ArgoWorkflow, error)); ok { + return rf(ctx, workFlow) + } + if rf, ok := ret.Get(0).(func(context.Context, database.ArgoWorkflow) *database.ArgoWorkflow); ok { + r0 = rf(ctx, workFlow) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*database.ArgoWorkflow) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, database.ArgoWorkflow) error); ok { + r1 = rf(ctx, workFlow) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockArgoWorkFlowStore_UpdateWorkFlow_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateWorkFlow' +type MockArgoWorkFlowStore_UpdateWorkFlow_Call struct { + *mock.Call +} + +// UpdateWorkFlow is a helper method to define mock.On call +// - ctx context.Context +// - workFlow database.ArgoWorkflow +func (_e *MockArgoWorkFlowStore_Expecter) UpdateWorkFlow(ctx interface{}, workFlow interface{}) *MockArgoWorkFlowStore_UpdateWorkFlow_Call { + return &MockArgoWorkFlowStore_UpdateWorkFlow_Call{Call: _e.mock.On("UpdateWorkFlow", ctx, workFlow)} +} + +func (_c *MockArgoWorkFlowStore_UpdateWorkFlow_Call) Run(run func(ctx context.Context, workFlow database.ArgoWorkflow)) *MockArgoWorkFlowStore_UpdateWorkFlow_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(database.ArgoWorkflow)) + }) + return _c +} + +func (_c *MockArgoWorkFlowStore_UpdateWorkFlow_Call) Return(_a0 *database.ArgoWorkflow, _a1 error) *MockArgoWorkFlowStore_UpdateWorkFlow_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockArgoWorkFlowStore_UpdateWorkFlow_Call) RunAndReturn(run func(context.Context, database.ArgoWorkflow) (*database.ArgoWorkflow, error)) *MockArgoWorkFlowStore_UpdateWorkFlow_Call { + _c.Call.Return(run) + return _c +} + +// NewMockArgoWorkFlowStore creates a new instance of MockArgoWorkFlowStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockArgoWorkFlowStore(t interface { + mock.TestingT + Cleanup(func()) +}) *MockArgoWorkFlowStore { + mock := &MockArgoWorkFlowStore{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/_mocks/opencsg.com/csghub-server/builder/store/database/mock_TagRuleStore.go b/_mocks/opencsg.com/csghub-server/builder/store/database/mock_TagRuleStore.go new file mode 100644 index 000000000..a149c0c06 --- /dev/null +++ b/_mocks/opencsg.com/csghub-server/builder/store/database/mock_TagRuleStore.go @@ -0,0 +1,99 @@ +// Code generated by mockery v2.49.1. DO NOT EDIT. + +package database + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + database "opencsg.com/csghub-server/builder/store/database" +) + +// MockTagRuleStore is an autogenerated mock type for the TagRuleStore type +type MockTagRuleStore struct { + mock.Mock +} + +type MockTagRuleStore_Expecter struct { + mock *mock.Mock +} + +func (_m *MockTagRuleStore) EXPECT() *MockTagRuleStore_Expecter { + return &MockTagRuleStore_Expecter{mock: &_m.Mock} +} + +// FindByRepo provides a mock function with given fields: ctx, category, namespace, repoName, repoType +func (_m *MockTagRuleStore) FindByRepo(ctx context.Context, category string, namespace string, repoName string, repoType string) (*database.TagRule, error) { + ret := _m.Called(ctx, category, namespace, repoName, repoType) + + if len(ret) == 0 { + panic("no return value specified for FindByRepo") + } + + var r0 *database.TagRule + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, string, string) (*database.TagRule, error)); ok { + return rf(ctx, category, namespace, repoName, repoType) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string, string, string) *database.TagRule); ok { + r0 = rf(ctx, category, namespace, repoName, repoType) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*database.TagRule) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string, string, string) error); ok { + r1 = rf(ctx, category, namespace, repoName, repoType) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockTagRuleStore_FindByRepo_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FindByRepo' +type MockTagRuleStore_FindByRepo_Call struct { + *mock.Call +} + +// FindByRepo is a helper method to define mock.On call +// - ctx context.Context +// - category string +// - namespace string +// - repoName string +// - repoType string +func (_e *MockTagRuleStore_Expecter) FindByRepo(ctx interface{}, category interface{}, namespace interface{}, repoName interface{}, repoType interface{}) *MockTagRuleStore_FindByRepo_Call { + return &MockTagRuleStore_FindByRepo_Call{Call: _e.mock.On("FindByRepo", ctx, category, namespace, repoName, repoType)} +} + +func (_c *MockTagRuleStore_FindByRepo_Call) Run(run func(ctx context.Context, category string, namespace string, repoName string, repoType string)) *MockTagRuleStore_FindByRepo_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].(string), args[4].(string)) + }) + return _c +} + +func (_c *MockTagRuleStore_FindByRepo_Call) Return(_a0 *database.TagRule, _a1 error) *MockTagRuleStore_FindByRepo_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockTagRuleStore_FindByRepo_Call) RunAndReturn(run func(context.Context, string, string, string, string) (*database.TagRule, error)) *MockTagRuleStore_FindByRepo_Call { + _c.Call.Return(run) + return _c +} + +// NewMockTagRuleStore creates a new instance of MockTagRuleStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockTagRuleStore(t interface { + mock.TestingT + Cleanup(func()) +}) *MockTagRuleStore { + mock := &MockTagRuleStore{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/_mocks/opencsg.com/csghub-server/builder/store/database/mock_TagStore.go b/_mocks/opencsg.com/csghub-server/builder/store/database/mock_TagStore.go index 862b878db..f2638d831 100644 --- a/_mocks/opencsg.com/csghub-server/builder/store/database/mock_TagStore.go +++ b/_mocks/opencsg.com/csghub-server/builder/store/database/mock_TagStore.go @@ -900,6 +900,67 @@ func (_c *MockTagStore_FindOrCreate_Call) RunAndReturn(run func(context.Context, return _c } +// FindTag provides a mock function with given fields: ctx, name, scope, category +func (_m *MockTagStore) FindTag(ctx context.Context, name string, scope string, category string) (*database.Tag, error) { + ret := _m.Called(ctx, name, scope, category) + + if len(ret) == 0 { + panic("no return value specified for FindTag") + } + + var r0 *database.Tag + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, string) (*database.Tag, error)); ok { + return rf(ctx, name, scope, category) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string, string) *database.Tag); ok { + r0 = rf(ctx, name, scope, category) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*database.Tag) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string, string) error); ok { + r1 = rf(ctx, name, scope, category) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockTagStore_FindTag_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FindTag' +type MockTagStore_FindTag_Call struct { + *mock.Call +} + +// FindTag is a helper method to define mock.On call +// - ctx context.Context +// - name string +// - scope string +// - category string +func (_e *MockTagStore_Expecter) FindTag(ctx interface{}, name interface{}, scope interface{}, category interface{}) *MockTagStore_FindTag_Call { + return &MockTagStore_FindTag_Call{Call: _e.mock.On("FindTag", ctx, name, scope, category)} +} + +func (_c *MockTagStore_FindTag_Call) Run(run func(ctx context.Context, name string, scope string, category string)) *MockTagStore_FindTag_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].(string)) + }) + return _c +} + +func (_c *MockTagStore_FindTag_Call) Return(_a0 *database.Tag, _a1 error) *MockTagStore_FindTag_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockTagStore_FindTag_Call) RunAndReturn(run func(context.Context, string, string, string) (*database.Tag, error)) *MockTagStore_FindTag_Call { + _c.Call.Return(run) + return _c +} + // GetTagsByScopeAndCategories provides a mock function with given fields: ctx, scope, categories func (_m *MockTagStore) GetTagsByScopeAndCategories(ctx context.Context, scope database.TagScope, categories []string) ([]*database.Tag, error) { ret := _m.Called(ctx, scope, categories) diff --git a/_mocks/opencsg.com/csghub-server/component/mock_TagComponent.go b/_mocks/opencsg.com/csghub-server/component/mock_TagComponent.go index 76b732844..af076cd79 100644 --- a/_mocks/opencsg.com/csghub-server/component/mock_TagComponent.go +++ b/_mocks/opencsg.com/csghub-server/component/mock_TagComponent.go @@ -82,6 +82,66 @@ func (_c *MockTagComponent_AllTags_Call) RunAndReturn(run func(context.Context) return _c } +// AllTagsByScopeAndCategory provides a mock function with given fields: ctx, scope, category +func (_m *MockTagComponent) AllTagsByScopeAndCategory(ctx context.Context, scope string, category string) ([]*database.Tag, error) { + ret := _m.Called(ctx, scope, category) + + if len(ret) == 0 { + panic("no return value specified for AllTagsByScopeAndCategory") + } + + var r0 []*database.Tag + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) ([]*database.Tag, error)); ok { + return rf(ctx, scope, category) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string) []*database.Tag); ok { + r0 = rf(ctx, scope, category) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*database.Tag) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, scope, category) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockTagComponent_AllTagsByScopeAndCategory_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AllTagsByScopeAndCategory' +type MockTagComponent_AllTagsByScopeAndCategory_Call struct { + *mock.Call +} + +// AllTagsByScopeAndCategory is a helper method to define mock.On call +// - ctx context.Context +// - scope string +// - category string +func (_e *MockTagComponent_Expecter) AllTagsByScopeAndCategory(ctx interface{}, scope interface{}, category interface{}) *MockTagComponent_AllTagsByScopeAndCategory_Call { + return &MockTagComponent_AllTagsByScopeAndCategory_Call{Call: _e.mock.On("AllTagsByScopeAndCategory", ctx, scope, category)} +} + +func (_c *MockTagComponent_AllTagsByScopeAndCategory_Call) Run(run func(ctx context.Context, scope string, category string)) *MockTagComponent_AllTagsByScopeAndCategory_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *MockTagComponent_AllTagsByScopeAndCategory_Call) Return(_a0 []*database.Tag, _a1 error) *MockTagComponent_AllTagsByScopeAndCategory_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockTagComponent_AllTagsByScopeAndCategory_Call) RunAndReturn(run func(context.Context, string, string) ([]*database.Tag, error)) *MockTagComponent_AllTagsByScopeAndCategory_Call { + _c.Call.Return(run) + return _c +} + // ClearMetaTags provides a mock function with given fields: ctx, repoType, namespace, name func (_m *MockTagComponent) ClearMetaTags(ctx context.Context, repoType types.RepositoryType, namespace string, name string) error { ret := _m.Called(ctx, repoType, namespace, name) diff --git a/cmd/csghub-server/cmd/start/server.go b/cmd/csghub-server/cmd/start/server.go index 1f3cf55ba..e66787af7 100644 --- a/cmd/csghub-server/cmd/start/server.go +++ b/cmd/csghub-server/cmd/start/server.go @@ -67,6 +67,7 @@ var serverCmd = &cobra.Command{ if err != nil { return fmt.Errorf("fail to initialize message queue, %w", err) } + s3Internal := len(cfg.S3.InternalEndpoint) > 0 err = deploy.Init(common.DeployConfig{ ImageBuilderURL: cfg.Space.BuilderEndpoint, ImageRunnerURL: cfg.Space.RunnerEndpoint, @@ -76,6 +77,7 @@ var serverCmd = &cobra.Command{ ModelDeployTimeoutInMin: cfg.Model.DeployTimeoutInMin, ModelDownloadEndpoint: cfg.Model.DownloadEndpoint, PublicRootDomain: cfg.Space.PublicRootDomain, + S3Internal: s3Internal, IsMasterHost: cfg.IsMasterHost, }) if err != nil { diff --git a/common/config/config.go b/common/config/config.go index e631e2d58..7d85a6428 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -16,11 +16,8 @@ type Config struct { EnableSwagger bool `env:"STARHUB_SERVER_ENABLE_SWAGGER, default=false"` APIToken string `env:"STARHUB_SERVER_API_TOKEN, default=0c11e6e4f2054444374ba3f0b70de4145935a7312289d404814cd5907c6aa93cc65cd35dbf94e04c13a3dedbf51f1694de84240c8acb7238b54a2c3ac8e87c59"` // enable if you want to acess csghub through https, especially for space rproxy - EnableHTTPS bool `env:"STARHUB_SERVER_ENABLE_HTTPS, default=false"` - //the api key to call lbs service, like tencent map or gaode map - LBSServiceKey string `env:"STARHUB_SERVER_LBS_SERVICE_KEY, default=123456"` - //the cdn domain for different city - CityToCdnDomain map[string]string `env:"STARHUB_SERVER_CITY_TO_CDN_DOMAIN, default="` + EnableHTTPS bool `env:"STARHUB_SERVER_ENABLE_HTTPS, default=false"` + DocsHost string `env:"STARHUB_SERVER_SERVER_DOCS_HOST, default=http://localhost:6636"` //the master host IsMasterHost bool `env:"STARHUB_SERVER_IS_MASTER_HOST, default=true"` diff --git a/common/tests/stores.go b/common/tests/stores.go index 463802746..9a548bec9 100644 --- a/common/tests/stores.go +++ b/common/tests/stores.go @@ -42,13 +42,13 @@ func NewMockStores(t interface { mock.TestingT }) *MockStores { return &MockStores{ - User: mockdb.NewMockUserStore(t), - UserLikes: mockdb.NewMockUserLikesStore(t), - Repo: mockdb.NewMockRepoStore(t), - RepoRelation: mockdb.NewMockRepoRelationsStore(t), - Model: mockdb.NewMockModelStore(t), - SpaceResource: mockdb.NewMockSpaceResourceStore(t), - Tag: mockdb.NewMockTagStore(t), + User: mockdb.NewMockUserStore(t), + UserLikes: mockdb.NewMockUserLikesStore(t), + Repo: mockdb.NewMockRepoStore(t), + RepoRelation: mockdb.NewMockRepoRelationsStore(t), + Model: mockdb.NewMockModelStore(t), + SpaceResource: mockdb.NewMockSpaceResourceStore(t), + //Tag: mockdb.NewMockTagStore(t), Dataset: mockdb.NewMockDatasetStore(t), PromptConversation: mockdb.NewMockPromptConversationStore(t), PromptPrefix: mockdb.NewMockPromptPrefixStore(t), diff --git a/component/evaluation_test.go b/component/evaluation_test.go index 83cafb223..a1620f146 100644 --- a/component/evaluation_test.go +++ b/component/evaluation_test.go @@ -3,7 +3,6 @@ package component import ( "context" "encoding/json" - "strconv" "testing" "github.com/stretchr/testify/require" @@ -125,23 +124,12 @@ func TestEvaluationComponent_CreateEvaluation(t *testing.T) { ID: 1, FrameImage: "lm-evaluation-harness:0.4.6", }, nil) - ac.EXPECT().QueryBalanceByUserIDInternal(ctx, "test").Return(&database.AccountUser{ - Balance: 123.4, - }, nil) resource, err := json.Marshal(req2.Hardware) require.Nil(t, err) stores.SpaceResourceMock().EXPECT().FindByID(ctx, int64(1)).Return(&database.SpaceResource{ ID: 1, Resources: string(resource), }, nil) - ac.EXPECT().QueryPricesBySKUType("", types.AcctPriceListReq{ - SkuType: types.SKUCSGHub, - SkuKind: strconv.Itoa(int(types.SKUPayAsYouGo)), - ResourceID: strconv.FormatInt(int64(1), 10), - }).Return(&database.PriceResp{ - Total: 100, - Prices: []database.AccountPrice{{SkuPrice: 10}}, - }, nil) deployerMock.EXPECT().SubmitEvaluation(ctx, req2).Return(&types.ArgoWorkFlowRes{ ID: 1, TaskName: "test", diff --git a/component/model_test.go b/component/model_test.go index f90bc1d54..3cea82d90 100644 --- a/component/model_test.go +++ b/component/model_test.go @@ -580,9 +580,9 @@ func TestModelComponent_SetRuntimeFrameworkModes(t *testing.T) { }, nil, ) rftags := []*database.Tag{{Name: "t1"}, {Name: "t2"}} - mc.mocks.stores.TagMock().EXPECT().GetTagsByScopeAndCategories( - ctx, database.TagScope("model"), []string{"runtime_framework", "resource"}, - ).Return(rftags, nil) + // mc.mocks.stores.TagMock().EXPECT().GetTagsByScopeAndCategories( + // ctx, database.TagScope("model"), []string{"runtime_framework", "resource"}, + // ).Return(rftags, nil) mc.mocks.stores.RepoRuntimeFrameworkMock().EXPECT().GetByIDsAndType( ctx, int64(1), int64(1), 1, diff --git a/component/user.go b/component/user.go index 16a8aab28..86cc31e91 100644 --- a/component/user.go +++ b/component/user.go @@ -88,8 +88,12 @@ type userComponentImpl struct { spaceStore database.SpaceStore namespaceStore database.NamespaceStore gitServer gitserver.GitServer + spaceComponent SpaceComponent + repoComponent RepoComponent + deployer deploy.Deployer userLikeStore database.UserLikesStore repoStore database.RepoStore + deploy database.DeployTaskStore collectionStore database.CollectionStore accountingComponent AccountingComponent // srs database.SpaceResourceStore From 5fe505f28b8cdeea375a8778ddb32911255229b9 Mon Sep 17 00:00:00 2001 From: Xiang Zhen Gan Date: Fri, 6 Dec 2024 08:53:49 +0800 Subject: [PATCH 08/15] add readme --- docker/evaluation/README.md | 66 +++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 docker/evaluation/README.md diff --git a/docker/evaluation/README.md b/docker/evaluation/README.md new file mode 100644 index 000000000..99ddde1ff --- /dev/null +++ b/docker/evaluation/README.md @@ -0,0 +1,66 @@ +# CSGHUB Nginx Images Building + +## Login Container Registry +```bash +OPENCSG_ACR="opencsg-registry.cn-beijing.cr.aliyuncs.com" +OPENCSG_ACR_USERNAME="" +OPENCSG_ACR_PASSWORD="" +echo "$OPENCSG_ACR_PASSWORD" | docker login $OPENCSG_ACR -u $OPENCSG_ACR_USERNAME --password-stdin +``` + +## Build Multi-Platform Images +```bash +export BUILDX_NO_DEFAULT_ATTESTATIONS=1 +export IMAGE_TAG=0.3.5 +docker buildx build --platform linux/amd64,linux/arm64 \ + -t ${OPENCSG_ACR}/public/opencompass:${IMAGE_TAG} \ + -t ${OPENCSG_ACR}/public/opencompass:latest \ + -f Dockerfile.opencompass \ + --push . +export IMAGE_TAG=0.4.6 +docker buildx build --platform linux/amd64,linux/arm64 \ + -t ${OPENCSG_ACR}/public/lm-evaluation-harness:${IMAGE_TAG} \ + -t ${OPENCSG_ACR}/public/lm-evaluation-harness:latest \ + -f Dockerfile.lm-evaluation-harness \ + --push . +``` +*The above command will create `linux/amd64` and `linux/arm64` images with the tags `${IMAGE_TAG}` and `latest` at the same time.* + +## Test the opencompass Image +```bash +docker run \ + -e ACCESS_TOKEN=xxxx \ + -e MODEL_ID="OpenCSG/csg-wukong-1B" \ + -e DATASET_IDS="xzgan/hellaswag" \ + -e HF_ENDPOINT=https://hub.opencsg.com \ + -e ASCEND_VISIBLE_DEVICES=7 \ + -e S3_ACCESS_ID="xxxx" \ + -e S3_ACCESS_SECRET="xxxx" \ + -e S3_BUCKET="xxxxx" \ + -e S3_ENDPOINT="xxxxx" \ + -e S3_SSL_ENABLED="true" \ + ${OPENCSG_ACR}/public/opencompass:${IMAGE_TAG} +``` + +## Test the lm-evaluation-harness Image +```bash +export IMAGE_TAG=0.4.6 +docker run \ + --gpus device=7 \ + -e ACCESS_TOKEN=xxxx \ + -e MODEL_ID="OpenCSG/csg-wukong-1B" \ + -e DATASET_IDS="Rowan/hellaswag" \ + -e HF_ENDPOINT=https://hub.opencsg.com\ + -e S3_ACCESS_ID="xxx" \ + -e S3_ACCESS_SECRET="xxx" \ + -e S3_BUCKET="xxx" \ + -e S3_ENDPOINT="xxx" \ + -e S3_SSL_ENABLED="true" \ + ${OPENCSG_ACR}/public/lm-evaluation-harness:${IMAGE_TAG} +``` + +## inference image name, version and cuda version +| Latest Image | Version | CUDA Version | +| --- | --- | --- | +| opencompass | 0.3.5 | 12.4 | +| lm-evaluation-harness | 0.4.6 | 12.4 | From 4952e7cc85cbb150ab89cf9726ce0952a37bedd5 Mon Sep 17 00:00:00 2001 From: Xiang Zhen Gan Date: Fri, 6 Dec 2024 08:56:16 +0800 Subject: [PATCH 09/15] update doc --- docs/docs.go | 15 +++++++++++++++ docs/swagger.json | 15 +++++++++++++++ docs/swagger.yaml | 10 ++++++++++ 3 files changed, 40 insertions(+) diff --git a/docs/docs.go b/docs/docs.go index 96566d3e2..3b75dac49 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -15363,6 +15363,9 @@ const docTemplate = `{ "id": { "type": "integer" }, + "last_updated_at": { + "type": "string" + }, "repository": { "$ref": "#/definitions/database.Repository" }, @@ -15485,6 +15488,9 @@ const docTemplate = `{ "database.Namespace": { "type": "object", "properties": { + "created_at": { + "type": "string" + }, "id": { "type": "integer" }, @@ -15497,6 +15503,9 @@ const docTemplate = `{ "path": { "type": "string" }, + "updated_at": { + "type": "string" + }, "user": { "$ref": "#/definitions/database.User" }, @@ -17532,12 +17541,18 @@ const docTemplate = `{ "readme": { "type": "string" }, + "recom_op_weight": { + "type": "integer" + }, "repository": { "$ref": "#/definitions/types.Repository" }, "repository_id": { "type": "integer" }, + "sensitive_check_status": { + "type": "string" + }, "source": { "$ref": "#/definitions/types.RepositorySource" }, diff --git a/docs/swagger.json b/docs/swagger.json index a588ae3f9..c5dfdef4b 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -15352,6 +15352,9 @@ "id": { "type": "integer" }, + "last_updated_at": { + "type": "string" + }, "repository": { "$ref": "#/definitions/database.Repository" }, @@ -15474,6 +15477,9 @@ "database.Namespace": { "type": "object", "properties": { + "created_at": { + "type": "string" + }, "id": { "type": "integer" }, @@ -15486,6 +15492,9 @@ "path": { "type": "string" }, + "updated_at": { + "type": "string" + }, "user": { "$ref": "#/definitions/database.User" }, @@ -17521,12 +17530,18 @@ "readme": { "type": "string" }, + "recom_op_weight": { + "type": "integer" + }, "repository": { "$ref": "#/definitions/types.Repository" }, "repository_id": { "type": "integer" }, + "sensitive_check_status": { + "type": "string" + }, "source": { "$ref": "#/definitions/types.RepositorySource" }, diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 6ba7c8aea..c8f611fc4 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -183,6 +183,8 @@ definitions: type: string id: type: integer + last_updated_at: + type: string repository: $ref: '#/definitions/database.Repository' repository_id: @@ -263,6 +265,8 @@ definitions: type: object database.Namespace: properties: + created_at: + type: string id: type: integer mirrored: @@ -271,6 +275,8 @@ definitions: $ref: '#/definitions/database.NamespaceType' path: type: string + updated_at: + type: string user: $ref: '#/definitions/database.User' user_id: @@ -1649,10 +1655,14 @@ definitions: type: boolean readme: type: string + recom_op_weight: + type: integer repository: $ref: '#/definitions/types.Repository' repository_id: type: integer + sensitive_check_status: + type: string source: $ref: '#/definitions/types.RepositorySource' status: From 164fe00aad29690dfbb3eb7936e4f1e2df422283 Mon Sep 17 00:00:00 2001 From: Xiang Zhen Gan Date: Fri, 6 Dec 2024 09:19:34 +0800 Subject: [PATCH 10/15] update lint and test --- builder/deploy/imagerunner/remote_runner.go | 1 + component/model_test.go | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/builder/deploy/imagerunner/remote_runner.go b/builder/deploy/imagerunner/remote_runner.go index bb2b2f926..fca1e22bb 100644 --- a/builder/deploy/imagerunner/remote_runner.go +++ b/builder/deploy/imagerunner/remote_runner.go @@ -342,6 +342,7 @@ func (h *RemoteRunner) ListWorkFlows(ctx context.Context, username string, per, if err != nil { return nil, fmt.Errorf("failed to list evaluation jobs, %w", err) } + defer response.Body.Close() var res types.ArgoWorkFlowListRes if err := json.NewDecoder(response.Body).Decode(&res); err != nil { return nil, err diff --git a/component/model_test.go b/component/model_test.go index 3cea82d90..3fcab707a 100644 --- a/component/model_test.go +++ b/component/model_test.go @@ -232,9 +232,10 @@ func TestModelComponent_Show(t *testing.T) { HTTPCloneURL: "https://foo.com/s/foo/bar.git", SSHCloneURL: "test@127.0.0.1:s/foo/bar.git", }, - EnableInference: true, - EnableFinetune: true, - WidgetType: types.ModelWidgetTypeGeneration, + EnableInference: true, + EnableFinetune: true, + EnableEvaluation: true, + WidgetType: types.ModelWidgetTypeGeneration, }, model) } From f20705af8ef46ac95569b4c9f750ef81fad4f4a9 Mon Sep 17 00:00:00 2001 From: Xiang Zhen Gan Date: Fri, 6 Dec 2024 09:58:52 +0800 Subject: [PATCH 11/15] fix ut --- common/tests/stores.go | 14 +++++++------- component/model_test.go | 17 +++++++---------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/common/tests/stores.go b/common/tests/stores.go index 9a548bec9..463802746 100644 --- a/common/tests/stores.go +++ b/common/tests/stores.go @@ -42,13 +42,13 @@ func NewMockStores(t interface { mock.TestingT }) *MockStores { return &MockStores{ - User: mockdb.NewMockUserStore(t), - UserLikes: mockdb.NewMockUserLikesStore(t), - Repo: mockdb.NewMockRepoStore(t), - RepoRelation: mockdb.NewMockRepoRelationsStore(t), - Model: mockdb.NewMockModelStore(t), - SpaceResource: mockdb.NewMockSpaceResourceStore(t), - //Tag: mockdb.NewMockTagStore(t), + User: mockdb.NewMockUserStore(t), + UserLikes: mockdb.NewMockUserLikesStore(t), + Repo: mockdb.NewMockRepoStore(t), + RepoRelation: mockdb.NewMockRepoRelationsStore(t), + Model: mockdb.NewMockModelStore(t), + SpaceResource: mockdb.NewMockSpaceResourceStore(t), + Tag: mockdb.NewMockTagStore(t), Dataset: mockdb.NewMockDatasetStore(t), PromptConversation: mockdb.NewMockPromptConversationStore(t), PromptPrefix: mockdb.NewMockPromptPrefixStore(t), diff --git a/component/model_test.go b/component/model_test.go index 3fcab707a..ea5571318 100644 --- a/component/model_test.go +++ b/component/model_test.go @@ -572,25 +572,22 @@ func TestModelComponent_SetRuntimeFrameworkModes(t *testing.T) { mc := initializeTestModelComponent(ctx, t) mc.mocks.stores.RuntimeFrameworkMock().EXPECT().FindByID(ctx, int64(1)).Return( - &database.RuntimeFramework{}, nil, + &database.RuntimeFramework{ + ID: 1, + }, nil, ) mc.mocks.stores.ModelMock().EXPECT().ListByPath(ctx, []string{"a", "b"}).Return( []database.Model{ {RepositoryID: 1, Repository: &database.Repository{ID: 1, Path: "m1/foo"}}, - {RepositoryID: 2, Repository: &database.Repository{ID: 2, Path: "m2/foo"}}, }, nil, ) rftags := []*database.Tag{{Name: "t1"}, {Name: "t2"}} - // mc.mocks.stores.TagMock().EXPECT().GetTagsByScopeAndCategories( - // ctx, database.TagScope("model"), []string{"runtime_framework", "resource"}, - // ).Return(rftags, nil) - + mc.mocks.stores.TagMock().EXPECT().GetTagsByScopeAndCategories( + ctx, database.TagScope("model"), []string{"runtime_framework", "resource"}, + ).Return(rftags, nil) mc.mocks.stores.RepoRuntimeFrameworkMock().EXPECT().GetByIDsAndType( ctx, int64(1), int64(1), 1, - ).Return([]database.RepositoriesRuntimeFramework{}, nil) - mc.mocks.stores.RepoRuntimeFrameworkMock().EXPECT().GetByIDsAndType( - ctx, int64(1), int64(2), 1, - ).Return([]database.RepositoriesRuntimeFramework{{}}, nil) + ).Return(nil, nil) mc.mocks.stores.RepoRuntimeFrameworkMock().EXPECT().Add(ctx, int64(1), int64(1), 1).Return(nil) mc.mocks.components.runtimeArchitecture.EXPECT().AddRuntimeFrameworkTag( From f1c1b6378532a3dc982b9626d3033efe18be2072 Mon Sep 17 00:00:00 2001 From: Xiang Zhen Gan Date: Fri, 6 Dec 2024 10:30:59 +0800 Subject: [PATCH 12/15] update repo --- docker/evaluation/Dockerfile.lm-evaluation-harness | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/evaluation/Dockerfile.lm-evaluation-harness b/docker/evaluation/Dockerfile.lm-evaluation-harness index fe32dc6a8..7ef795d8e 100644 --- a/docker/evaluation/Dockerfile.lm-evaluation-harness +++ b/docker/evaluation/Dockerfile.lm-evaluation-harness @@ -5,7 +5,7 @@ RUN pip install --upgrade pip && pip config set global.index-url https://pypi.tu && pip install --no-cache-dir accelerate transformers==4.46.3 \ minio oss2 langdetect huggingface_hub openpyxl WORKDIR /workspace/ -RUN git clone --depth 1 https://gitee.com/xzgan/lm-evaluation-harness.git --branch v0.4.6 --single-branch && \ +RUN git clone --depth 1 https://github.com/EleutherAI/lm-evaluation-harness.git --branch v0.4.6 --single-branch && \ cd lm-evaluation-harness && pip install setuptools --upgrade --no-cache-dir -e \ ".[ifeval,math,multilingual,sentencepiece]" COPY ./lm-evaluation-harness/ /etc/csghub/ From bff29efd60f46f3833c5d33f1cdeeb9c8042467b Mon Sep 17 00:00:00 2001 From: Xiang Zhen Gan Date: Fri, 6 Dec 2024 10:48:03 +0800 Subject: [PATCH 13/15] rollback test --- builder/store/database/repository_test.go | 733 +++++++++++++++++++++- 1 file changed, 731 insertions(+), 2 deletions(-) diff --git a/builder/store/database/repository_test.go b/builder/store/database/repository_test.go index 1cb209edb..45d8db5c8 100644 --- a/builder/store/database/repository_test.go +++ b/builder/store/database/repository_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "testing" + "time" "github.com/google/uuid" "github.com/stretchr/testify/require" @@ -12,7 +13,262 @@ import ( "opencsg.com/csghub-server/common/types" ) -func TestRepoStore_PublicToUser(t *testing.T) { +func TestRepoStore_CRUD(t *testing.T) { + db := tests.InitTestDB() + defer db.Close() + ctx := context.TODO() + + store := database.NewRepoStoreWithDB(db) + + _, err := store.CreateRepo(ctx, database.Repository{ + Name: "repo1", + UserID: 123, + GitPath: "foos_u/bar", + }) + require.Nil(t, err) + + rp := &database.Repository{} + err = db.Core.NewSelect().Model(rp).Where("user_id=?", 123).Scan(ctx) + require.Nil(t, err) + require.Equal(t, "repo1", rp.Name) + + rp, err = store.FindById(ctx, rp.ID) + require.Nil(t, err) + require.Equal(t, "repo1", rp.Name) + + rps, err := store.FindByIds(ctx, []int64{rp.ID}) + require.Nil(t, err) + require.Equal(t, 1, len(rps)) + require.Equal(t, "repo1", rps[0].Name) + + rps, err = store.All(ctx) + require.Nil(t, err) + require.Equal(t, 1, len(rps)) + require.Equal(t, "repo1", rps[0].Name) + + rp, err = store.Find(ctx, "u", "foO", "bAr") + require.Nil(t, err) + require.Equal(t, "repo1", rp.Name) + + exist, err := store.Exists(ctx, "foO", "u", "bar") + require.Nil(t, err) + require.True(t, exist) + + rp, err = store.FindByPath(ctx, "foO", "u", "bAr") + require.Nil(t, err) + require.Equal(t, "repo1", rp.Name) + + rp, err = store.FindByGitPath(ctx, "foos_u/bAr") + require.Nil(t, err) + require.Equal(t, "repo1", rp.Name) + + rps, err = store.FindByGitPaths(ctx, []string{"foos_u/bAr"}) + require.Nil(t, err) + require.Equal(t, 1, len(rps)) + require.Equal(t, "repo1", rps[0].Name) + + rpsp, err := store.ByUser(ctx, 123) + require.Nil(t, err) + require.Equal(t, 1, len(rpsp)) + require.Equal(t, "repo1", rpsp[0].Name) + rpsp, err = store.ByUser(ctx, 125) + require.Nil(t, err) + require.Equal(t, 0, len(rpsp)) + + rpn := *rp + rpn.Name = "repo1-new" + _, err = store.UpdateRepo(ctx, rpn) + require.Nil(t, err) + err = db.Core.NewSelect().Model(rp).Where("user_id=?", 123).Scan(ctx) + require.Nil(t, err) + require.Equal(t, "repo1-new", rp.Name) + + err = store.DeleteRepo(ctx, database.Repository{ + ID: rp.ID, + }) + require.Nil(t, err) + err = db.Core.NewSelect().Model(rp).Where("user_id=?", 123).Scan(ctx) + require.NotNil(t, err) + + _, err = store.UpdateOrCreateRepo(ctx, database.Repository{ + Name: "repo3", + UserID: 231, + Path: "bars_u/bar", + RepositoryType: types.CodeRepo, + }) + require.Nil(t, err) + rp = &database.Repository{} + err = db.Core.NewSelect().Model(rp).Where("user_id=?", 231).Scan(ctx) + require.Nil(t, err) + require.Equal(t, "repo3", rp.Name) + + _, err = store.UpdateOrCreateRepo(ctx, database.Repository{ + Name: "repo3n", + UserID: 231, + Path: "bars_u/bar", + RepositoryType: types.CodeRepo, + }) + require.Nil(t, err) + rp = &database.Repository{} + err = db.Core.NewSelect().Model(rp).Where("user_id=?", 231).Scan(ctx) + require.Nil(t, err) + require.Equal(t, "repo3n", rp.Name) + + cnt, err := store.CountByRepoType(ctx, types.CodeRepo) + require.Nil(t, err) + require.Equal(t, 1, cnt) + +} + +func TestRepoStore_UpdateRepoFileDownloads(t *testing.T) { + db := tests.InitTestDB() + defer db.Close() + ctx := context.TODO() + + store := database.NewRepoStoreWithDB(db) + + repo, err := store.CreateRepo(ctx, database.Repository{ + Name: "repo1", + UserID: 123, + GitPath: "foos_u/bar", + }) + require.Nil(t, err) + + dt := time.Date(2022, 12, 11, 0, 0, 0, 0, time.UTC) + dw := &database.RepositoryDownload{} + // create + err = store.UpdateRepoFileDownloads(ctx, repo, dt, 111) + require.Nil(t, err) + err = db.Core.NewSelect().Model(dw).Where("repository_id = ?", repo.ID).Scan(ctx) + require.Nil(t, err) + require.Equal(t, 111, int(dw.ClickDownloadCount)) + err = db.Core.NewSelect().Model(repo).Where("user_id=?", 123).Scan(ctx) + require.Nil(t, err) + require.Equal(t, 111, int(repo.DownloadCount)) + + // update + err = store.UpdateRepoFileDownloads(ctx, repo, dt, 5) + require.Nil(t, err) + err = db.Core.NewSelect().Model(dw).Where("repository_id = ?", repo.ID).Scan(ctx) + require.Nil(t, err) + require.Equal(t, 116, int(dw.ClickDownloadCount)) + err = db.Core.NewSelect().Model(repo).Where("user_id=?", 123).Scan(ctx) + require.Nil(t, err) + require.Equal(t, 116, int(repo.DownloadCount)) + +} + +func TestRepoStore_UpdateRepoCloneDownloads(t *testing.T) { + db := tests.InitTestDB() + defer db.Close() + ctx := context.TODO() + + store := database.NewRepoStoreWithDB(db) + + repo, err := store.CreateRepo(ctx, database.Repository{ + Name: "repo1", + UserID: 123, + GitPath: "foos_u/bar", + }) + require.Nil(t, err) + + dt := time.Date(2022, 12, 11, 0, 0, 0, 0, time.UTC) + dw := &database.RepositoryDownload{} + // create + err = store.UpdateRepoCloneDownloads(ctx, repo, dt, 111) + require.Nil(t, err) + err = db.Core.NewSelect().Model(dw).Where("repository_id = ?", repo.ID).Scan(ctx) + require.Nil(t, err) + require.Equal(t, 111, int(dw.CloneCount)) + err = db.Core.NewSelect().Model(repo).Where("user_id=?", 123).Scan(ctx) + require.Nil(t, err) + require.Equal(t, 111, int(repo.DownloadCount)) + + // update + err = store.UpdateRepoCloneDownloads(ctx, repo, dt, 5) + require.Nil(t, err) + err = db.Core.NewSelect().Model(dw).Where("repository_id = ?", repo.ID).Scan(ctx) + require.Nil(t, err) + // clone count will be override, not add to previous + require.Equal(t, 5, int(dw.CloneCount)) + err = db.Core.NewSelect().Model(repo).Where("user_id=?", 123).Scan(ctx) + require.Nil(t, err) + require.Equal(t, 5, int(repo.DownloadCount)) + +} + +func TestRepoStore_Tags(t *testing.T) { + db := tests.InitTestDB() + defer db.Close() + ctx := context.TODO() + + store := database.NewRepoStoreWithDB(db) + repo, err := store.CreateRepo(ctx, database.Repository{ + Name: "repo1", + UserID: 123, + }) + require.Nil(t, err) + + tag := &database.Tag{ + Name: "tg", + Category: "foo", + } + _, err = db.Core.NewInsert().Model(tag).Exec(ctx, tag) + require.Nil(t, err) + tag2 := &database.Tag{ + Name: "tg2", + Category: "bar", + } + _, err = db.Core.NewInsert().Model(tag2).Exec(ctx, tag2) + require.Nil(t, err) + + rtags := []database.RepositoryTag{ + {TagID: tag.ID, RepositoryID: repo.ID, Count: 1}, + {TagID: tag2.ID, RepositoryID: repo.ID, Count: 1}, + } + err = store.BatchCreateRepoTags(ctx, rtags) + require.Nil(t, err) + + tags, err := store.Tags(ctx, repo.ID) + require.Nil(t, err) + require.Equal(t, 2, len(tags)) + require.Equal(t, tags[0].Name, "tg") + require.Equal(t, tags[1].Name, "tg2") + + tags, err = store.TagsWithCategory(ctx, repo.ID, "foo") + require.Nil(t, err) + require.Equal(t, 1, len(tags)) + require.Equal(t, tags[0].Name, "tg") + + ids, err := store.TagIDs(ctx, repo.ID, "foo") + require.Nil(t, err) + require.Equal(t, []int64{tag.ID}, ids) +} + +func TestRepoStore_SetUpdateTimeByPath(t *testing.T) { + db := tests.InitTestDB() + defer db.Close() + ctx := context.TODO() + + store := database.NewRepoStoreWithDB(db) + repo, err := store.CreateRepo(ctx, database.Repository{ + Name: "repo1", + UserID: 123, + GitPath: "foos_u/bar", + }) + require.Nil(t, err) + + dt := time.Date(2022, 12, 6, 1, 2, 0, 0, time.UTC) + err = store.SetUpdateTimeByPath(ctx, "foo", "u", "bar", dt) + require.Nil(t, err) + + err = db.Core.NewSelect().Model(repo).WherePK().Scan(ctx) + require.Nil(t, err) + require.Equal(t, dt, repo.UpdatedAt) + +} + +func TestRepoStore_PublicToUserSimple(t *testing.T) { db := tests.InitTestDB() defer db.Close() ctx := context.TODO() @@ -84,5 +340,478 @@ func TestRepoStore_PublicToUser(t *testing.T) { // case 2: two tag repos, _, err = rs.PublicToUser(ctx, repo.RepositoryType, []int64{1}, filter, 20, 1) require.Nil(t, err) - require.NotNil(t, repos) + require.Nil(t, repos) +} + +func TestRepoStore_PublicToUser(t *testing.T) { + + cases := []struct { + admin bool + repoType types.RepositoryType + source string + search string + tags []types.TagReq + sort string + expected []string + }{ + { + admin: false, repoType: types.CodeRepo, + expected: []string{"rp1", "rp2", "rp4", "rp5", "rp6"}, + }, + { + admin: false, repoType: types.CodeRepo, source: string(types.HuggingfaceSource), + expected: []string{"rp2"}, + }, + { + admin: true, repoType: types.CodeRepo, + expected: []string{"rp1", "rp2", "rp4", "rp5", "rp6"}, + }, + { + admin: false, repoType: types.CodeRepo, search: "rp4", + expected: []string{"rp4", "rp6"}, + }, + { + admin: false, repoType: types.CodeRepo, sort: "most_download", + expected: []string{"rp1", "rp2", "rp4", "rp5", "rp6"}, + }, + { + admin: false, repoType: types.CodeRepo, sort: "trending", + expected: []string{"rp1", "rp2", "rp4", "rp5", "rp6"}, + }, + { + admin: false, repoType: types.CodeRepo, tags: []types.TagReq{{Name: "foo"}}, + expected: []string{"rp4"}, + }, + } + + for _, c := range cases { + t.Run(fmt.Sprintf("%+v", c), func(t *testing.T) { + db := tests.InitTestDB() + defer db.Close() + ctx := context.TODO() + + store := database.NewRepoStoreWithDB(db) + + repos := []*database.Repository{ + { + Name: "rp1", Path: "rp1", UserID: 123, RepositoryType: types.CodeRepo, + DownloadCount: 10, + }, + { + Name: "rp2", Path: "rp2", UserID: 123, RepositoryType: types.CodeRepo, + Private: true, Source: types.HuggingfaceSource, + DownloadCount: 10, + }, + { + Name: "rp3", Path: "rp3", UserID: 456, RepositoryType: types.CodeRepo, + Private: true, + DownloadCount: 15, + }, + { + Name: "rp4", Path: "rp4", UserID: 456, RepositoryType: types.CodeRepo, + Private: false, Tags: []database.Tag{{Name: "foo"}}, + DownloadCount: 10, + }, + { + Name: "rp5", Path: "rp5", UserID: 789, RepositoryType: types.CodeRepo, + Private: false, Tags: []database.Tag{{Name: "bar"}}, + DownloadCount: 10, + }, + { + Name: "rp6", Path: "rp6", UserID: 789, RepositoryType: types.CodeRepo, + Private: false, Description: "rp4desc", + DownloadCount: 10, + }, + } + + for _, repo := range repos { + repo.GitPath = repo.Path + rn, err := store.CreateRepo(ctx, *repo) + require.Nil(t, err) + for _, tag := range repo.Tags { + _, err = db.Core.NewInsert().Model(&tag).Exec(ctx, &tag) + require.Nil(t, err) + rtags := []database.RepositoryTag{ + {TagID: tag.ID, RepositoryID: rn.ID, Count: 1}, + } + err = store.BatchCreateRepoTags(ctx, rtags) + require.Nil(t, err) + } + } + + rs, count, err := store.PublicToUser(ctx, c.repoType, []int64{123}, &types.RepoFilter{ + Tags: c.tags, + Sort: c.sort, + Search: c.search, + Source: c.source, + }, 10, 1) + require.Nil(t, err) + names := []string{} + for _, r := range rs { + names = append(names, r.Name) + } + require.Equal(t, len(c.expected), count) + require.Equal(t, c.expected, names) + + }) + } +} + +func TestRepoStore_IsMirrorRepo(t *testing.T) { + db := tests.InitTestDB() + defer db.Close() + ctx := context.TODO() + + store := database.NewRepoStoreWithDB(db) + rn, err := store.CreateRepo(ctx, database.Repository{ + GitPath: "codes_ns/n", + }) + require.Nil(t, err) + mi := &database.Mirror{RepositoryID: rn.ID} + _, err = db.Core.NewInsert().Model(mi).Exec(ctx) + require.Nil(t, err) + m, err := store.IsMirrorRepo(ctx, types.CodeRepo, "ns", "n") + require.Nil(t, err) + require.True(t, m) + m, err = store.IsMirrorRepo(ctx, types.CodeRepo, "ns", "n2") + require.NotNil(t, err) + require.False(t, m) + +} + +func TestRepoStore_WithMirror(t *testing.T) { + db := tests.InitTestDB() + defer db.Close() + ctx := context.TODO() + + store := database.NewRepoStoreWithDB(db) + rn, err := store.CreateRepo(ctx, database.Repository{ + GitPath: "codes_ns/n", + }) + require.Nil(t, err) + mi := &database.Mirror{RepositoryID: rn.ID} + _, err = db.Core.NewInsert().Model(mi).Exec(ctx) + require.Nil(t, err) + _, err = store.CreateRepo(ctx, database.Repository{ + GitPath: "codes_ns2/n", + Path: "zzz", + }) + require.Nil(t, err) + rs, count, err := store.WithMirror(ctx, 10, 1) + require.Nil(t, err) + require.Equal(t, 1, count) + require.Equal(t, "codes_ns/n", rs[0].GitPath) + +} + +func TestRepoStore_ListRepoPublicToUserByRepoIDs(t *testing.T) { + + cases := []struct { + admin bool + repoType types.RepositoryType + search string + sort string + expected []string + }{ + { + admin: false, repoType: types.CodeRepo, + expected: []string{"rp1", "rp2", "rp4", "rp5", "rp6"}, + }, + { + admin: true, repoType: types.CodeRepo, + expected: []string{"rp1", "rp2", "rp4", "rp5", "rp6"}, + }, + { + admin: false, repoType: types.CodeRepo, search: "rp4", + expected: []string{"rp4", "rp6"}, + }, + { + admin: false, repoType: types.CodeRepo, sort: "most_download", + expected: []string{"rp1", "rp2", "rp4", "rp5", "rp6"}, + }, + { + admin: false, repoType: types.CodeRepo, sort: "trending", + expected: []string{"rp1", "rp2", "rp4", "rp5", "rp6"}, + }, + } + + for _, c := range cases { + t.Run(fmt.Sprintf("%+v", c), func(t *testing.T) { + db := tests.InitTestDB() + defer db.Close() + ctx := context.TODO() + + store := database.NewRepoStoreWithDB(db) + + repos := []*database.Repository{ + { + Name: "rp1", Path: "rp1", UserID: 123, RepositoryType: types.CodeRepo, + DownloadCount: 10, + }, + { + Name: "rp2", Path: "rp2", UserID: 123, RepositoryType: types.CodeRepo, + Private: true, + DownloadCount: 10, + }, + { + Name: "rp3", Path: "rp3", UserID: 456, RepositoryType: types.CodeRepo, + Private: true, + DownloadCount: 15, + }, + { + Name: "rp4", Path: "rp4", UserID: 456, RepositoryType: types.CodeRepo, + Private: false, Tags: []database.Tag{{Name: "foo"}}, + DownloadCount: 10, + }, + { + Name: "rp5", Path: "rp5", UserID: 789, RepositoryType: types.CodeRepo, + Private: false, Tags: []database.Tag{{Name: "bar"}}, + DownloadCount: 10, + }, + { + Name: "rp6", Path: "rp6", UserID: 789, RepositoryType: types.CodeRepo, + Private: false, Description: "rp4desc", + DownloadCount: 10, + }, + } + + rids := []int64{} + for _, repo := range repos { + repo.GitPath = repo.Path + rn, err := store.CreateRepo(ctx, *repo) + require.Nil(t, err) + rids = append(rids, rn.ID) + for _, tag := range repo.Tags { + _, err = db.Core.NewInsert().Model(&tag).Exec(ctx, &tag) + require.Nil(t, err) + rtags := []database.RepositoryTag{ + {TagID: tag.ID, RepositoryID: rn.ID, Count: 1}, + } + err = store.BatchCreateRepoTags(ctx, rtags) + require.Nil(t, err) + } + } + + rs, count, err := store.ListRepoPublicToUserByRepoIDs(ctx, c.repoType, 123, c.search, c.sort, 10, 1, rids) + require.Nil(t, err) + names := []string{} + for _, r := range rs { + names = append(names, r.Name) + } + require.Equal(t, len(c.expected), count) + require.Equal(t, c.expected, names) + + }) + } +} + +func TestRepoStore_CleanRelationsByRepoID(t *testing.T) { + db := tests.InitTestDB() + defer db.Close() + ctx := context.TODO() + + store := database.NewRepoStoreWithDB(db) + rn, err := store.CreateRepo(ctx, database.Repository{ + GitPath: "codes_ns/n", + }) + require.Nil(t, err) + + _, err = db.Core.NewInsert().Model(&database.RepositoriesRuntimeFramework{ + RepoID: rn.ID, + }).Exec(ctx) + require.Nil(t, err) + _, err = db.Core.NewInsert().Model(&database.UserLike{ + RepoID: rn.ID, + }).Exec(ctx) + require.Nil(t, err) + count, err := db.Core.NewSelect().Model(&database.RepositoriesRuntimeFramework{}).Where("repo_id=?", rn.ID).Count(ctx) + require.Nil(t, err) + require.Equal(t, 1, count) + count, err = db.Core.NewSelect().Model(&database.UserLike{}).Where("repo_id=?", rn.ID).Count(ctx) + require.Nil(t, err) + require.Equal(t, 1, count) + + err = store.CleanRelationsByRepoID(ctx, rn.ID) + require.Nil(t, err) + + count, err = db.Core.NewSelect().Model(&database.RepositoriesRuntimeFramework{}).Where("repo_id=?", rn.ID).Count(ctx) + require.Nil(t, err) + require.Equal(t, 0, count) + count, err = db.Core.NewSelect().Model(&database.UserLike{}).Where("repo_id=?", rn.ID).Count(ctx) + require.Nil(t, err) + require.Equal(t, 0, count) + +} + +func TestRepoStore_DeleteAllFiles(t *testing.T) { + db := tests.InitTestDB() + defer db.Close() + ctx := context.TODO() + + store := database.NewRepoStoreWithDB(db) + + _, err := db.Core.NewInsert().Model(&database.File{RepositoryID: 123}).Exec(ctx) + require.Nil(t, err) + err = store.DeleteAllFiles(ctx, 123) + require.Nil(t, err) + count, err := db.Core.NewSelect().Model(&database.File{}).Where("repository_id = ?", 123).Count(ctx) + require.Nil(t, err) + require.Equal(t, 0, count) +} + +func TestRepoStore_DeleteAllTags(t *testing.T) { + db := tests.InitTestDB() + defer db.Close() + ctx := context.TODO() + + store := database.NewRepoStoreWithDB(db) + + _, err := db.Core.NewInsert().Model(&database.RepositoryTag{RepositoryID: 123}).Exec(ctx) + require.Nil(t, err) + err = store.DeleteAllTags(ctx, 123) + require.Nil(t, err) + count, err := db.Core.NewSelect().Model(&database.RepositoryTag{}).Where("repository_id = ?", 123).Count(ctx) + require.Nil(t, err) + require.Equal(t, 0, count) +} + +func TestRepoStore_UpdateLicenseByTag(t *testing.T) { + db := tests.InitTestDB() + defer db.Close() + ctx := context.TODO() + + store := database.NewRepoStoreWithDB(db) + rn, err := store.CreateRepo(ctx, database.Repository{ + GitPath: "codes_ns/n", + License: "foo", + }) + require.Nil(t, err) + tag := &database.Tag{ + Category: "license", + Name: "MIT", + } + _, err = db.Core.NewInsert().Model(tag).Exec(ctx, tag) + require.Nil(t, err) + rtags := []database.RepositoryTag{ + {TagID: tag.ID, RepositoryID: rn.ID, Count: 1}, + } + err = store.BatchCreateRepoTags(ctx, rtags) + require.Nil(t, err) + + err = store.UpdateLicenseByTag(ctx, rn.ID) + require.Nil(t, err) + + rn, err = store.FindById(ctx, rn.ID) + require.Nil(t, err) + require.Equal(t, "MIT", rn.License) +} + +func TestRepoStore_GetRepoRuntimeByID(t *testing.T) { + db := tests.InitTestDB() + defer db.Close() + ctx := context.TODO() + + store := database.NewRepoStoreWithDB(db) + repo, err := store.CreateRepo(ctx, database.Repository{ + Path: "codes_ns/n", + License: "foo", + RepositoryType: types.ModelRepo, + }) + require.Nil(t, err) + + rf := &database.RepositoriesRuntimeFramework{ + RepoID: repo.ID, + RuntimeFrameworkID: 999, + } + _, err = db.Core.NewInsert().Model(rf).Exec(ctx, rf) + require.Nil(t, err) + + rs, err := store.GetRepoWithRuntimeByID(ctx, rf.RuntimeFrameworkID, []string{"codes_ns/n"}) + require.Nil(t, err) + require.Equal(t, 1, len(rs)) + + rs, err = store.GetRepoWithoutRuntimeByID(ctx, rf.RuntimeFrameworkID, []string{"codes_ns/n"}) + require.Nil(t, err) + require.Equal(t, 0, len(rs)) + +} + +func TestRepoStore_BatchMethods(t *testing.T) { + db := tests.InitTestDB() + defer db.Close() + ctx := context.TODO() + + store := database.NewRepoStoreWithDB(db) + + repos := []*database.Repository{ + { + Name: "rp1", Path: "rp1", UserID: 123, RepositoryType: types.CodeRepo, + SensitiveCheckStatus: types.SensitiveCheckPending, + Source: types.HuggingfaceSource, + }, + { + Name: "rp2", Path: "rp2", UserID: 123, RepositoryType: types.CodeRepo, + SensitiveCheckStatus: types.SensitiveCheckPending, + Source: types.HuggingfaceSource, + }, + { + Name: "rp3", Path: "rp3", UserID: 123, RepositoryType: types.CodeRepo, + SensitiveCheckStatus: types.SensitiveCheckPending, + Source: types.HuggingfaceSource, + }, + { + Name: "rp4", Path: "rp4", UserID: 456, RepositoryType: types.CodeRepo, + SensitiveCheckStatus: types.SensitiveCheckPass, + Source: types.HuggingfaceSource, + }, + { + Name: "rp5", Path: "rp5", UserID: 456, RepositoryType: types.DatasetRepo, + SensitiveCheckStatus: types.SensitiveCheckPending, + }, + } + + names := func(rs []database.Repository) []string { + names := []string{} + for _, r := range rs { + names = append(names, r.Name) + } + return names + } + + rids := []int64{} + for _, repo := range repos { + repo.GitPath = repo.Path + rn, err := store.CreateRepo(ctx, *repo) + require.Nil(t, err) + rids = append(rids, rn.ID) + } + rs, err := store.BatchGet(ctx, types.CodeRepo, 0, 10) + require.Nil(t, err) + require.Equal(t, len(rs), 3) + require.Equal(t, []string{"rp1", "rp2", "rp3"}, names(rs)) + + rs, err = store.BatchGet(ctx, types.CodeRepo, rids[1], 10) + require.Nil(t, err) + require.Equal(t, len(rs), 1) + require.Equal(t, []string{"rp3"}, names(rs)) + + rs, err = store.BatchGet(ctx, types.CodeRepo, 0, 1) + require.Nil(t, err) + require.Equal(t, len(rs), 1) + require.Equal(t, []string{"rp1"}, names(rs)) + + rs, err = store.FindWithBatch(ctx, 2, 1) + require.Nil(t, err) + require.Equal(t, len(rs), 2) + require.Equal(t, []string{"rp3", "rp2"}, names(rs)) + + rs, err = store.ByUser(ctx, 123) + require.Nil(t, err) + require.Equal(t, len(rs), 3) + require.ElementsMatch(t, []string{"rp1", "rp2", "rp3"}, names(rs)) + + rs, err = store.FindByRepoSourceWithBatch(ctx, types.HuggingfaceSource, 2, 1) + require.Nil(t, err) + require.Equal(t, len(rs), 2) + require.Equal(t, []string{"rp2", "rp1"}, names(rs)) } From 1358739b4cd9e29ac00229672041f6d84c734e38 Mon Sep 17 00:00:00 2001 From: Xiang Zhen Gan Date: Fri, 6 Dec 2024 10:57:29 +0800 Subject: [PATCH 14/15] update --- builder/store/database/repository_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/store/database/repository_test.go b/builder/store/database/repository_test.go index 45d8db5c8..5de852ba7 100644 --- a/builder/store/database/repository_test.go +++ b/builder/store/database/repository_test.go @@ -340,7 +340,7 @@ func TestRepoStore_PublicToUserSimple(t *testing.T) { // case 2: two tag repos, _, err = rs.PublicToUser(ctx, repo.RepositoryType, []int64{1}, filter, 20, 1) require.Nil(t, err) - require.Nil(t, repos) + require.NotNil(t, repos) } func TestRepoStore_PublicToUser(t *testing.T) { From a52c1e05156f2b8c2d3fb2d7b10ef4a083852b12 Mon Sep 17 00:00:00 2001 From: Xiang Zhen Gan Date: Fri, 6 Dec 2024 11:22:33 +0800 Subject: [PATCH 15/15] update readme --- docker/evaluation/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/evaluation/README.md b/docker/evaluation/README.md index 99ddde1ff..cf271903b 100644 --- a/docker/evaluation/README.md +++ b/docker/evaluation/README.md @@ -1,4 +1,4 @@ -# CSGHUB Nginx Images Building +# LLM evalution docker image ## Login Container Registry ```bash @@ -59,7 +59,7 @@ docker run \ ${OPENCSG_ACR}/public/lm-evaluation-harness:${IMAGE_TAG} ``` -## inference image name, version and cuda version +## evaluation image name, version and cuda version | Latest Image | Version | CUDA Version | | --- | --- | --- | | opencompass | 0.3.5 | 12.4 |