diff --git a/api/router/api.go b/api/router/api.go index bb8515076..69f6777e7 100644 --- a/api/router/api.go +++ b/api/router/api.go @@ -183,14 +183,12 @@ func NewRouter(config *config.Config, enableSwagger bool) (*gin.Engine, error) { r.Use(middleware.LocalizedErrorMiddleware()) r.Use(middleware.Authenticator(config)) + useAdvancedMiddleware(r, config) apiGroup := r.Group("/api/v1") versionHandler := handler.NewVersionHandler() apiGroup.GET("/version", versionHandler.Version) - // Admin user get repo path list - apiGroup.GET("/repos", middlewareCollection.Auth.NeedAdmin, repoCommonHandler.GetRepos) - // TODO:use middleware to handle common response // memoryStore := persist.NewMemoryStore(1 * time.Minute) @@ -314,6 +312,7 @@ func NewRouter(config *config.Config, enableSwagger bool) (*gin.Engine, error) { apiGroup.POST("/jwt/token", middlewareCollection.Auth.NeedAPIKey, userProxyHandler.Proxy) apiGroup.GET("/jwt/:token", middlewareCollection.Auth.NeedAPIKey, userProxyHandler.ProxyToApi("/api/v1/jwt/%s", "token")) apiGroup.GET("/users", userProxyHandler.Proxy) + apiGroup.GET("/users/stream-export", middlewareCollection.Auth.NeedAdmin, userProxyHandler.Proxy) // callback callbackCtrl, err := callback.NewGitCallbackHandler(config) @@ -404,6 +403,9 @@ func NewRouter(config *config.Config, enableSwagger bool) (*gin.Engine, error) { adminGroup.POST("/:repo_type/:namespace/:name/change_path", repoCommonHandler.ChangePath) + // Admin user get repo path list + adminGroup.GET("/repos", repoCommonHandler.GetRepos) + // routes for broadcast broadcastHandler, err := handler.NewBroadcastHandler() if err != nil { @@ -745,7 +747,6 @@ func createDatasetRoutes( ) { // gin cache memoryStore := persist.NewMemoryStore(2 * time.Minute) - datasetsGroup := apiGroup.Group("/datasets") // allow access without login datasetsGroup.GET("", dsHandler.Index) @@ -761,7 +762,7 @@ func createDatasetRoutes( datasetsGroup.GET("/:namespace/:name/tags", middleware.MustLogin(), repoCommonHandler.Tags) datasetsGroup.POST("/:namespace/:name/preupload/:revision", middlewareCollection.Auth.NeedPhoneVerified, repoCommonHandler.Preupload) // update tags of a certain category - datasetsGroup.GET("/:namespace/:name/all_files", cache.Cache(memoryStore, time.Minute*2, middleware.CacheRepoInfo()), middleware.MustLogin(), repoCommonHandler.AllFiles) + datasetsGroup.GET("/:namespace/:name/all_files", middleware.MustLogin(), cache.Cache(memoryStore, time.Minute*2, middleware.CacheRepoInfo()), repoCommonHandler.AllFiles) datasetsGroup.POST("/:namespace/:name/tags/:category", middleware.MustLogin(), repoCommonHandler.UpdateTags) datasetsGroup.GET("/:namespace/:name/last_commit", repoCommonHandler.LastCommit) datasetsGroup.GET("/:namespace/:name/commit/:commit_id", middleware.MustLogin(), repoCommonHandler.CommitWithDiff) @@ -774,7 +775,7 @@ func createDatasetRoutes( datasetsGroup.GET("/:namespace/:name/refs/:ref/logs_tree/*path", middleware.MustLogin(), repoCommonHandler.LogsTree) datasetsGroup.GET("/:namespace/:name/commits", middleware.MustLogin(), repoCommonHandler.Commits) datasetsGroup.POST("/:namespace/:name/raw/*file_path", middlewareCollection.Auth.NeedPhoneVerified, repoCommonHandler.CreateFile) - datasetsGroup.GET("/:namespace/:name/raw/*file_path", cache.Cache(memoryStore, time.Minute*2, middleware.CacheRepoInfo()), middleware.MustLogin(), repoCommonHandler.FileRaw) + datasetsGroup.GET("/:namespace/:name/raw/*file_path", middleware.MustLogin(), cache.Cache(memoryStore, time.Minute*2, middleware.CacheRepoInfo()), repoCommonHandler.FileRaw) datasetsGroup.GET("/:namespace/:name/blob/*file_path", repoCommonHandler.FileInfo) datasetsGroup.GET("/:namespace/:name/download/*file_path", middleware.MustLogin(), repoCommonHandler.DownloadFile) datasetsGroup.GET("/:namespace/:name/resolve/*file_path", middleware.MustLogin(), repoCommonHandler.ResolveDownload) @@ -1125,6 +1126,8 @@ func createMappingRoutes( hfDSAPIGroup.POST("/:namespace/:name/paths-info/:ref", middleware.RepoMapping(types.DatasetRepo), hfdsHandler.DatasetPathsInfo) hfDSAPIGroup.GET("/:namespace/:name/tree/:ref/*path_in_repo", middleware.RepoMapping(types.DatasetRepo), hfdsHandler.DatasetTree) hfDSAPIGroup.GET("/:namespace/:name/resolve/:ref/.huggingface.yaml", middleware.RepoMapping(types.DatasetRepo), hfdsHandler.HandleHFYaml) + hfDSAPIGroup.POST("/:namespace/:name/preupload/:revision", middleware.RepoMapping(types.DatasetRepo), repoCommonHandler.PreuploadHF) + hfDSAPIGroup.POST("/:namespace/:name/commit/:revision", middleware.RepoMapping(types.DatasetRepo), repoCommonHandler.CommitFilesHF) } hfSpaceAPIGroup := hfAPIGroup.Group("/spaces") { diff --git a/api/router/api_ce.go b/api/router/api_ce.go index 45a348af2..295df295e 100644 --- a/api/router/api_ce.go +++ b/api/router/api_ce.go @@ -9,6 +9,8 @@ import ( "opencsg.com/csghub-server/common/config" ) +func useAdvancedMiddleware(r *gin.Engine, config *config.Config) {} + func createAdvancedRoutes(apiGroup *gin.RouterGroup, middlewareCollection middleware.MiddlewareCollection, config *config.Config) error { return nil } diff --git a/common/config/config.go b/common/config/config.go index 946e1eacc..49eb383a0 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -31,6 +31,7 @@ type Config struct { CityToCdnDomain map[string]string `env:"STARHUB_SERVER_CITY_TO_CDN_DOMAIN" default:""` UniqueServiceName string `env:"STARHUB_SERVER_UNIQUE_SERVICE_NAME" default:""` ServerFailureRedirectURL string `env:"STARHUB_SERVER_FAIL_REDIRECT_URL" default:"http://localhost:3000/errors/server-error"` + NeedPhoneVerify bool `env:"STARHUB_SERVER_NEED_PHONE_VERIFY" default:"false"` APIServer struct { Port int `env:"STARHUB_SERVER_SERVER_PORT" default:"8080"` @@ -211,11 +212,12 @@ type Config struct { } Accounting struct { - Host string `env:"OPENCSG_ACCOUNTING_SERVER_HOST" default:"http://localhost"` - Port int `env:"OPENCSG_ACCOUNTING_SERVER_PORT" default:"8086"` - ChargingEnable bool `env:"OPENCSG_ACCOUNTING_CHARGING_ENABLE" default:"false"` - SubscriptionCronExpression string `env:"OPENCSG_ACCOUNTING_SUBSCRIPTION_CRON_EXPRESSION" default:"*/5 * * * *"` - ThresholdOfStopDeploy int `env:"OPENCSG_ACCOUNTING_THRESHOLD_OF_STOP_DEPLOY" default:"5000"` + Host string `env:"OPENCSG_ACCOUNTING_SERVER_HOST" default:"http://localhost"` + Port int `env:"OPENCSG_ACCOUNTING_SERVER_PORT" default:"8086"` + ChargingEnable bool `env:"OPENCSG_ACCOUNTING_CHARGING_ENABLE" default:"false"` + SubscriptionCronExpression string `env:"OPENCSG_ACCOUNTING_SUBSCRIPTION_CRON_EXPRESSION" default:"*/5 * * * *"` + ExpiredPresentCronExpression string `env:"OPENCSG_ACCOUNTING_EXPIRED_PRESENT_CRON_EXPRESSION" default:"0 0 * * *"` + ThresholdOfStopDeploy int `env:"OPENCSG_ACCOUNTING_THRESHOLD_OF_STOP_DEPLOY" default:"5000"` } User struct { @@ -334,12 +336,15 @@ type Config struct { IncreaseMultisyncRepoLimitCronExpression string `env:"STARHUB_SERVER_CRON_JOB_INCREASE_MULTISYNC_REPO_LIMIT_CRON_EXPRESSION" default:"0 0 * * *"` MigrateRepoPathCronExpression string `env:"STARHUB_SERVER_CRON_JOB_MIGRATE_REPO_PATH_CRON_EXPRESSION" default:"* 16-20 * * *"` DeletePendingDeletionCronExpression string `env:"STARHUB_SERVER_CRON_JOB_DELETE_PENDING_DELETION_CRON_EXPRESSION" default:"0 16-20 * * *"` + ReleaseInvitationCreditCronExpression string `env:"STARHUB_SERVER_CRON_JOB_RELEASE_INVITATION_CREDIT_CRON_EXPRESSION" default:"0 0 5 * *"` } Agent struct { - AutoHubServiceHost string `env:"OPENCSG_AGENT_AUTOHUB_SERVICE_HOST" default:"http://internal.opencsg-stg.com:8190"` - AgentHubServiceHost string `env:"OPENCSG_AGENT_AGENTHUB_SERVICE_HOST" default:""` - AgentHubServiceToken string `env:"OPENCSG_AGENT_AGENTHUB_SERVICE_TOKEN" default:""` + AutoHubServiceHost string `env:"OPENCSG_AGENT_AUTOHUB_SERVICE_HOST" default:"http://internal.opencsg-stg.com:8190"` + AgentHubServiceHost string `env:"OPENCSG_AGENT_AGENTHUB_SERVICE_HOST" default:""` + AgentHubServiceToken string `env:"OPENCSG_AGENT_AGENTHUB_SERVICE_TOKEN" default:""` + CodeInstanceQuotaPerUser int `env:"STARHUB_SERVER_AGENT_CODE_INSTANCE_QUOTA_PER_USER" default:"5"` + LangflowInstanceQuotaPerUser int `env:"STARHUB_SERVER_AGENT_LANGFLOW_INSTANCE_QUOTA_PER_USER" default:"5"` } DataViewer struct { @@ -419,6 +424,7 @@ type Config struct { DirectMailRegionId string `env:"STARHUB_SERVER_DIRECT_MAIL_REGION_ID" default:"cn-hangzhou"` MailerRechargeAdmin string `env:"STARHUB_SERVER_MAILER_RECHARGE_ADMIN" default:"contact@opencsg.com"` MailerWeeklyRechargesMail string `env:"STARHUB_SERVER_MAILER_WEEKLY_RECHARGES_MAIL" default:"reconcile@opencsg.com"` + EmailInvoiceCreatedReceiver string `env:"STARHUB_SERVER_EMAIL_INVOICE_CREATED_RECEIVER" default:"contact@opencsg.com"` RepoSyncTimezone string `env:"STARHUB_SERVER_REPO_SYNC_TIMEZONE" default:"Asia/Shanghai"` NotificationRetryCount int `env:"STARHUB_SERVER_NOTIFIER_NOTIFICATION_RETRY_COUNT" default:"3"` BroadcastUserPageSize int `env:"STARHUB_SERVER_NOTIFIER_BROADCAST_USER_PAGE_SIZE" default:"100"` @@ -494,7 +500,9 @@ type Config struct { HealthInterval int `env:"STARHUB_SERVER_LOGCOLLECTOR_HEALTH_INTERVAL" default:"5"` AcceptLabelPrefix string `env:"STARHUB_SERVER_LOGCOLLECTOR_ACCEPT_LABEL_PREFIX" default:"csghub_"` // the separator of log lines, default is "\\n" by client formats, "\n" sse auto newline - LineSeparator string `env:"STARHUB_SERVER_LOGCOLLECTOR_LINE_SEPARATOR" default:"\\n"` + LineSeparator string `env:"STARHUB_SERVER_LOGCOLLECTOR_LINE_SEPARATOR" default:"\\n"` + MaxStoreTimeDay int `env:"STARHUB_SERVER_LOGCOLLECTOR_MAX_STORE_TIME_DAY" default:"7"` + QueryLastReportTimeout int `env:"STARHUB_SERVER_LOGCOLLECTOR_QUERY_LAST_REPORT_TIMEOUT" default:"10"` } Temporal struct { @@ -503,6 +511,25 @@ type Config struct { MaxConcurrentWorkflowTaskExecutionSize int `env:"OPENCSG_TEMPORAL_MAX_CONCURRENT_WORKFLOW_TASK_EXECUTION_SIZE" default:"50"` } + APIRateLimiter struct { + Enable bool `env:"STARHUB_SERVER_API_RATE_LIMITER_ENABLE" default:"false"` + Limit int64 `env:"STARHUB_SERVER_API_RATE_LIMITER_LIMIT" default:"10"` + Window int64 `env:"STARHUB_SERVER_API_RATE_LIMITER_WINDOW" default:"60"` + } + + APILocationCheck struct { + Enable bool `env:"STARHUB_SERVER_API_LOCATION_CHECK_ENABLE" default:"false"` + WhiteList []string `env:"STARHUB_SERVER_API_LOCATION_CHECK_WHITE_LIST" default:"[China,Hong Kong,Singapore]"` + } + + Captcha struct { + ExceptionPaths []string `env:"STARHUB_SERVER_CAPTCHA_EXCEPTION_PATHS" default:"[/api/v1/broadcasts,/api/v1/notifications]"` + } + + GeoIP struct { + DBFile string `env:"STARHUB_SERVER_GEOIP_DB_FILE" default:"/starhub-bin/GeoLite2-Country.mmdb"` + } + Xnet struct { Endpoint string `env:"STARHUB_SERVER_XNET_ENDPOINT" default:"http://localhost:8097"` ApiKey string `env:"STARHUB_SERVER_XNET_API_KEY" default:"f3a7b9c1d6e5f8e2a1b5d4f9e6a2b8d7c3a4e2b1d9f6e7a8d2c5a7b4c1e3f5b8a1d4f9b7d6e2f8a5d3b1e7f9c6a8b2d1e4f7d5b6e9f2a4b3c8e1d7f995hd82hf"`