diff --git a/api-contracts/openapi/components/schemas/_index.yaml b/api-contracts/openapi/components/schemas/_index.yaml index aae7fd8cdb..3ff8229e7f 100644 --- a/api-contracts/openapi/components/schemas/_index.yaml +++ b/api-contracts/openapi/components/schemas/_index.yaml @@ -298,6 +298,8 @@ V2TaskSummaryList: $ref: "./v2/task.yaml#/V2TaskSummaryList" V2TaskSummary: $ref: "./v2/task.yaml#/V2TaskSummary" +V2DagChildren: + $ref: "./v2/task.yaml#/V2DagChildren" V2Task: $ref: "./v2/task.yaml#/V2Task" V2TaskEventList: @@ -310,3 +312,7 @@ V2TaskPointMetric: $ref: "./v2/task.yaml#/V2TaskPointMetric" V2TaskPointMetrics: $ref: "./v2/task.yaml#/V2TaskPointMetrics" +V2WorkflowRun: + $ref: "./v2/workflow_run.yaml#/V2WorkflowRun" +V2WorkflowRunList: + $ref: "./v2/workflow_run.yaml#/V2WorkflowRunList" diff --git a/api-contracts/openapi/components/schemas/v2/task.yaml b/api-contracts/openapi/components/schemas/v2/task.yaml index d1d700838f..59a3276906 100644 --- a/api-contracts/openapi/components/schemas/v2/task.yaml +++ b/api-contracts/openapi/components/schemas/v2/task.yaml @@ -1,4 +1,4 @@ -V2TaskSummarySingle: +V2TaskSummary: properties: metadata: $ref: ".././metadata.yaml#/APIResourceMeta" @@ -55,21 +55,16 @@ V2TaskSummarySingle: - workflowId - output -V2TaskSummary: +V2DagChildren: + type: object properties: - metadata: - $ref: ".././metadata.yaml#/APIResourceMeta" - parent: - $ref: "#/V2TaskSummarySingle" + dagId: + type: string + format: uuid children: type: array items: - $ref: "#/V2TaskSummarySingle" - description: The list of child tasks - required: - - metadata - - parent - - children + $ref: "#/V2TaskSummary" V2TaskSummaryList: type: object diff --git a/api-contracts/openapi/components/schemas/v2/workflow_run.yaml b/api-contracts/openapi/components/schemas/v2/workflow_run.yaml new file mode 100644 index 0000000000..7335cda77f --- /dev/null +++ b/api-contracts/openapi/components/schemas/v2/workflow_run.yaml @@ -0,0 +1,61 @@ +V2WorkflowRun: + properties: + metadata: + $ref: ".././metadata.yaml#/APIResourceMeta" + status: + $ref: "./task.yaml#/V2TaskStatus" + startedAt: + type: string + format: date-time + description: The timestamp the task run started. + finishedAt: + type: string + format: date-time + description: The timestamp the task run finished. + duration: + type: integer + description: The duration of the task run, in milliseconds. + tenantId: + type: string + description: The ID of the tenant. + example: bb214807-246e-43a5-a25d-41761d1cff9e + minLength: 36 + maxLength: 36 + format: uuid + additionalMetadata: + type: object + description: Additional metadata for the task run. + displayName: + type: string + description: The display name of the task run. + workflowId: + type: string + format: uuid + output: + type: object + description: The output of the task run (for the latest run) + errorMessage: + type: string + description: The error message of the task run (for the latest run) + required: + - metadata + - id + - status + - tenantId + - displayName + - workflowId + - output + +V2WorkflowRunList: + type: object + properties: + pagination: + $ref: ".././metadata.yaml#/PaginationResponse" + rows: + type: array + items: + $ref: "#/V2WorkflowRun" + description: The list of workflow runs + required: + - pagination + - rows diff --git a/api-contracts/openapi/openapi.yaml b/api-contracts/openapi/openapi.yaml index c1a1e63758..31a8cf67f4 100644 --- a/api-contracts/openapi/openapi.yaml +++ b/api-contracts/openapi/openapi.yaml @@ -26,6 +26,10 @@ paths: $ref: "./paths/v2/tasks/tasks.yaml#/getTask" /api/v2/tasks/{task}/task-events: $ref: "./paths/v2/tasks/tasks.yaml#/listTaskEvents" + /api/v2/dags/tasks: + $ref: "./paths/v2/tasks/tasks.yaml#/listTasksByDAGIds" + /api/v2/tenants/{tenant}/workflow-runs: + $ref: "./paths/v2/workflow-runs/workflow_run.yaml#/listWorkflowRuns" /api/v2/tenants/{tenant}/task-metrics: $ref: "./paths/v2/tasks/tasks.yaml#/getTaskStatusMetrics" /api/v2/tenants/{tenant}/task-point-metrics: diff --git a/api-contracts/openapi/paths/v2/tasks/tasks.yaml b/api-contracts/openapi/paths/v2/tasks/tasks.yaml index 6f70fe772f..cc445fd09d 100644 --- a/api-contracts/openapi/paths/v2/tasks/tasks.yaml +++ b/api-contracts/openapi/paths/v2/tasks/tasks.yaml @@ -100,6 +100,57 @@ listTasks: tags: - Task +listTasksByDAGIds: + get: + description: Lists all tasks that belong a specific list of dags + operationId: v2-dag:list:tasks + parameters: + - description: The external id of the DAG + in: query + name: dag_ids + required: true + schema: + type: array + items: + type: string + format: uuid + minLength: 36 + maxLength: 36 + - description: The tenant id + in: query + name: tenant + required: true + schema: + type: string + format: uuid + minLength: 36 + maxLength: 36 + responses: + "200": + content: + application/json: + schema: + type: array + items: + $ref: "../../../components/schemas/_index.yaml#/V2DagChildren" + description: The list of tasks + description: Successfully listed the tasks + "400": + content: + application/json: + schema: + $ref: "../../../components/schemas/_index.yaml#/APIErrors" + description: A malformed or bad request + "403": + content: + application/json: + schema: + $ref: "../../../components/schemas/_index.yaml#/APIErrors" + description: Forbidden + summary: List tasks + tags: + - Task + getTask: get: x-resources: ["tenant", "task"] diff --git a/api-contracts/openapi/paths/v2/workflow-runs/workflow_run.yaml b/api-contracts/openapi/paths/v2/workflow-runs/workflow_run.yaml new file mode 100644 index 0000000000..7fec4083d9 --- /dev/null +++ b/api-contracts/openapi/paths/v2/workflow-runs/workflow_run.yaml @@ -0,0 +1,92 @@ +listWorkflowRuns: + get: + x-resources: ["tenant"] + description: Lists workflow runs for a tenant. + operationId: v2-workflow-run:list + parameters: + - description: The tenant id + in: path + name: tenant + required: true + schema: + type: string + format: uuid + minLength: 36 + maxLength: 36 + - description: The number to skip + in: query + name: offset + required: false + schema: + type: integer + format: int64 + - description: The number to limit by + in: query + name: limit + required: false + schema: + type: integer + format: int64 + - description: A list of statuses to filter by + in: query + name: statuses + required: false + schema: + type: array + items: + $ref: "../../../components/schemas/_index.yaml#/V2TaskStatus" + - description: The earliest date to filter by + in: query + name: since + required: true + schema: + type: string + format: date-time + - description: The latest date to filter by + in: query + name: until + required: false + schema: + type: string + format: date-time + - description: Additional metadata k-v pairs to filter by + in: query + name: additional_metadata + required: false + schema: + type: array + items: + type: string + - description: The workflow ids to find runs for + in: query + name: workflow_ids + required: false + schema: + type: array + items: + type: string + format: uuid + minLength: 36 + maxLength: 36 + responses: + "200": + content: + application/json: + schema: + $ref: "../../../components/schemas/_index.yaml#/V2WorkflowRunList" + description: Successfully listed the tasks + "400": + content: + application/json: + schema: + $ref: "../../../components/schemas/_index.yaml#/APIErrors" + description: A malformed or bad request + "403": + content: + application/json: + schema: + $ref: "../../../components/schemas/_index.yaml#/APIErrors" + description: Forbidden + summary: List workflow runs + tags: + - Workflow Runs diff --git a/api/v1/server/handlers/v2/tasks/list.go b/api/v1/server/handlers/v2/tasks/list.go index 98f734be7b..8425fb59a5 100644 --- a/api/v1/server/handlers/v2/tasks/list.go +++ b/api/v1/server/handlers/v2/tasks/list.go @@ -78,7 +78,7 @@ func (t *TasksService) V2TaskList(ctx echo.Context, request gen.V2TaskListReques opts.FinishedBefore = request.Params.Until } - tasks, total, err := t.config.EngineRepository.OLAP().ListTaskRuns( + tasks, total, err := t.config.EngineRepository.OLAP().ListTasks( ctx.Request().Context(), tenant.ID, opts, diff --git a/api/v1/server/handlers/v2/tasks/list_by_dag_id.go b/api/v1/server/handlers/v2/tasks/list_by_dag_id.go new file mode 100644 index 0000000000..62380d4980 --- /dev/null +++ b/api/v1/server/handlers/v2/tasks/list_by_dag_id.go @@ -0,0 +1,37 @@ +package tasks + +import ( + "github.com/jackc/pgx/v5/pgtype" + "github.com/labstack/echo/v4" + + "github.com/hatchet-dev/hatchet/api/v1/server/oas/gen" + "github.com/hatchet-dev/hatchet/api/v1/server/oas/transformers/v2" + "github.com/hatchet-dev/hatchet/pkg/repository/prisma/sqlchelpers" +) + +func (t *TasksService) V2DagListTasks(ctx echo.Context, request gen.V2DagListTasksRequestObject) (gen.V2DagListTasksResponseObject, error) { + tenant := request.Params.Tenant + dagIds := request.Params.DagIds + + pguuids := make([]pgtype.UUID, 0) + for _, dagId := range dagIds { + pguuids = append(pguuids, sqlchelpers.UUIDFromStr(dagId.String())) + } + + tasks, err := t.config.EngineRepository.OLAP().ListTasksByDAGId( + ctx.Request().Context(), + tenant.String(), + pguuids, + ) + + if err != nil { + return nil, err + } + + result := transformers.ToDagChildren(tasks) + + // Search for api errors to see how we handle errors in other cases + return gen.V2DagListTasks200JSONResponse( + result, + ), nil +} diff --git a/api/v1/server/handlers/v2/workflow-runs/list.go b/api/v1/server/handlers/v2/workflow-runs/list.go new file mode 100644 index 0000000000..20e7faafd0 --- /dev/null +++ b/api/v1/server/handlers/v2/workflow-runs/list.go @@ -0,0 +1,91 @@ +package workflowruns + +import ( + "strings" + + "github.com/google/uuid" + "github.com/labstack/echo/v4" + + "github.com/hatchet-dev/hatchet/api/v1/server/oas/gen" + "github.com/hatchet-dev/hatchet/api/v1/server/oas/transformers/v2" + "github.com/hatchet-dev/hatchet/pkg/repository" + "github.com/hatchet-dev/hatchet/pkg/repository/prisma/db" +) + +func (t *V2WorkflowRunsService) V2WorkflowRunList(ctx echo.Context, request gen.V2WorkflowRunListRequestObject) (gen.V2WorkflowRunListResponseObject, error) { + tenant := ctx.Get("tenant").(*db.TenantModel) + + var ( + statuses = []gen.V2TaskStatus{ + gen.V2TaskStatusCANCELLED, + gen.V2TaskStatusCOMPLETED, + gen.V2TaskStatusFAILED, + gen.V2TaskStatusQUEUED, + gen.V2TaskStatusRUNNING, + } + since = request.Params.Since + workflowIds = []uuid.UUID{} + limit int64 = 50 + offset int64 = 0 + ) + + if request.Params.Statuses != nil { + if len(*request.Params.Statuses) > 0 { + statuses = *request.Params.Statuses + } + } + + if request.Params.Limit != nil { + limit = *request.Params.Limit + } + + if request.Params.Offset != nil { + offset = *request.Params.Offset + } + + if request.Params.WorkflowIds != nil { + workflowIds = *request.Params.WorkflowIds + } + + opts := repository.ListWorkflowRunOpts{ + CreatedAfter: since, + Statuses: statuses, + WorkflowIds: workflowIds, + Limit: limit, + Offset: offset, + } + + additionalMetadataFilters := make(map[string]interface{}) + + if request.Params.AdditionalMetadata != nil { + for _, v := range *request.Params.AdditionalMetadata { + kv_pairs := strings.Split(v, ":") + if len(kv_pairs) == 2 { + additionalMetadataFilters[kv_pairs[0]] = kv_pairs[1] + } + } + + opts.AdditionalMetadata = additionalMetadataFilters + } + + if request.Params.Until != nil { + opts.FinishedBefore = request.Params.Until + } + + tasks, total, err := t.config.EngineRepository.OLAP().ListWorkflowRuns( + ctx.Request().Context(), + tenant.ID, + opts, + ) + + if err != nil { + return nil, err + } + + result := transformers.ToWorkflowRunMany(tasks, total, limit, offset) + + // Search for api errors to see how we handle errors in other cases + return gen.V2WorkflowRunList200JSONResponse( + result, + ), nil +} diff --git a/api/v1/server/handlers/v2/workflow-runs/service.go b/api/v1/server/handlers/v2/workflow-runs/service.go new file mode 100644 index 0000000000..35f3a81146 --- /dev/null +++ b/api/v1/server/handlers/v2/workflow-runs/service.go @@ -0,0 +1,15 @@ +package workflowruns + +import ( + "github.com/hatchet-dev/hatchet/pkg/config/server" +) + +type V2WorkflowRunsService struct { + config *server.ServerConfig +} + +func NewV2WorkflowRunsService(config *server.ServerConfig) *V2WorkflowRunsService { + return &V2WorkflowRunsService{ + config: config, + } +} diff --git a/api/v1/server/oas/gen/openapi.gen.go b/api/v1/server/oas/gen/openapi.gen.go index ba18573b6a..aeae6172c4 100644 --- a/api/v1/server/oas/gen/openapi.gen.go +++ b/api/v1/server/oas/gen/openapi.gen.go @@ -1146,6 +1146,12 @@ type UserTenantPublic struct { Name *string `json:"name,omitempty"` } +// V2DagChildren defines model for V2DagChildren. +type V2DagChildren struct { + Children *[]V2TaskSummary `json:"children,omitempty"` + DagId *openapi_types.UUID `json:"dagId,omitempty"` +} + // V2Task defines model for V2Task. type V2Task struct { // AdditionalMetadata Additional metadata for the task run. @@ -1232,10 +1238,38 @@ type V2TaskStatus string // V2TaskSummary defines model for V2TaskSummary. type V2TaskSummary struct { - // Children The list of child tasks - Children []V2TaskSummarySingle `json:"children"` - Metadata APIResourceMeta `json:"metadata"` - Parent V2TaskSummarySingle `json:"parent"` + // AdditionalMetadata Additional metadata for the task run. + AdditionalMetadata *map[string]interface{} `json:"additionalMetadata,omitempty"` + + // DisplayName The display name of the task run. + DisplayName string `json:"displayName"` + + // Duration The duration of the task run, in milliseconds. + Duration *int `json:"duration,omitempty"` + + // ErrorMessage The error message of the task run (for the latest run) + ErrorMessage *string `json:"errorMessage,omitempty"` + + // FinishedAt The timestamp the task run finished. + FinishedAt *time.Time `json:"finishedAt,omitempty"` + Metadata APIResourceMeta `json:"metadata"` + + // Output The output of the task run (for the latest run) + Output map[string]interface{} `json:"output"` + + // StartedAt The timestamp the task run started. + StartedAt *time.Time `json:"startedAt,omitempty"` + Status V2TaskStatus `json:"status"` + + // TaskId The ID of the task. + TaskId int `json:"taskId"` + + // TaskInsertedAt The timestamp the task was inserted. + TaskInsertedAt time.Time `json:"taskInsertedAt"` + + // TenantId The ID of the tenant. + TenantId openapi_types.UUID `json:"tenantId"` + WorkflowId openapi_types.UUID `json:"workflowId"` } // V2TaskSummaryList defines model for V2TaskSummaryList. @@ -1246,8 +1280,8 @@ type V2TaskSummaryList struct { Rows []V2TaskSummary `json:"rows"` } -// V2TaskSummarySingle defines model for V2TaskSummarySingle. -type V2TaskSummarySingle struct { +// V2WorkflowRun defines model for V2WorkflowRun. +type V2WorkflowRun struct { // AdditionalMetadata Additional metadata for the task run. AdditionalMetadata *map[string]interface{} `json:"additionalMetadata,omitempty"` @@ -1271,17 +1305,19 @@ type V2TaskSummarySingle struct { StartedAt *time.Time `json:"startedAt,omitempty"` Status V2TaskStatus `json:"status"` - // TaskId The ID of the task. - TaskId int `json:"taskId"` - - // TaskInsertedAt The timestamp the task was inserted. - TaskInsertedAt time.Time `json:"taskInsertedAt"` - // TenantId The ID of the tenant. TenantId openapi_types.UUID `json:"tenantId"` WorkflowId openapi_types.UUID `json:"workflowId"` } +// V2WorkflowRunList defines model for V2WorkflowRunList. +type V2WorkflowRunList struct { + Pagination PaginationResponse `json:"pagination"` + + // Rows The list of workflow runs + Rows []V2WorkflowRun `json:"rows"` +} + // WebhookWorker defines model for WebhookWorker. type WebhookWorker struct { Metadata APIResourceMeta `json:"metadata"` @@ -1901,6 +1937,15 @@ type WorkflowVersionGetParams struct { Version *openapi_types.UUID `form:"version,omitempty" json:"version,omitempty"` } +// V2DagListTasksParams defines parameters for V2DagListTasks. +type V2DagListTasksParams struct { + // DagIds The external id of the DAG + DagIds []openapi_types.UUID `form:"dag_ids" json:"dag_ids"` + + // Tenant The tenant id + Tenant openapi_types.UUID `form:"tenant" json:"tenant"` +} + // V2TaskEventListParams defines parameters for V2TaskEventList. type V2TaskEventListParams struct { // Offset The number to skip @@ -1955,6 +2000,30 @@ type V2TaskListParams struct { WorkerId *openapi_types.UUID `form:"worker_id,omitempty" json:"worker_id,omitempty"` } +// V2WorkflowRunListParams defines parameters for V2WorkflowRunList. +type V2WorkflowRunListParams struct { + // Offset The number to skip + Offset *int64 `form:"offset,omitempty" json:"offset,omitempty"` + + // Limit The number to limit by + Limit *int64 `form:"limit,omitempty" json:"limit,omitempty"` + + // Statuses A list of statuses to filter by + Statuses *[]V2TaskStatus `form:"statuses,omitempty" json:"statuses,omitempty"` + + // Since The earliest date to filter by + Since time.Time `form:"since" json:"since"` + + // Until The latest date to filter by + Until *time.Time `form:"until,omitempty" json:"until,omitempty"` + + // AdditionalMetadata Additional metadata k-v pairs to filter by + AdditionalMetadata *[]string `form:"additional_metadata,omitempty" json:"additional_metadata,omitempty"` + + // WorkflowIds The workflow ids to find runs for + WorkflowIds *[]openapi_types.UUID `form:"workflow_ids,omitempty" json:"workflow_ids,omitempty"` +} + // AlertEmailGroupUpdateJSONRequestBody defines body for AlertEmailGroupUpdate for application/json ContentType. type AlertEmailGroupUpdateJSONRequestBody = UpdateTenantAlertEmailGroupRequest @@ -2317,6 +2386,9 @@ type ServerInterface interface { // Get workflow version // (GET /api/v1/workflows/{workflow}/versions) WorkflowVersionGet(ctx echo.Context, workflow openapi_types.UUID, params WorkflowVersionGetParams) error + // List tasks + // (GET /api/v2/dags/tasks) + V2DagListTasks(ctx echo.Context, params V2DagListTasksParams) error // Get a task // (GET /api/v2/tasks/{task}) V2TaskGet(ctx echo.Context, task openapi_types.UUID) error @@ -2332,6 +2404,9 @@ type ServerInterface interface { // List tasks // (GET /api/v2/tenants/{tenant}/tasks) V2TaskList(ctx echo.Context, tenant openapi_types.UUID, params V2TaskListParams) error + // List workflow runs + // (GET /api/v2/tenants/{tenant}/workflow-runs) + V2WorkflowRunList(ctx echo.Context, tenant openapi_types.UUID, params V2WorkflowRunListParams) error } // ServerInterfaceWrapper converts echo contexts to parameters. @@ -4696,6 +4771,35 @@ func (w *ServerInterfaceWrapper) WorkflowVersionGet(ctx echo.Context) error { return err } +// V2DagListTasks converts echo context to params. +func (w *ServerInterfaceWrapper) V2DagListTasks(ctx echo.Context) error { + var err error + + ctx.Set(BearerAuthScopes, []string{}) + + ctx.Set(CookieAuthScopes, []string{}) + + // Parameter object where we will unmarshal all parameters from the context + var params V2DagListTasksParams + // ------------- Required query parameter "dag_ids" ------------- + + err = runtime.BindQueryParameter("form", true, true, "dag_ids", ctx.QueryParams(), ¶ms.DagIds) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter dag_ids: %s", err)) + } + + // ------------- Required query parameter "tenant" ------------- + + err = runtime.BindQueryParameter("form", true, true, "tenant", ctx.QueryParams(), ¶ms.Tenant) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter tenant: %s", err)) + } + + // Invoke the callback with all the unmarshaled arguments + err = w.Handler.V2DagListTasks(ctx, params) + return err +} + // V2TaskGet converts echo context to params. func (w *ServerInterfaceWrapper) V2TaskGet(ctx echo.Context) error { var err error @@ -4902,6 +5006,77 @@ func (w *ServerInterfaceWrapper) V2TaskList(ctx echo.Context) error { return err } +// V2WorkflowRunList converts echo context to params. +func (w *ServerInterfaceWrapper) V2WorkflowRunList(ctx echo.Context) error { + var err error + // ------------- Path parameter "tenant" ------------- + var tenant openapi_types.UUID + + err = runtime.BindStyledParameterWithLocation("simple", false, "tenant", runtime.ParamLocationPath, ctx.Param("tenant"), &tenant) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter tenant: %s", err)) + } + + ctx.Set(BearerAuthScopes, []string{}) + + ctx.Set(CookieAuthScopes, []string{}) + + // Parameter object where we will unmarshal all parameters from the context + var params V2WorkflowRunListParams + // ------------- Optional query parameter "offset" ------------- + + err = runtime.BindQueryParameter("form", true, false, "offset", ctx.QueryParams(), ¶ms.Offset) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter offset: %s", err)) + } + + // ------------- Optional query parameter "limit" ------------- + + err = runtime.BindQueryParameter("form", true, false, "limit", ctx.QueryParams(), ¶ms.Limit) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter limit: %s", err)) + } + + // ------------- Optional query parameter "statuses" ------------- + + err = runtime.BindQueryParameter("form", true, false, "statuses", ctx.QueryParams(), ¶ms.Statuses) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter statuses: %s", err)) + } + + // ------------- Required query parameter "since" ------------- + + err = runtime.BindQueryParameter("form", true, true, "since", ctx.QueryParams(), ¶ms.Since) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter since: %s", err)) + } + + // ------------- Optional query parameter "until" ------------- + + err = runtime.BindQueryParameter("form", true, false, "until", ctx.QueryParams(), ¶ms.Until) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter until: %s", err)) + } + + // ------------- Optional query parameter "additional_metadata" ------------- + + err = runtime.BindQueryParameter("form", true, false, "additional_metadata", ctx.QueryParams(), ¶ms.AdditionalMetadata) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter additional_metadata: %s", err)) + } + + // ------------- Optional query parameter "workflow_ids" ------------- + + err = runtime.BindQueryParameter("form", true, false, "workflow_ids", ctx.QueryParams(), ¶ms.WorkflowIds) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter workflow_ids: %s", err)) + } + + // Invoke the callback with all the unmarshaled arguments + err = w.Handler.V2WorkflowRunList(ctx, tenant, params) + return err +} + // This is a simple interface which specifies echo.Route addition functions which // are present on both echo.Echo and echo.Group, since we want to allow using // either of them for path registration @@ -5024,11 +5199,13 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL router.GET(baseURL+"/api/v1/workflows/:workflow/metrics", wrapper.WorkflowGetMetrics) router.POST(baseURL+"/api/v1/workflows/:workflow/trigger", wrapper.WorkflowRunCreate) router.GET(baseURL+"/api/v1/workflows/:workflow/versions", wrapper.WorkflowVersionGet) + router.GET(baseURL+"/api/v2/dags/tasks", wrapper.V2DagListTasks) router.GET(baseURL+"/api/v2/tasks/:task", wrapper.V2TaskGet) router.GET(baseURL+"/api/v2/tasks/:task/task-events", wrapper.V2TaskEventList) router.GET(baseURL+"/api/v2/tenants/:tenant/task-metrics", wrapper.V2TaskListStatusMetrics) router.GET(baseURL+"/api/v2/tenants/:tenant/task-point-metrics", wrapper.V2TaskGetPointMetrics) router.GET(baseURL+"/api/v2/tenants/:tenant/tasks", wrapper.V2TaskList) + router.GET(baseURL+"/api/v2/tenants/:tenant/workflow-runs", wrapper.V2WorkflowRunList) } @@ -8552,6 +8729,41 @@ func (response WorkflowVersionGet404JSONResponse) VisitWorkflowVersionGetRespons return json.NewEncoder(w).Encode(response) } +type V2DagListTasksRequestObject struct { + Params V2DagListTasksParams +} + +type V2DagListTasksResponseObject interface { + VisitV2DagListTasksResponse(w http.ResponseWriter) error +} + +type V2DagListTasks200JSONResponse []V2DagChildren + +func (response V2DagListTasks200JSONResponse) VisitV2DagListTasksResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + + return json.NewEncoder(w).Encode(response) +} + +type V2DagListTasks400JSONResponse APIErrors + +func (response V2DagListTasks400JSONResponse) VisitV2DagListTasksResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(400) + + return json.NewEncoder(w).Encode(response) +} + +type V2DagListTasks403JSONResponse APIErrors + +func (response V2DagListTasks403JSONResponse) VisitV2DagListTasksResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(403) + + return json.NewEncoder(w).Encode(response) +} + type V2TaskGetRequestObject struct { Task openapi_types.UUID `json:"task"` } @@ -8749,6 +8961,42 @@ func (response V2TaskList403JSONResponse) VisitV2TaskListResponse(w http.Respons return json.NewEncoder(w).Encode(response) } +type V2WorkflowRunListRequestObject struct { + Tenant openapi_types.UUID `json:"tenant"` + Params V2WorkflowRunListParams +} + +type V2WorkflowRunListResponseObject interface { + VisitV2WorkflowRunListResponse(w http.ResponseWriter) error +} + +type V2WorkflowRunList200JSONResponse V2WorkflowRunList + +func (response V2WorkflowRunList200JSONResponse) VisitV2WorkflowRunListResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + + return json.NewEncoder(w).Encode(response) +} + +type V2WorkflowRunList400JSONResponse APIErrors + +func (response V2WorkflowRunList400JSONResponse) VisitV2WorkflowRunListResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(400) + + return json.NewEncoder(w).Encode(response) +} + +type V2WorkflowRunList403JSONResponse APIErrors + +func (response V2WorkflowRunList403JSONResponse) VisitV2WorkflowRunListResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(403) + + return json.NewEncoder(w).Encode(response) +} + type StrictServerInterface interface { LivenessGet(ctx echo.Context, request LivenessGetRequestObject) (LivenessGetResponseObject, error) @@ -8938,6 +9186,8 @@ type StrictServerInterface interface { WorkflowVersionGet(ctx echo.Context, request WorkflowVersionGetRequestObject) (WorkflowVersionGetResponseObject, error) + V2DagListTasks(ctx echo.Context, request V2DagListTasksRequestObject) (V2DagListTasksResponseObject, error) + V2TaskGet(ctx echo.Context, request V2TaskGetRequestObject) (V2TaskGetResponseObject, error) V2TaskEventList(ctx echo.Context, request V2TaskEventListRequestObject) (V2TaskEventListResponseObject, error) @@ -8947,6 +9197,8 @@ type StrictServerInterface interface { V2TaskGetPointMetrics(ctx echo.Context, request V2TaskGetPointMetricsRequestObject) (V2TaskGetPointMetricsResponseObject, error) V2TaskList(ctx echo.Context, request V2TaskListRequestObject) (V2TaskListResponseObject, error) + + V2WorkflowRunList(ctx echo.Context, request V2WorkflowRunListRequestObject) (V2WorkflowRunListResponseObject, error) } type StrictHandlerFunc func(ctx echo.Context, args interface{}) (interface{}, error) type StrictMiddlewareFunc func(f StrictHandlerFunc, operationID string) StrictHandlerFunc @@ -11458,6 +11710,31 @@ func (sh *strictHandler) WorkflowVersionGet(ctx echo.Context, workflow openapi_t return nil } +// V2DagListTasks operation middleware +func (sh *strictHandler) V2DagListTasks(ctx echo.Context, params V2DagListTasksParams) error { + var request V2DagListTasksRequestObject + + request.Params = params + + handler := func(ctx echo.Context, request interface{}) (interface{}, error) { + return sh.ssi.V2DagListTasks(ctx, request.(V2DagListTasksRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "V2DagListTasks") + } + + response, err := handler(ctx, request) + + if err != nil { + return err + } else if validResponse, ok := response.(V2DagListTasksResponseObject); ok { + return validResponse.VisitV2DagListTasksResponse(ctx.Response()) + } else if response != nil { + return fmt.Errorf("Unexpected response type: %T", response) + } + return nil +} + // V2TaskGet operation middleware func (sh *strictHandler) V2TaskGet(ctx echo.Context, task openapi_types.UUID) error { var request V2TaskGetRequestObject @@ -11587,223 +11864,252 @@ func (sh *strictHandler) V2TaskList(ctx echo.Context, tenant openapi_types.UUID, return nil } +// V2WorkflowRunList operation middleware +func (sh *strictHandler) V2WorkflowRunList(ctx echo.Context, tenant openapi_types.UUID, params V2WorkflowRunListParams) error { + var request V2WorkflowRunListRequestObject + + request.Tenant = tenant + request.Params = params + + handler := func(ctx echo.Context, request interface{}) (interface{}, error) { + return sh.ssi.V2WorkflowRunList(ctx, request.(V2WorkflowRunListRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "V2WorkflowRunList") + } + + response, err := handler(ctx, request) + + if err != nil { + return err + } else if validResponse, ok := response.(V2WorkflowRunListResponseObject); ok { + return validResponse.VisitV2WorkflowRunListResponse(ctx.Response()) + } else if response != nil { + return fmt.Errorf("Unexpected response type: %T", response) + } + return nil +} + // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ "H4sIAAAAAAAC/+y9e2/jOLIo/lUE/37A3QWcZ3fPmRPg/OFO3N3eTidZO5lg7yAIaIm2OZEljUjlcRr5", - "7hd8SZRESpRfsTsCFjtpi49isapYLNbjZ8cN51EYwIDgzsnPDnZncA7Yn72rQT+Ow5j+HcVhBGOCIPvi", - "hh6k//UgdmMUERQGnZMOcNwEk3DufAPEnUHiQNrbYY27HfgM5pEPOydHHw8Pu51JGM8B6Zx0EhSQ3z52", - "uh3yEsHOSQcFBE5h3Hnt5ocvz6b825mEsUNmCPM51ek6vazhIxQwzSHGYAqzWTGJUTBlk4YuvvdR8KCb", - "kv7ukNAhM+h4oZvMYUCABoCugyYOIg58RpjgHDhTRGbJeN8N5wczjqc9Dz7Kv3UQTRD0vTI0FAb2ySEz", - "QJTJHYQdgHHoIkCg5zwhMmPwgCjykQvGfm47OgGYaxDx2u3E8O8ExdDrnPyZm/oubRyO/4IuoTBKWsFl", - "YoHp74jAOfvj/4/hpHPS+f8OMto7EIR3kFLdazoNiGPwUgJJjGuA5gckoAwL8P3w6XQGgim8Ahg/hbEG", - "sU8zSGYwdsLYCULiJBjG2HFB4LisI918FDuR7K/gksQJTMEZh6EPQUDh4dPGEBB4DQMQkCaTsm5OAJ8c", - "wvpi6xkHwSMifOGWkyHWwwnZV/4zo3aEHRRgAgIXWs8+QtMgiRpMjtE0cJIoY6VGUyZkZkFalCx6tOlr", - "txOFmMzCqWWvK9Gadnzxw6AXRQMDV17R75TdnMEZW02CIetDuZ5SEXFwEkVhTHKMeHT84eOn3/7r9z36", - "R+H/6O//fXh0rGVUE/33BE7yPMDWpaMKCrqAC3oOHRQ74cShmIUBQS4TdCrEf3bGACO30+1Mw3DqQ8qL", - "KY+XxFiJmU1gD+gJEAMp9gvSJKACrIJrBeWkQ1BpKDo5YcAkt0JXZUJi4lCLG/qFIoQPkcFYlu614lTI", - "XLmYChl2lRFpQZRF6FuIiYECQ0y+hVOndzVwZrSVCuOMkAifHBwI+t8XXyhx6o4fEKHv8KV+ngf4kpsm", - "mj3cZ6QLxq4HJ9bkO4Q4TGIX6sU4l4lez7B6guZQORRjMZbzBLAQpzmp3Tk+PD7eOzreO/pwffTp5PC3", - "k4+/7//+++8fPv2+d/jp5PCwo6grHiBwj06gQxUyCATkcbpRgOk6KHBubriAoEOrAI3Hx0cffz/8r73j", - "j7/BvY8fwKc9cPzJ2/t49F+/HXlH7mTy33T+OXg+h8GUMvmH3zTgJJG3KJp8gIkj+q8DVwV+QHSSbFdV", - "0A28cR0+QJ14eI5QDLFuybczyNmfEiuh3R3Ret96g+eQAA9wkqw5M3IUbJQr1wW5ksK2n9/f40+f6nCY", - "wtZNxUuKDC0SXRdGhOsIQ/h3ArkwyeOTKwQcs8tR5xwFZmLtdp73QhChPXpZmMJgDz6TGOwRMGVQPAIf", - "0X3pnKQr7iYJ8jqvJULi8OrW+znxH7gO1n+EATEuGT7Ku5CVvqoZslZz5TPcvXY7p/Qc8i0AGnh5kBpv", - "R3bhShi3NdkeqwVRCNmSwsBN4hgG7ss5miMyIjEgcPrCT+9kTjuc9i5O++f3g4v7q+Hl12F/NOp0O2fD", - "y6v7i/5tf3Td6Xb+fdO/6Wf//Dq8vLm6H17eXJzdDy8/Dy6UPc6g5JshxYMZo5wxBoGeIb0kzi51TzPk", - "zhhvcpmBsMPIcb+zOBGHc0QC5HflRAyhegHR4+KB68RLyQc2vo4xikjDURhgWMYakSK3jLEcWNVg8FHM", - "cJzGYXAbxg8TP3y6jtF0CmPjPgLPQxQK4P9QBHNpYDcOg/5zFEOMhU5ZIhza5EJsQPlYD6KEaEYuyR7a", - "rKuDSpmgBM5duvRqMaBfbIFa0jaOPA5S0mFMquxPhh/9WIwT7AZ40OmHtP8DfDF2N9AHVyMZSBlmRhcj", - "5VZgRBEJI+T2YhORzsH/hoEjD2aHbofzj97w4p/y9B1djBw2xjLMnZ5QcxT8z1F3Dp7/5/jTb+WjKgXW", - "zAvcWNDzYUz6c4D8r3GYRGapRptgnQjxESZ0jbyFvJLGuGN9X1tg+R56hF02Y3ntAtS6ldcoJ3xw7V6z", - "T3Jb6VodEgrrxkr2Vq6r24lDH9bpCHw1P+B8DOMhba/FR0cMVocVIz7sVExuRVoFFtgysJ9M9ZPSL6uf", - "tCsspUyYvhou1gwoPR6z0wXbytjs1yuldc4SlT9stPykWC7KVof0iGk01xLXkTkks9CrV24VdP3gXRRV", - "pSwzONt62o9PYqCaz8ZjWDb4A8b04NQOY74TpaDpBirMnoNVbGm2gSnyagnsHOnYNAJTFKTmrSr0X6Ut", - "U62MSZynJtcTleCtzHC6TVd097P+l97NOdXJe1cDgxauDHAZezD+/PJFPmLIYQKpDMHSRT8biWlEm1SF", - "ltJklmJIkj4M1J8kRVYrgzs4y0ve4oOQeC4yLkTS/zAJRsl8DuKXOsjYVt2Wu1WwJFf10oXcyQ0/Azqj", - "XxMt1fnHv0aXF874hUD8z3qdM9U22fTfl6MBOcYWMH+6nDLfS0C3BcoKEIUEOUMxdCVIUooA7Hb4Q7FZ", - "fpgkkIXoGUEQuzPtaWSi9xIuJwBpHyyYXpZQlZCyKm/lxEmQN0eaX8cjGHgUlpqBRbMmI/+dwKQeYt6q", - "ybhxEgQWEItmTUbGietC6NUDnTa0Hz2lQ1xlK9RcQdi3ffWOtQAXLHGmmAWvYoD8VzjWiNoqxwsmcRXX", - "C3HO/BWO99dkMi+NiQmM7OXLiMBIh9hKZZWgOQwTol+++Fi39MdlFdVHRUGVNxu2dJ3m+a9wPEw0TyIu", - "MzH78h3I7qEj7ZR6AJmbDCHAhjvPBAUIz5pN/RenyKodpUTLWxp2bwmiiyFOfKK1I2ICYtJsMZgAkmCL", - "9dAThLcV9D1MgmYkTje/OZW7DzCuZoEmy1XUxjqQlaOz0HP5ix0fRBJIugtmrhml2ySVg6v+xdng4mun", - "2xneXFzwv0Y3p6f9/ln/rNPtfOkNztkf/CmD/q3TIqh6pXdrsHWGKnbVbLGYhJnvsdl+v1GlLn2i1ep1", - "FOK8TRe/Mbx5aGpfvBTYxEQ64mLL9IH7cAvHszB8ePNFKrCsaonh9BwFsJGPBj1C2WeqPlB5Ig9SP5w6", - "Pgpgkwd57sipnYMOJxrUqiam3ryFxlZQwJbqvJB5l6Yz3GWoOoeP0M8bVD7fUPEyuPhy2el2bnvDi063", - "0x8OL4d6maKMk15qrPY/B4FOkIjvb38nlGSllx784xL3wvwIDW+GonPF3VCDAPXJ/meHP5CT+4jR7nG3", - "E8Bn+a8P3U6QzNk/cOfk6JBe9fKcleus8+wRLZyIU2E68bHVZUqBResGB5/LI3+wGzlbl9YhKSTAV6+u", - "tCmzuPgIE/5ckLmRH9rc3TQS69/03voDkhi5GnkcJPMru4s1o2N5vd43rfffVndpPhbi/knsYm0ccGh3", - "ieYjiqv0vh41uYeTFNTcLF0VITr5PwQEMjePMiqtbKkxFf8+HUAron2AyRBOkG9452N+asKRTR2MObHF", - "rCNkrhpr8PZjE/0B/MRw/MzBM5onc9WywV/usMMcpIUpVuz6Ewq88Em/7auw9dYg+tG8DilNNOuYAw/a", - "LoJ/00/Bv7Fl0L1EgeJ2k6GZu/JOwtiFnq0jgXI7UPZLrjeFKkdpdypdb8FhmPGY9jhMPy9xIBbHKB2J", - "HJsSawoqtaNBFwZkpNxiC+83DDwTPfOvjs7FSjU7NLmXLmKHWMKGsDZDgUBpZikoXZuLbn7VPJJuRFe9", - "UQtYiqNrxT+kf70fJ9IhjHzw8kv5a/IlKeYYbFxZjh7edn1K80+HhzXrLcBtWrXJcKJ0txfaBfuWLXwS", - "uphyOWP2CrZq4JZIRy3YODQDTiEmN7FB17oZnjskdDAMPOYpJ6652CHheh7DTQdEEqC/qTbgwYCgCYJx", - "qk0KBUgENXCHPjUWaAz9MJhKiGtkZXed/oR2Bs1KH8GRO4Ne4kOF0pb1lDWRVLdDuCuu/ZHWxDk2G/xO", - "WZe3KsOscCanf4xOv/XPbkzW2nTm9bqIbamzV3n1mcdX9StCU9pYnS/YMAlOVUNj42cKDsCmTy8FAJsl", - "jqyUw9tSh7d0msuIotJfrkx0W3Dh0sgBK885Iwc1cp8rj2K6lKk4rrZZjuAcRLMwhiM/JCu+keVuO/rH", - "cm6CwH7IDTOih72Zf8HbkXhHNS2LfnbiRC6sXh1QH0TrF4p8X3oK2K+0JJo0xhLRxB70AoNnaOmqN8Di", - "66l8NaXkoz4clZ96ZiAIoG+CV3x2kKe3TGE6uPPER9ff+fkIF0b/djkF83NfcJKl1FUwN62eflti6bS7", - "ed1s8GUWvRWKtp0qLBGRojtPF12FDLUHDYGRSe7p/VtmyPdimH+sr7lnr8knJQJxKSa1FpIYAg+MfWja", - "XPk9jU7nArGWTJZylTLMYKYAZRU5cpCuHWID+atVxdavwTWqR/pRmHsBVKzdK3KgYkR4a7I/1NJArjs+", - "DZOA6MGFRigXMZ1mfSowVLxr5jzALByIhL9b2n71bBcmxATighzJnvZ6EwJje2Su3CGNd6nYmSW0LVtf", - "TNrWJE4sZE2TFaddKlZMVR+DH5zV4ZRSYLqySqczgbpe7M7QI9xJudT80r1VIiakNyp9pwqujyGJXyqk", - "6Nr4UbnGbIYlKm4MChIkHvW3TxO9b8MFP8+A2mdV0cYQguaaqcBsXfX0HRQnNg3JSR60WI94l2I9KN3A", - "Rxgj8tKk90j2saK7LyjGZAS5kmxPe+egaa+G7sH8lpEDsDBzilkFTarnHt/fCmLeluipHJnWEnIm0qUN", - "adjnxvH7i8v728vh9/6w081+HPau+/fngx+D68x4Prj4en89+NE/u7+8YXas0Wjw9YKb1697w2v2V+/0", - "+8Xl7Xn/7Cu3yg8uBqNveQP9sH89/A834Ku2ejr05c31/bD/ZdgXfYZ9ZRJ17tH5JW153u+N0jEH/bP7", - "z/+5vxmxpdA1fTm/vL0f3lzc8ywy3/v/uVefDAxNBKBac5qOYxSkKq6cYoHDwfXgtHdeNVrVW4f4656j", - "4Uf/ooD4Bm8h4m/aWgdMlqCymDoTxiKFQd+QaOJWpuALHdZaWgnmrBfe1+bbAwHwXwhy8WVELhNSMWpm", - "dpgB7IQRgZ4jrpbpIPo51p62y5TeYOn8CPVJvoypDrTJQzabNWRN0Wvm5CHaNW+BkNbvhS7JyjTc4yTX", - "GbJnh9f8qlAwHUFC/4M3x6I88UH/OUJ0l1lYBwOmenzei0+DnSeWfY9FqDgghg6IojgE7gwFU56GjyG4", - "an6Z/IQTCXNWWxAKvmSZ77AMD/Nuq8SFYpH5ApCfxNACFOY4oQKiGvIxiwDWz+kDzJdqfmTJ/GBBIHaW", - "PbSIIHVLjzfwLInsC7NVBO6L0bXVmcgmDiDSXVNQ1Wrt62ZJoAXYLBcGqR/aevIIvaYpFysfiGTCTZFs", - "eZNJKBdLVlT3TCAYyvTIIT+bscZbVD1zsBFymfAWODFzWZayvVJTUNTQztYcJYKUm50gfE/L8L8ZQdln", - "O6GsV9f6BsOY97hKxj5yq0iBjVeRb0uFeWs2XezfIps+FPskbxaXtxfsdtQ7+zG46HQ7P/o/PveHFReC", - "6qgZZtfGZpcmndWj7GIWEuDXYSIHh2IYqJq7yXhFX8oUAZLyVSym9+X+H/xGpt4k2a3v8kJxOqtAb06t", - "0Wl2IJ5XhJqw7w7zztfLYB4UQ0LnCcQsZUNJ3+G99aEbzaJw9AE4q4mp4WObl6iHf7l0AOm213NoSiR2", - "ETV1G9Y8kGYOCYxlOI08KvlYzj/QPtx3jhwPvHSdI+cJwgf633kYkNk/F3yVT9GjDa8xS1aJqKvQR64m", - "aQ5XwatupWm6ct5Uoxc0kKx59qtz1xbAmVcnDDprl5lMOnEfsA04ARv9ym9Ysvb3mKxUXXlNEMxK8oQa", - "9RUVEPP+77AJr7VBvK0NYo22gbXkTbe20L4auemWOQWYw2/wFUiwLrZbJXfuWeAg7ESstQMCz3FBEITE", - "AawCAyvtJLORFRGvhQ7rLnG1RgzgeTHEWDVm5PQyeTsu2zToh28Az3TSegbwTB3y/+DCdEJ+c9WGV0Ya", - "8SJDzukMEOOEf8AYTVAdeplJhsqSR9FcVOfKwaCn6BnA5hpg2jlAWvTLwZBs8KnBQzjywUuOoOX+NbZ+", - "5LF7ZyCwfJE0c8pp+GRGIuNB+JRhTepoetgXOLbTImyvzCerCpAUiEr8LQdDKYNOWiJOxZMJ5efhFAWL", - "JztfjL+Xyn2+dRiXa4zqcD2EU4RJhXTfRnTbnXQGwbCFuyXLFNlumqoe4xmK8K5a5kqWyg2e5us4Zfhk", - "um374/ga4IfVZ/AmAD9QpUlfz4Qv5KLRSjUjKkVhkyxmuqJeUGEoVuNtjnwfYeiGgYf19hzmGPmjKpFZ", - "rmhtcRbnH6mdBxCICf3tn/VJMPTPU5iAeZQfXnazN+KlBoTyHOxT1S6u2HG7DAH/tjASLYLdNDhcU8Ab", - "567MA5vOZ5ElCOAHQ54h2j/AsOEKnwCr1QqbLbFZUqP9RQokrihKd5HMKorizVvzrSnhWMm2ophVVQlW", - "iKvNzG589w3+qUWhUr5k0X7X7FcbIuunzRd0ZK1wq87I1ioEk1FfM9dR6dvfbA8LG5dOrSIvW7T5+NsW", - "V1GVYLSeosWtXr+baEO/UDlWzh+06AOqdyAt+oWO+hfX99fqYtI13PPcmyUn1tNhv3ddyCfxfXB1ZXAS", - "5di8ChHVD0msU/PEmrXMlDly6p8CENduFsjFIRppPEXNNKwsQ/OWwWOUmlKiihqr+HrebZgEJny6lbEZ", - "jU/UUhSEkNRm5/AChE0xki1N40qZg01hzZQYMyfg08sfV+f965Lvb4VLsxjdVMVCjYbVPcvyNxvWiikF", - "2PYBLjfvCAVTH67Uj1TG6i0Eh/lQF4MqYcJ3RSSuV+KbN2Fx9K8oJ7QOle0N8D3dALfp7mYIl23vbu3d", - "bdN3N0Hb9KQQyVT4G+NKXTLtrMQiKYh4n9SKvMSUCFD2TWJ/kYgVOq7u2MihhGf1WrLUqcUiMXRjaGAX", - "/i1NVy2M2ygM9p3BxAlC4kRx+Ig86HUd4MQg8MK57MSy/4yhM4UBjIFgLpVoj9eG8eZo9raTABfbm02T", - "cgpnLbKpQrgl5Vny4sfq8pXrYmRMEQh+D4ixViNkPhBZ0nY+FDuJRO9GioZFZkMd6FluQy7ET0PPQLXf", - "rq+vHN7IcUMvpeBYIN8iu76ClRTm3MR3lgivJiGZl736fiBpXra29pXUUsDCtFNOjfe1f93pdq4uR+w/", - "N9f8oDSckDzxD65KWIe5K61wwXFB4EQwpnS13yiEETwC5IOxD2X+nZriheVp4TN0EwIdNwyE66//otfi", - "qPLA6lvHOuWKUl2WrQlgjKYB9JysE7tv3NwMzhzBPptXvHwwhj6u9ntmbRhLZcdBegxYZ1eG8TkdR7dl", - "PsDkGwQxGUNgka9PbBVzY8cUQODMZO91FY8AnJmpetDHBIx9dsXaQkjn4NlM+JoaF8sxwPr1DrO+EZfK", - "FuiyptE2aerIzO+8IQEXSiToskUlAd2SQTAJ7bhhqHRggeeh6STAMhsoz1TJGXHBhRQyi2oWkt2HdSk4", - "2bFa2ht5JPROrwd/9FlxrPTPq97NyGDEJBbvWRxZ8ilLHIbGXJvirOQStQBk7WuV6H1Tp33eDM81wzdV", - "Rll7rSKhCMtmVXpkXTbaddWP9BXxMTwupmby6rriFXh4+4c4o9qdAjnMM38hOAYE00SYCq3FwujsO+YH", - "D+8s8jjr32r1ipGQSP1nEgN9XV7vwTxsaXEMIlX9uzzv8WQn/7n+xgLnrv9z1R+dDgdX11puVzhZGWbU", - "P//y7XLEHz9+9C56/PHutv/52+Xld+NAMohw+UrI0uSjZRh7r3HmlZ/6jeu9jf8KxwbBSr/oALKiT1Ff", - "d4VvMPZnsxFz0sdQox6B6eJrlXt/DbTKvyjB3Lzuh2CENEVw1bNKUZabhBcd91SqULqwuSkkyvc040vh", - "VSWQCb15ZMYUEsxw52ZdnSntmx5KSqzJvjFsc0RiQOC0Nt2YAuF5rl9zZTPTJ/OBLMXC7h+O6+/ocuri", - "arparFZt0eBMFymRAjg40+JQ9v6Ogtyt+MvNxen1gMnDs5th7/M51YHOel8rJRkdRB50jciWza7hA/ld", - "f3ou9Tq74YOXCXo7q4VobXR7YEzyHWYpITWyqVDTssxjD/DF8Cwoh6dkWTFF4e5FeRY4OIIumiA3m8T5", - "RwQwhp7ziIAzQT6B8T8tS2be5st6r7wgSuG5tvLxVSkAqtR9Wlsq48VqtfB8sPZ0meUyXrnfw9sUOOFz", - "j9QEkpsGYW1F+LR1VmwK5EDv80uDwa+VXuVKLg31kLXXgsmeQhWw76qFyZZcxarKtFWBX1Vvszc6pcd0", - "f3RaeU5no1QUoVZpOSfFFMlYM8loBiLYyu5Wdrey+y1ld025s19ItK+2cF+ddGOTLXTfyROC4dJT2FDN", - "o3cYXCkcq0mXHwayrJc5GmNdhWNumyXPTuer2WJ8ygoFLFItdp3FbYvFXmsWYbzcMSfvJnQkhzrlHeu0", - "h0LzRrEBkpe0HwXPaL9J1msecVC1mmsw1eHP52f58kbTpa2Heg8pDmEVgQiuP42phjnRM35FfZh7ZGC3", - "uglFrvaJobj0vXi0WfW0WL/C5tp0AW8a0crWsfDAKX5Wq3Xxc1CPvuxovBc24eZo5nllVpBRpv5toAoM", - "Rc0osmzOtmyzIao5mmr7cAISn1zFKJQ58XXszxo5kWilY+Ba6232+PFGTxppCRkLULE4+6+zUmkaBRa5", - "Dy+mZ3L6zcHCJm33XqLwdAPWwsqrh+F1ln+0AkJNz2lrmK1Uls1KrIQ5K0qjDHRXzw5sX1dp2W5CIO8K", - "4fzdNjNp5zE+iSHzJamoszQHzzUtGtaLMQX0cSfkhAopqr7POYRjCGIY9xLCMmIxjDLZy37ONmVGCMvx", - "74bhA4KyOaK7yn+Sz30nnRlz31OSYYEIfYfCIwAJJwCNZyrv5vSuBixAmbArev7XlLI6R/uH+4eMMCMY", - "gAh1Tjof9o/2D1k0G5mxpR2ACB34oijZVOd8/VW+FtJWAcTYSa+HdBeBrCPcORffv7J1SWdZNsvx4WF5", - "4G8Q+GTGpPIn3feLkKRz5namc/LnXbeDZeQihTBrKN+N/xTjuzPoPnTuaH+21hgC76V+sbQZqlrtUDZY", - "5XIZcCxzHs8UR2IwmYjE1lWrT6GtXf7j0QEQaf32WBaXPfZehA9+sp/V3145jD4kGl38jP2OHSAzG/Ls", - "kTxXDetewlghUygfgUdWApbGloJdkQ2+NIODeH6Ezgmj54y7SkvpqNzPzYBcLi59N329K+39xzK2Ronr", - "Qownie+/OBylnpocs4y8127nI6cSNwyICGcFUeQjl2H04C9R1ilbR81pxer/iXxExafqOfApFqDnhLEz", - "Bp50FedgfFg5GDoovoTxGHke5LpsRt+cTqrITFK8yB5/1+0876WJNln8O//Q1RDGHbtEEVeT65Ar78uQ", - "OB/h1yBxRg+fQy47V0IMFlmENWRSiS0SOonEeR4br3oRvZKFGIr9lGHPiQEOaCsGLMUAp5b1iQH1gIzQ", - "Hs8afPAz/ZudhlGINUrDED6GD6wQT+9qwPMNC6eMdMaCmIgQS2gszQO0u42USIc3yAQJ61YddzFbnqBz", - "Bt2vTdS4CVUL0qEbey12TpJx9lsVJadbnqNg1w8T70C9ypq13VKiBXmdYIM4KMAEBCzhf56IT+ln+Yps", - "VoLXj1sGiJMEadjW1hBYjdbOEaw+y4mt/6E8yDzvySH2woi/aYsTTdlvblw9+Mn++1q131RKsVb7pQ1l", - "Nla+kbWSiA1hVE7Y140KodVttsiWVXN4x5DECD4KscaxwXaslW05Elcwk5E3R3GFVOP0c2em8IM6sca2", - "JZVqNTR/lgqw9073Z4yEW9rfLtqfw4XPcOPpvbmDW9R0bEJT6ZG4Iwf5Ko5wOsYBM2jzXcLGHT9HmF6A", - "fCfX2rTBtPUg33Btu03nEjuuTNlw82XWgtzqtokQ0q1nG1HYhPL+5zY5DBAJqTQ/+Mk5/vUgisMxNF8u", - "5SudA7KHYBI6zK7L8JWPqDUzfDr1VYjJMAmu2Lz2tinToZdKrg2fehUEJaLPOT0x/O5v9FS4CIkDEjIL", - "Y/S/FIpQ5qHgcfI8GKtk5iQA+dBzuN3eYdvjfBHyfJBtq/7gyJEZ9oH7cPCT/cfCiu+MaEMZnFyiHPZV", - "JPSwN9rnxjQSDwNxK63zeZxsk2pztBkwboKMhPnEnzYzMc8Tw9JtAd8Pn6CnfxEoUq0Uvez3KhWLE12e", - "YwJ88BMH2IpbLkaq1C/zS4AbsEl+MDOjiJN769ikgIyWUbaQUUoEm7LKxaiSUQKsYROpuCjWJr3qQueV", - "V+ISizR+G3sz/aNrNgQ8wJdFLQEKDMefPuWAOFqFDhTFIf0H9NozbItY03SJRGSWjB0QRZLay8cab1Pg", - "RwKjvThhh5f48/UAxO4MPcK6C6RoJcOHRX6jMqvysCB2tZMDWzCtHM98oAl4N824IniahA5+QJGE7e8E", - "xi8ZcOFkgplhRAMKCshvH7Vx1NXT8UrW4xfDlOxzwxnXaQ8U+y72nIXaLGAYxO/cKEhn/biZWXNc9wQw", - "Ez6TMAk8ndkix/4K86eaAf1pmFQ+PqYsXC+TMu9/s0TibRrIoz4ftJVG70YaZYV1Wln068gihfHXL4n8", - "cFoth7Djh1PHR0FJNyo/H56H03MU8NOxFUPbIYa65UxM8knBh4/Qx3Reng6nYmLWMjdz5cOHoAPai+d1", - "MKwcQ3rwOmw2BY5JGBsA4R2aAjLivTRA3M4AoROzCA7z+kM1R0XDyXP5LQx44NN7aSKNSijOlGaLQJL1", - "X+8hpUqDuvOJkmR7OBlez9mpkEph5Sw4D6fNjwH+GZvtVLwmAnYAK15u8NnkXqW8aWc9DtF8cD6RnQc0", - "CUUG/Tfwd64lcZHPRnFwbt2ZUxLne50RW53zso6iU1MsT6JUEcTAPKCeESYomFYT+O6YZTcQlWDHhFk0", - "45vGH7T8uLLwggbBBJV8qQ+1q3blAllZP0OoA64LO7K9jmypY8f6YnIWsByYN6HlnZy6VkWt9szUbaCi", - "NY/HS7W393q4qRrm6kLurFXQozcOuSufgG3Ina2OulTInd0peYAhof/F9eH5sosju1QH3CnkgoLpSPSx", - "9Pl/J8ekgpglzkh1T1pWynmJG9G0Mj5K41arH9rSMFJsF6ba6pOpazvDB86SDzfiE+m/3dr6ispjGuuK", - "mwXA1imMC8RktzoiQ4CkdUUtXKcJozhpy1+r4i/BCAtGmFcfOBZeHZhFKuVcO7Ia4ZpYzF05a97zM+oD", - "fLF6RKXtcrNaJW5kZMByoZXz/pphUortWMGWyYrGACpVfxYDMU4CEbUFrWCVba2fP/WZst/oSZrt59s8", - "SLOpt+A5WoVDfYyuIJY0ovcBvojyiBFAcYle0kT9f1J2OzphTY94TcRj/q9jKt5169EUg9AyQ206bvMy", - "ZLy8FZ2LnOgGllxtCvG1h9K3XgAruRlA6eNpGUBva0KuygfRXgEYAkTO7UqzMOfvt3FDsMvUotp8eVTO", - "u/cCPf7vzcwq8yML9RQ+uxB6pSA1cUGREVPWfF5/MTkYJ/6D2e3nc+I/CPLAmUzAlUKB9nnHgoEuv6Fw", - "wG8pHXBz8dB6iW+ZfGBsqgoJvGIp4bKqNhXugew7N2QoBTdzKq5JanC3Ej7Ce1YoGALsFQpxYYhh5IOX", - "lYuNN6taVEw2XyOaGNKglxFdK6S2VUgNGaWuRz4xM5qljZXb5izsrN/hS/uslxkbF7qtM2S3N3bdjd0R", - "tt9V8oE4DSrSMNPvuNnRPJRHzHs9mjkCtuVoXo1ZjQPXavXv7cBEwSMisKmDteyldxobsK/tWSl9xRR8", - "LOQlJrHd+obp3KczWlyTzzSfoJLWW/O34iXNUWLnHM1x+6Ye0RzcRRyhBWG0bKn3fk75ZjWumoLP5Q97", - "/N/NKm5ZsHLjGlvb5U+T56tq2PZSdOz62VrLvZoCYlvGvboshOn+mKK38/vYpDCXBSfseLrBLeSE9Ybe", - "LnbuvlnwrSXnamp+bTPniqDYxpxbdfLN4XwsihQ3uKPJXnoW/8G+tnc0SY0KPha6o0lst8qg7o6W0eJq", - "dEEx3sFP/odNCmoggHAmcTivC3vj1PBrqIJi2SbY+OfNJ8peOe8uogO+D67doix3F4akdimT5jZmZfLi", - "7wQmcG9OBbeLa4tgsdaOaJ2+IlcKjK+Q/Jv2+iGm2EWZsVORAbvk7L1+7SVHe4tFgDmiCL6k+1YmvrVM", - "pOIo3Z15KlikRJScs6hMjAGBe+zBycZVgrbmz1N1vhJDQOA5bdjGpW1rXNqqYphqMbnOSKWUzrYgWqkI", - "y6bSZ+Z5rYEzjsLOrTdO4c6q4iYTtxTVzjn/dVGJK3rsRaGP3Jf6lC2yg8M72CRska4EV6xHm67lQIeW", - "xUw8hd1oTT0bz3rEq5BVJmrJVTjDlYX5WuMnz9Gi4qTJ7aGA6rZW0haVMVN4wVBttabknwUjHmACYmJk", - "xxH9ys+xy15CZg67rBQZ8gbDmL+ZMIAuKUJZz13kzA+HxzUlxhjKxLGSw8oMAk+88fghJ5g8rRTnfi0U", - "x6JkFz4gSAdlyY9z1bIYSvMzSkKgO7AwHdTlzSrU0cO6snatHBZy+GKUqzrdQBIXsdzK4q2TxWVGsKoo", - "WZuuy6K0auudyBCQ56/KLF2ro9n8pNZehm2N2C1maCPnWXJ05Ykq6nHsbeLJSpQI27WXq/WbC3SIaWYz", - "SOtW5XamfVTZhkeVdG/KjypL2ic01dMqWTcrlOaMXzhDaUs37ogdr7utFdw2UGdxQfnQSoStK7CoioiV", - "FFW0khO1OTV6hMB5JJLDsLYWNV93LZlGK0GqHNgQZu79QoRwIvC374Lwxo94dYyyKYaOIe1YEXvPkpTY", - "8jBr3rLwNmYDiJNAbFVN8AUKooT5Q/DHXd1yX7dCU2lzAVTIF7bhbyFQsjVV2gJ4M8ui8F8hGfFhW9Hy", - "dtpBsyxXBkuDGK69UGzzhULu0lqkhniL33sK44eqgLHMrdPoKNH6SGQu6hwVtwypFCFVtTYoMlI3et7R", - "kdvRGvG37VVOIf/FU4WIQUws9O5f33L8w7GxoRI5mpm9Rok+5Na2nLt9z28q4y1irOdSudo8T09ILryr", - "fW+zs+HdH5YZJtpKVEtfNWUIUD52muN40UcqiWh+vWyeIVKtyaNJFKkU0mnTRSrpIhW84BozUa7q0dsl", - "j9TBbV1kTrEg5QimvZ5uZVLJ/B6VgwyrL6hNBM5P9Z91r+M5Tqg9gQWZ7vJjeYH19aCpGNxhNUFs16Lx", - "yu3juTlaOG+Xro8U7uZpanF+PmBPHLUmav4QwhlaBXq/hq8HbPSWud+eubPcCFdKaQgO4zLW7DyO2Ha3", - "Bu0NGbRvVdwHNlkJsk1qqjKsTuLgGYjgmvSIERu7lTc7o0zwDWs1il9Io0g94i1KZ+eqZvt++uqGNbpG", - "FeuzcCz+QN6X6fZbGbByAM8BJs7gjCWtnEHHB3IHTclPACYDz5j95MOxLvvJBjz3mpTZUCVP61uzpS/2", - "C8gS++d8O1mIrV4mWEs7jeZdpmPy4AQkPumcHHZzomITiZnSuT8tMjkv/+6MXxw2gX5S8ckcJb4Jtat9", - "7Fm9vrXKRG/pmJZlOx3gjAFxZ6XHniqN6d3X61TfSTgybJ2BhY96+ankXRfx9NvXo5qkS5xsNvFygw/c", - "OAzqNRLayvkrHGdAkRhNp7XuE6dxGLxrNWVnskamG4s8Ou0UklQl3q9JDmy6uK06efEuZQauyFU5fnEm", - "Ih/mylJmqnyG7dNmjl/WlzlTOTY3nDszh4wldNj2YNLosaWTYE0KLT2WDn7S/+zJX+2KQZSPKuunAUo4", - "O14aIl29CawcRjdfHMKyioN2E9u8nMWqCno0NbPm5wni7rVb9dy2JHPtsgPPFnPWmo7O9tjcBdN3o8N6", - "BfLB7vxmNGBr51aN7/Wv9+09cpvvkbIwvu0lkrVf7w1yq6+3FLgIxBRphhfdAli88a1q49sQfJp4bC1s", - "4u10U2aBHNowASTB0Kq4kWy7yJV2xPqKy6UNcA8o8KygYg0bg/QdBV49NDtvQSFoDh0woYCWfAqfAJYh", - "fuoSOseHx0d7h/R/14eHJ+x//9eAe9G9RyfQE68HCNyjUHRsa/VRiMdwEsZwnSB/ZjOsEuYKLE9QgPBs", - "cZhl/43ieVVArxTT67MIls1v79YeWNQd22vNWrwI12MIZI6DNslygSNAowddnv3V7LmW/sG7XO6xVcNb", - "NXzzanirW7a65ZtEBuAly6MyAdSm8a4/39dQqjQ75ymoXuLT47HGapi2XMR+OJKdWyviNlsR13cvSglg", - "p9wlWmWqVaZ2RpnKlpGJ6pXYZq3qzqcMnlppN1y4vSxhWqvDarUSgwawXr3k4Gf6514p00mtV5Ie5IY6", - "y477JmlwYMzsq0X11ror6Xe39Vcq+isZ8NTMIcFAGzWeSythwJ2u1rNT3LfO47g9infdr2m9csROMUiT", - "GbxmMTSV9TyBE8AncySNfSDNNe+wO+mHq2+vahSsPntBJWgbrTSq2YYmlUGMm7/R9I/NnDzVrMlm+Fux", - "uPnyh1uXclIIuioqX08QoyKLc3ZkvTyWGoGQyPb6YEmVGCZBK4U3KYXlDigb0ET+GvWGDZZqaq6OqhL4", - "Xd40W/FrJX6FQlKnE69c5PI85ntumASkxkWHtZFZoWQCfvAIkA/GPmTSVxE3+tv4V0h4nnR8ymbcedFb", - "l7xrx5P35TZrwas3JxVOPq013PBGn0PSYin98uyfYBjjAzeJY1jN2ZjfDnhDh3Yrce8NhvFXSE7FYGuk", - "OzpTQzpjELelYN6+FAx0kxiRFybG3TB8QLCXUNn15x0VVYXgtjy5SXJn268h4ykis2R84ALfHwP3wUjO", - "p+E88iGBnKYv6fyO9jyiE/FCGF/Z0JcUl6dy+AKBfzg8rnlPcMW8XnneGQSeqPrmh3wztFUGU7H+WkBm", - "Dndygfk5LNGHCYjNomBEvy6GONa1OdYYPOvHGYOuIcLCcOrD9dAbG/oXpzeOvhXTW4a4X47eUPCICLQp", - "DSm1Yd6BKd1Wxzcd4Zr1HYi51niKqxNZ+U/4CMuNyS+w1Retj1WWHbWAvYzyrjU3xBztHQDXhRExW956", - "7DtOLWxikhK1qZvP+3TWY0/ig/OJ6ksXVlAfX7mO/lovgKx+P0NSae/t6SuGLM9gRU0z+r0ZffE+nXVV", - "CKODr4C++Mpb+qqp306RtAB9+eEUBWayOg+n2EGBA9jZuF+hYJyzgdZDS+wIpuNvqMaq1T3aD6dT6Dko", - "aK/PW3V9zh/rlGps78l+OA0TUsMMYULsuCFM3t7WI2g03LKKQy2R1iijjHpsyXYO52MY4xmKGlyBlE52", - "1yB+hPzIuokworUSuH7S5vchFUXtnWiRO5GKwXqSjADGT2Fc4YnAxaSQpI5sXyVSr+SY69MxTmcgmKYT", - "bZOy4TLIvBRRrTjfIXHOySpP6RZMFMMpFWRx1aWPt8CVGknqp7MutpFgbBPDSOS1z1w7oadLErLVebAP", - "3Ie1vDCM6Mhb/MBQI2oavjg8whgLECqL24p20n8Fw/hRoyMOgkn4FZI/xKArLe2hQJpldDjaP9w/1OWM", - "UNxG/ky73llU7biuWGzBVa6CnG+hE0OSxEEOeQU9m0qpJAhQMM2meN6TQ+6FEQ9RzWaTm/YEx7MwfNgT", - "XkQHP8UPFvF49KQQrcteRvx3+1A7MZDZiyedaMNOPJaxaxK+9lx4+3OhGC+nkqnRdUe0uLNijgOBZ5tL", - "smwqy+JVc4zQe7BtYo2t5ZvVOL9x6Lnvm0ANxcxQTGiSumneUIGddLta9twi9mQ2gdIWNeXRlDfZH68W", - "la412ganMMvAVOEhWOVwqjnjd8fdtLHjn1hxaw0reZSWonWo0lztQMrUakqFxJ1V2LoqCZm32hlaXoMp", - "gSEgd26YzgqBgUSibHNBLJa8xiFrOU3PaYIhlmG2wmlSjMywykySuo9bpUJocC/ayvCGJlk9UgDb6KrN", - "R1fprkMKxSwY3NCt07DsOaGByvUeonwWjOxpeeuteUsNIVqGsWzUPnvuaqYHbgWDra/yNEeGbaAz17ry", - "XLZp5dBKIhTVw1YeGBXE5ZizRk20Sq9PNymfRz9lvMf0pcN4UjZIp78N/KxJackTUq6g3tDi1Yb0gE3j", - "MIlYntAMBLlRRlBYp+/wpVObw2HNQmLJ3N3yUalN372F2sRC+cIbCS6ZV8boGyJTIjTN9LJQgpetlFzX", - "GnbZdwYTZt3GCaUO6HUZV/mAQExSnkLYmUDizqBnyiadCf4tV6QEGSyYNebNcsUo8DZKEtOmhmlTw6wh", - "NUwj0SxkA7Z41cqd5FZiWfjW7JAJ5leQy2uWctJhajlVsJV3W6UCZqS4rAp4fEAAfsAHP+l/6h7LaRtn", - "/MI5Pi9D/ji+BvjBNpMzHceYNgrgXXVP4UhoyGpsvS17bYi9UvJ7ApgdCiZe49SuxrnSf1ZwF2tu5Cz2", - "jz1WD7DaSYzXGEzrLnEYdLzWp+2s6y29Icf98rWW1i9Ssr1uJlsEvbXSZZuki5HLl5A0xYySTNo0ru/K", - "QLcu7cpJk66GG1J3vbgrS0LEqz+Kul8KGkzGZxS40A7QxnUoC4XIJijw0kpkNYXI7pFnqPG0FN42WvCJ", - "k9cwCRYzZhdpuX0Cy+s3DD9lU3K17LEQOlGIAmIpeuYoSAikNwr5VwzBgxc+Bak0aiCJvkJyRSffdTlU", - "qJibnifbXyk3A1UGqS0K7AQFCM+2s0gup7YcqS0gmhiftMKpQjjlMbQ6EWWVyYK2ywmd/Qr9p614u40V", - "b7OCmYycNlMnkxOFsUKmFi0QxD6CmDjcw8gCvDVqnY2BSQKC/BUcKL20cKpSp3XvUV+htbry6v189XXs", - "FW0cb6s63q2M6LDBIW98j7zOLllcR/zwaJzAhx0G7dFbSMrDsNLguC3mEBhDEMM4zSHQ1WYVYEHp/KxM", - "Yr9z0um83r3+vwAAAP//3odHIzkuAgA=", + "7hd8SZRESpRfsTsCFjtpi49isV4sFqt+dtxwHoUBDAjunPzsYHcG54D92bsa9OM4jOnfURxGMCYIsi9u", + "6EH6Xw9iN0YRQWHQOekAx00wCefON0DcGSQOpL0d1rjbgc9gHvmwc3L08fCw25mE8RyQzkknQQH57WOn", + "2yEvEeycdFBA4BTGnddufvjybMq/nUkYO2SGMJ9Tna7Tyxo+QgHTHGIMpjCbFZMYBVM2aejiex8FD7op", + "6e8OCR0yg44XuskcBgRoAOg6aOIg4sBnhAnOgTNFZJaM991wfjDjeNrz4KP8WwfRBEHfK0NDYWCfHDID", + "RJncQdgBGIcuAgR6zhMiMwYPiCIfuWDs57ajE4C5BhGv3U4M/05QDL3OyZ+5qe/SxuH4L+gSCqOkFVwm", + "Fpj+jgicsz/+/xhOOied/+8go70DQXgHKdW9ptOAOAYvJZDEuAZofkACyrAA3w+fTmcgmMIrgPFTGGsQ", + "+zSDZAZjJ4ydICROgmGMHRcEjss60s1HsRPJ/gouSZzAFJxxGPoQBBQePm0MAYHXMAABaTIp6+YE8Mkh", + "rC+2nnEQPCLCF245GWI9nJB95T8zakfYQQEmIHCh9ewjNA2SqMHkGE0DJ4kyVmo0ZUJmFqRFyaJHm752", + "O1GIySycWva6Eq1pxxc/DHpRNDBw5RX9TtnNGZyx1SQYsj6U6ykVEQcnURTGJMeIR8cfPn767b9+36N/", + "FP6P/v7fh0fHWkY10X9P4CTPA2xdOqqgoAu4oOfQQbETThyKWRgQ5DJBp0L8Z2cMMHI73c40DKc+pLyY", + "8nhJjJWY2QT2gGqAGEixX5AmARVgFVwrKCcdgkpD0ckJAya5FboqExITh1rc0C8UIXyIDMaydK8Vp0Lm", + "ysVUyLCrjEgLoixC30JMDBQYYvItnDq9q4Ezo61UGGeERPjk4EDQ/774QolTp35AhL7Dl/p5HuBLbppo", + "9nCfkS4Yux6cWJPvEOIwiV2oF+NcJno9w+oJmkNFKcZiLOcJYCFOc1K7c3x4fLx3dLx39OH66NPJ4W8n", + "H3/f//333z98+n3v8NPJ4WFHMVc8QOAenUCHKmQQCMjjdKMA03VQ4NzccAFBh1YBGo+Pjz7+fvhfe8cf", + "f4N7Hz+AT3vg+JO39/Hov3478o7cyeS/6fxz8HwOgyll8g+/acBJIm9RNPkAE0f0XweuCvyA6CTZrqqg", + "G3jjOnyAOvHwHKEYYt2Sb2eQsz8lVkK7O6L1vvUGzyEBHuAkWaMzchRslCvXBbmSwraf39/jT5/qcJjC", + "1k3FS4oMLRJdF0aE2whD+HcCuTDJ45MbBByzy1HnHAVmYu12nvdCEKE9eliYwmAPPpMY7BEwZVA8Ah/R", + "femcpCvuJgnyOq8lQuLw6tb7OfEfuA3Wf4QBMS4ZPsqzkJW9qhmy1nLlM9y9djunVA/5FgANvDxIjbcj", + "O3AljNuabI/VgiiEbElh4CZxDAP35RzNERmRGBA4feHaO5nTDqe9i9P++f3g4v5qePl12B+NOt3O2fDy", + "6v6if9sfXXe6nX/f9G/62T+/Di9vru6HlzcXZ/fDy8+DC2WPMyj5ZkjxYMYoZ4xBoGdIL4mzQ93TDLkz", + "xptcZiDsMHLc7yxOxOEckQD5XTkRQ6heQPS4eOA28VLygY2vY4wi0nAUBhiWsUakyC1jLAdWNRh8FDMc", + "p3EY3Ibxw8QPn65jNJ3C2LiPwPMQhQL4PxTBXBrYjcOg/xzFEGNhU5YIhza5EBtQVutBlBDNyCXZQ5t1", + "dVApE5TAuUuXXi0G9IstUEvaxpHqICUdxqTK/mT40Y/FOMFugAedfUj7P8AXY3cDfXAzkoGUYWZ0MVJO", + "BUYUkTBCbi82Eekc/G8YOFIxO3Q7nH/0hhf/lNp3dDFy2BjLMHeqoeYo+J+j7hw8/8/xp9/KqioF1swL", + "3FnQ82FM+nOA/K9xmERmqUabYJ0I8REmdI28hTySxrhjfV5bYPkeeoRdNmN57QLUupXXGCd8cO1es09y", + "W+laHRIK78ZK9lauq9uJQx/W2Qh8NT/gfAzjIW2vxUdHDFaHFSM+7ExM7kVaBRbYMrCfTPWT0i+rn7Qr", + "PKVMmL4aDtYMKD0eM+2CbWVs9uuV0jrnicorGy0/KZ6LstchVTGN5lriODKHZBZ69catgq4fvItiqpRl", + "BmdbT/vxSQxU89mohmWDP2BMFad2GPOZKAVNN1Bh9hysYkuzDUyRV0tg50jHphGYoiB1b1Wh/yptmVpl", + "TOI8NTmeqARv5YbTbbpiu5/1v/RuzqlN3rsaGKxwZYDL2IPx55cv8hJDDhNIYwiWDvrZSMwi2qQptJQl", + "sxRDkvRioF6TFFmtDO7gLC95ixdC4rrIuBBJ/8MkGCXzOYhf6iBjW3Vb7lbBktzUSxdyJzf8DOicfk2s", + "VOcf/xpdXjjjFwLxP+ttztTaZNN/X44G5BhbwPzpcsp8LwHdFigrQBQS5AzF0JUgSSkCsNvhF8Vm+WGS", + "QBaiZwRB7M602shE7yVcTgDSXlgwuyyhJiFlVd7KiZMg7440345HMPAoLDUDi2ZNRv47gUk9xLxVk3Hj", + "JAgsIBbNmoyME9eF0KsHOm1oP3pKh7jKV6g5grBv++oZawEuWEKnmAWv4oD8VzjWiNqqwAsmcZXQC6Fn", + "/grH+2tymZfGxARG9vJlRGCkQ2ylsUrQHIYJ0S9ffKxb+uOyhuqjYqDKkw1bus7y/Fc4HiaaKxGXuZh9", + "eQ9kd9GRdkojgMxNhhBgw5lnggKEZ82m/otTZNWOUqLlLQ27twTRxRAnPtH6ETEBMWm2GEwASbDFeqgG", + "4W0FfQ+ToBmJ081vTuXuA4yrWaDJchWzsQ5kRXUWei5/sOODSAJJd8HMNaN0m6RxcNW/OBtcfO10O8Ob", + "iwv+1+jm9LTfP+ufdbqdL73BOfuDX2XQv3VWBDWv9GENtsFQxa6aLRaTMPc9NvvvN2rUpVe0WruOQpz3", + "6eI3hjcPTe2NlwKbmEhHXGyZPnAfbuF4FoYPb75IBZZVLTGcnqMANorRoCqUfabmA5UnUpH64dTxUQCb", + "XMjzQE7tHHQ40aDWNDH15i00voICttTghSy6NJ3hLkPVOXyEft6h8vmGipfBxZfLTrdz2xtedLqd/nB4", + "OdTLFGWc9FBjtf85CHSCRHx/+zOhJCu99OAflzgX5kdoeDIUnSvOhhoEqFf2Pzv8gpzcR4x2j7udAD7L", + "f33odoJkzv6BOydHh/Sol+esXGddZI9o4UScCtOJj60OUwos2jA4+Fwe+YPdyNm6tAFJIQG+enSlTZnH", + "xUeY8OuCLIz80ObsppFY/6bn1h+QxMjVyOMgmV/ZHawZHcvj9b5pvf+2OkvzsRCPT2IHa+OAQ7tDNB9R", + "HKX39ajJXZykoOZm6aoI0cn/ISCQhXmUUWnlS42p+PfpAFoR7QNMhnCCfMM9H4tTE4Fs6mAsiC1mHSEL", + "1VhDtB+b6A/gJwb1MwfPaJ7MVc8Gv7nDDguQFq5YsetPKPDCJ/22r8LXW4PoR/M6pDTRrGMOPGi7CP5N", + "PwX/xpZB9xIFSthNhmYeyjsJYxd6toEEyulA2S+53hSqHKXdqXS9Bcow4zGtOkw/L6EQi2OUVCLHpsSa", + "gkrtaNCFARkpp9jC/Q0Dz0TP/KujC7FS3Q5NzqWL+CGW8CGszVEgUJp5CkrH5mKYXzWPpBvRVU/UApbi", + "6FrxD+lf7yeIdAgjH7z8UvGafEmKOwYbV5ajh7ddn9L80+FhzXoLcJtWbXKcKN3thXbBv2ULn4QuplzO", + "mL2CrRqEJdJRCz4OzYBTiMlNbLC1bobnDgkdDAOPRcqJYy52SLiey3CTgkgC9De1BjwYEDRBME6tSWEA", + "iUcNPKBPfQs0hn4YTCXENbKyu854QjuHZmWM4MidQS/xoUJpy0bKmkiq2yE8FNdepTUJjs0Gv1PW5a3K", + "MSuCyekfo9Nv/bMbk7c2nXm9IWJbGuxVXn0W8VV9i9CUNlYXCzZMglPV0dj4moIDsGntpQBgs8SRlXF4", + "W+rwlkFzGVFUxsuViW4LDlwaOWAVOWfkoEbhc+VRTIcyFcfVPssRnINoFsZw5IdkxSey3GlHf1nOXRDY", + "D7ljRvSwd/MveDoS96imZdHPTpzIhdWbA+qFaP1Cke/LSAH7lZZEk8ZZIprYg15g8AwtXfUEWLw9lbem", + "lHzUi6PyVc8MBAH0TfCKzw7y9J4pTAd3nvjo+jM/H+HCGN8up2Bx7gtOspS5Cuam1dNvSyyddjevmw2+", + "zKK3wtC2M4UlIlJ05+miq5ChVtEQGJnknj6+ZYZ8L4b5y/qac/aaYlIiEJfepNZCEkPggbEPTZsrv6ev", + "07lArCWTpUKlDDOYKUBZRY4cZGiH2EB+a1Wx9WsIjeqRfhTmbgAVb/eKAqgYEd6a/A+1NJDrjk/DJCB6", + "cKERykVcp1mfCgwVz5q5CDCLACIR75a2Xz3bhQkxgbggR7Krvd6EwNgemSsPSONdKnZmCWvLNhaTtjWJ", + "EwtZ02TFaZeKFVPTxxAHZ6WcUgpMV1YZdCZQ14vdGXqEOymXmh+6t0rEhPREpe9UwfUxJPFLhRRdGz8q", + "x5jNsETFiUFBgsSj/vRpovdtOODnGVB7rSraGJ6guWYqMHtXPX0HJYhNQ3KSBy3WI+6lWA9KN/ARxoi8", + "NOk9kn2s6O4LijEZQW4k29PeOWjaq2F4MD9l5AAszJxiVkGTGrnH97eCmLfl9VSOTGsJORPp0oc07HPn", + "+P3F5f3t5fB7f9jpZj8Oe9f9+/PBj8F15jwfXHy9vx786J/dX94wP9ZoNPh6wd3r173hNfurd/r94vL2", + "vH/2lXvlBxeD0be8g37Yvx7+hzvwVV89Hfry5vp+2P8y7Is+w74yiTr36PyStjzv90bpmIP+2f3n/9zf", + "jNhS6Jq+nF/e3g9vLu55Fpnv/f/cq1cGhiYCUK07TccxClKVUE6xwOHgenDaO68arequQ/x1z9Hwo39R", + "QHyDuxDxN22tAyZLUFlMnQljkcKgb0g0cStT8IUOay29BHPWC+9r8+2BAPgvBLn4MiKXCakYNXM7zAB2", + "wohAzxFHy3QQ/RxrT9tlSm+wdH6E+iRfxlQH2uQhm80asqbXa+bkIdo1b4GQ1u+FLsnKNNzjJNcZsmuH", + "1/yqUDAdQUL/gzfHojzxQf85QnSX2bMOBkz1+LwXnwY7Tyz7Hnuh4oAYOiCK4hC4MxRMeRo+huCq+WXy", + "E04kLFhtQSj4kmW+wzI8LLqtEheKR+YLQH4SQwtQWOCECojqyMfsBbB+Th9gvlTzJUsWBwsCsbPsokU8", + "UreMeAPPksi+MF9F4L4YQ1udiWziACLDNQVVrda/bpYEWoDNcmGQxqGtJ4/Qa5pysfKCSCbcFMmWN5mE", + "crFkRXXXBIKhTJcc8rMZa7xF1TUHGyGXCW8BjZnLspTtlZqCooZ2tkaVCFJupkH4npbhfzOCss92Qlmv", + "rvUNhjHvcZWMfeRWkQIbryLflgrz1my62L9FNn0o9kmeLC5vL9jpqHf2Y3DR6XZ+9H987g8rDgTVr2aY", + "XxubQ5p0Xo9yiFlIgF+HiRwcimOgau4m4xVjKVMESMpXsZiel/t/8BOZepJkp77LCyXorAK9ObNGZ9mB", + "eF7x1IR9d1h0vl4G80cxJHSeQMxSNpTsHd5b/3Sj2Ssc/QOc1byp4WObl6iHf7l0AOm213NoSiR2L2rq", + "Nqz5Q5o5JDCWz2mkquRjOf9A+3DfOXI88NJ1jpwnCB/of+dhQGb/XPBWPkWP9nmNWbJKRF2FPnI1SXO4", + "CV51Kk3TlfOmGruggWTNs19duLYAzrw64dBZu8xk0onHgG0gCNgYV37DkrW/x2Sl6sprHsGsJE+o0V5R", + "ATHv/w678FofxNv6INboG1hL3nRrD+2rkZtuWVCA+fkNvgIJ1r3tVsmdRxY4CDsRa+2AwHNcEAQhcQCr", + "wMBKO8lsZEXEa6HDukNcrRMDeF4MMVadGTm7TJ6Oyz4N+uEbwDOdtJ4BPFOH/D+4MJ2Q39y04ZWRRrzI", + "kHM6A8Q44R8wRhNUh17mkqGy5FE0F9W5cjDoKXoGsLkGmHYOkBb9cjAkG7xq8BCOfPCSI2i5f429H3ns", + "3hkILF8kzZxyGj6Zkch4ED5lWJM2mh72BdR2WoTtlcVkVQGSAlGJv+VgKGXQSUvEqXgyofw8nKJg8WTn", + "i/H3UrnPtw7jco1RHa6HcIowqZDu24huO01nEAxbuFuyTJHtpqnmMZ6hCO+qZ67kqdygNl+HluGT6bbt", + "j+MzMD1VIumLL0c0MfZV6P7j+BrghzS3dPm+2QNT20QIGmDp4KtPN04AfqAWnr74Csf6RaNt0YyoVLBN", + "sgfeFcWNCkOxgnRz5PsIQzcMPKx3PrEozh9VWddyFXaLszj/SJ1SgEBM6G//rM/Yob9LwwTMo/zwspu9", + "xzH1dpTnYJ+qdnHFUeZlCPi3hZFo8TJPg8M1vc4TrJuGi9P5LFIaAfxgSIpE+wcYNlzhE2CFZWGzJTbL", + "wLS/SDXHFT0pXiQNjHJK4K351pRwrKSGUXzAqgQrPALOfIR89w3BtEWhUj4R0n7X7FcbIuunzReMuq2I", + "Ac/I1uq9KKO+ZnGu8iFCsz0sbFw6tYq8bNF3RvW3LXGtKsFow1qLW73+mNaGQaxyrFzwajFgVR/tWgxi", + "HfUvru+v1cWka7jniUJLEbenw37vupD84vvg6soQ0cqxeRUiasySWGeTijVrmSmLOtXfWyBu3SyQOEQ0", + "0oS1mmlYWYbm4oU/qGpKiSpqrJIB8G7DJDDh0618SNJYo5aebAhJbY5kL0DYFCPZ0jR2eA42hTVTYswi", + "lk8vf1yd969LgcoV8df5Y0BrsL8ng32bTG3DU8zW1G5N7U2b2oK274rScb2mnC5Ch1/f0/Vg2yiMGrfO", + "Qpnp/zi+zVejaHVEqyNaHWHUEe9K7C4jXBWx8lbCNRe1YS9km+QktROyIicUD5VYaWS53WWXyG0kwiy0", + "kjMx5TOVfZPYX+ThHR23FiU8OeGSFZstFomhG0ODXOHf0qz74o4OhcG+M5g4QUicKA4fkQe9rgOcGARe", + "OJedWBKzMXSmMIAxEGJIZdTjtWG8OZq97STAxfZm06ScwlmLbCr0tqTKVF78WLllcl2MjCnyWdwDYiw5", + "C1koV1Z7gg/FDj2idyN7xSJBqw70LEUrV2mnoWeg2m/X11cOb+S4oZdScCyQb1EkRMFKCnNu4jtLhFeT", + "kCwvUaMDBc3L1tYh31oKWJh2yhk+v/avO93O1eWI/efmmpkNJg3J85fhqrybmL8IEJGELgicCMaUrvYb", + "vcQGjwD5YOxDmUaspgZreVr4DN2EQMcNA/GCwX/RHyioKcXK9Mc6g5JSXZZ0DmCMpgH0nKwTO7bc3AzO", + "HME+mzc2fTCGPq5+vsHaMJbK1EGqBqyTxMP4nI6j2zIfYPINgpiMIbBIOyq2ir3GwRRA4Mxk73XVwAGc", + "mal50McEjH12UttCSOfg2Uz4mlI9yzHA+u0Os70Rl6qv6JI/0jZpBtzs+UxDAi5UetElvUsCuiWDYBLa", + "ccNQ6cDyZ4QmTYBlUmOecJcz4oILKSRI1iwkO1brMgkztVraG6kSeqfXgz/6rMZf+udV72ZkuN4gFjfd", + "HFnyklsoQ2PKYKEruUQtAFl7jy1639RZnzfDc83wTY1R1l5rSCjCslmxMVleknZddfhOxTM//ryvZnIz", + "PuiSKvDw9lf0RrM7BXKYZ/7CGz8QTBPhcbQWC6Oz75grHt5ZpKPXR3HoDSMhkfrPJAb68uLeg3nY0uIY", + "RKr5d3ne4zmb/nP9jb3/vf7PVX90OhxcXWu5XeFkZZhR//zLt8sRvxb90bvo8Wv92/7nb5eX340DybfQ", + "yxd0lz4mLcPYP35hbqr0+Yv+0cRf4dggWOkXHUBW9CnKhK8sJVET3WzEnAyV1phHYLr4WuXeXwOt8S8q", + "yTcvXyQYIc10XuUyLMpyk/Ci455KE0r3+ncKifI9TVxVuJwJZF0C/sBsCglmuHOzrs6U9k2VkuL/3De+", + "Ph+RGBA4rc2aqEB4nuvX3NjM7Mm8ZzennVFAPhzXn9Hl1MXVdLVYrdqiwZnuwVcK4OBMi0PZ+zsKcqfi", + "LzcXp9cDJg/Pboa9z+fUBjrrfa2UZHQQqegakS2bXcMH8rteey6Von3DipcJejuvhWhtDIhiTPIdZplt", + "NbKpUJq3zGMP8MVwuyiHp2RZMUXh7EV5Fjg4gi6aIDebxPlHBDCGnvOIgDNBPoHxPy0r/y5wH9yoQlPh", + "1rfyDlepY6yUr1tbRvbFSk7xtNb2dJmlZF+hzuWp1t+mThOfe6Tmwd00CGurJaotF2VT5wt6n18aDH6t", + "9CoXpGpoh6y9pFV2MayAfVctTLbkKFZ1s1sFflXZ4N7olKrp/ui0Uk9no1TU0ldpOSfFFMlYM8loBiLY", + "yu5Wdrey+y1ld03Vxl9ItK+2/middGOTLXTeyROC4dBT2FDNpXcYXCkcq6n6EQayOqH5nda66l/dNqsB", + "kM5Xs8X4lNU7WaTo9TprdBdrVtcswni4Y88/mtCRHOqUd6yzHgrNG70akryk/Sh4RvtNsl7zt0hVq7kG", + "Ux3+fK7Ll3eaLu091EdIcQirCERw/WlMLcyJnvErylzdIwO71U0oSk5MDDXy78WlzaqnxfoVNremC3jT", + "iFa2joUHTvGzWquL60E9+jLVeC98ws3RzNNjrSAxVv3dQBUYiplRZNmcb9lmQ1R3NLX24QQkPrmKUShL", + "e+jYnzVyItFKx8C13tvs8uONrjTSSlgWoGKh+6+zio8aAxa5Dy+ma3L6zcHCJ213X6LwdAPWwsqth+F2", + "ln+0AkLNMmzrmK00ls1GrIQ5q62lDHRXzw5sX1fp2W5CIO8K4fzeNnNp5zE+iSGLJakoFzcHzzUtGpa9", + "Mj315UHICRVS1HyfcwjHEMQw7iWEJfZjGGWyl/2cbcqMEFaqxA3DBwRlc0R3lf8kr/tOOjMWvqfk9AMR", + "+g5FRAASQQCayFTezeldDVjqAsKO6PlfU8rqHO0f7h8ywoxgACLUOel82D/aP2QZpMiMLe0AROjAF7UV", + "p7rg66/ytpC2CiDGTno8pLsIZDn0zrn4/pWtSwbLslmODw/LA3+DwCczJpU/6b5fhCSdM7cznZM/77od", + "LN80UwizhvLe+E8xvjuD7kPnjvZna40h8F7qF0uboarVDmWDVS6XAccSgPKElyQGk4nIz1+1+hTa2uU/", + "Hh0AkZ10jyWj2mP3RfjgJ/tZ/e2Vw+hDorHFz9jv2AEyQStPgstTbrHuJYwVEh7zERgtxoBl46ZgVxS1", + "KM3gIJ45pXPC6DnjrtJSOir3czcgl4tLn01f70p7/7GMrVHiuhDjSeL7Lw5Hqafm+C0j77Xb+cipxA0D", + "InLBgCjykcswevCXqE6XraNGW7EypiKtWvGqeg58igXoOWHsjIEnQ8U5GB9WDoYOii9hPEaeB7ktm9E3", + "p5MqMpMUL4pg3HU7z3tpvmCWGYN/6GoI444dooirSdnKjfdlSJyP8GuQOKOHzyGXnSshBotk6BoyqcQW", + "CZ1E4jyPjVe9iF7JQgw1y8qw58QAB7QVA5ZigFPL+sSAqiAjtMeTnx/8TP9m2jAKscZoGMLH8IHVE+td", + "DXjadBGUkc5YEBMRYnnZpXuAdreREunwBpkgYd0qdRez5Qk6Z9D92kSNm1C1IB26sddi5yQZZ79VUXK6", + "5TkKdv0w8Q7Uo6zZ2i3la5DHCTaIgwJMQMDqluSJ+JR+lrfIZiN4/bhlgDhJkD7b2hoCq7HaOYLVazmx", + "9T+UC5nnPTnEXhjxO22h0ZT95s7Vg5/sv69V+02lFGu1X9pQ5mPlG1kridgQRuOEfd2oEFrdZos8ejXK", + "O4YkRvBRiDWODbZjrWzLkbiCmYy8OYorpBqnnzszhR/UiTW2LalUq6H5s1SAvXe6P2Mk3NL+dtH+HC6s", + "w43ae3OKW5SmbUJTqUrcEUW+ChVOxzhgDm2+S9i44+cI0wOQ7+RamzaYth7kG65tt+lcYseVKRtuvsxa", + "kFvdNhFCuvVsIwqbUN7/3CaHASIhleYHPznHvx5EcTiG5sOlvKVzQHYRTEKH+XUZvvIvas0Mn059FWIy", + "TIIrNq+9b8qk9FLJtWGtV0FQ4vU5pyeG3/2NaoWLkDggIbMwRv9LoQhlHgr+Tp4/xiq5OQlAPvQc7rd3", + "2PY4X4Q8H2TbqlccOTLDPnAfDn6y/1h48Z0RbSgfJ5coh30VCT3snfa5MY3Ew0DcSu98HifbZNocbQaM", + "myAjYT7xp81MzPPEsHRbwPfDJ+jpbwSKVCtFL/u9ysTiRJfnmAAf/MQBtuKWi5Eq9cv8EuAGbJIfzMwo", + "QnNvHZsUkNEyyhYySolgU1a5GFUySoA1bCINF8XbpDdd6LzySFxikcZ3Y29mf3TNjoAH+LKoJ0CB4fjT", + "pxwQR6uwgaI4pP+AXqvDtog1TYdIRGbJ2AFRJKm9rNZ4mwI/EhjtxQlTXuLP1wMQuzP0COsOkKKVfD4s", + "8huVWZU/C2JHOzmwBdPK8cwKTcC7acYVj6dJ6OAHFEnY/k5g/JIBF04mmDlGNKCggPz2UfuOuno6XpB/", + "/GKYkn1uOOM6/YFi38Wes6c2CzgG8Tt3CtJZP25m1hzXPQHMhM8kTAJP57bIsb/C/KllQH8aJpWXjykL", + "18ukLPrfLJF4mwbyqM8HbaXRu5FGWcmtVhb9OrJIYfz1SyI/nFbLIez44dTxUVCyjcrXh+fh9BwFXDu2", + "Ymg7xFC3nIlJXin48BH6mM7L0+FUTMxa5mauvPgQdEB78bwOhpVjSBWvw2ZT4JiEsQEQ3qEpICPeSwPE", + "7QwQOjF7wWFef6jmqGg4eS6/hQEPfHovTaRRCcWZ0mwRSLL+61VSqjSo00+UJFvlZLg9Z1ohlcKKLjgP", + "p83VAP+MzX4qXhMBO8AJ4JMpZpNHlfKmnfUERPPB+UR2EdAkFBn03yDeuZbERT4bJcC5DWdOSZzvdUZs", + "dcHLOopOXbE8iVLFIwYWAfWMMEHBtJrAd8ctu4FXCXZMmL1mfNP3By0/rux5QYPHBJV8qX9qVx3KBbK6", + "gIanDrju2ZHtcWRLAzvW9yZnAc+BeRNa3smZa1XUas9M3QYmWvP3eKn19l6Vm2phru7JnbUJevTGT+7K", + "GrB9cmdroy715M5OSx5gSOh/cf3zfNnFkV2qH9wp5IKC6Uj0sYz5fydqUkHMEjpS3ZOWlXJR4kY0rYyP", + "0ner1Rdt6TNSbPdMtbUn09B2hg+cJR9uxCcyfrv19RWNx/StK272ALbOYFzgTXZrIzIESFpXzMJ1ujCK", + "k7b8tSr+Eoyw4AvzaoVjEdWB2UulXGhHVhdd8xZzV3TNe75GfYAvVpeotF1uVqvEjYwMWC60ct5fM0xK", + "sR0r2DJZ0RhAperPYiDGSSBebUErWGVb6+tPfabsN7qSZvv5NhfSbOotuI5W4VAvoyuIJX3R+wBfRHnE", + "CKC4RC9pov4/KbsdnbCmR7wm4jH/1zEV77r1aIpBaJmhNh23eRnyvbwVnYuc6AaWXG0K8bU/pW+jAFZy", + "MoAyxtPyAb2tC7kqH0R7BGAIEDm3K93CnL/fJgzBLlOL6vPlr3LefRTo8X9vZlaZH1mYp/DZhdArPVIT", + "BxT5Ysqaz+sPJgfjxH8wh/18TvwHQR44kwm4UijQPu9YMNDlNxQO+C2lA24uHtoo8S2TD4xNVSGBVywl", + "XFbVpiI8kH3njgyl4GbOxDVJDR5Wwkd4zwYFQ4C9QSEODDGMfPCycrHxZlWLisnma0QTQxr0MqJrhdS2", + "Cqkho9T1yCfmRrP0sXLfnIWf9Tt8aa/1MmfjQqd1huz2xK47sTvC97tKPhDaoCINM/2Om6nmoVQx71U1", + "cwRsi2pejVuNA9da9e9NYaLgERHYNMBa9tIHjQ3Y11ZXylgxBR8LRYlJbLexYbrw6YwW1xQzzSeopPXW", + "/a1ESXOU2AVHc9y+aUQ0B3eRQGhBGC1b6qOfU75ZTaim4HP5wx7/d7OKWxas3LjG1nbF0+T5qhq2vRQd", + "u65ba7lXU0Bsy7hXl4Uw3R/T6+38PjYpzGXBCTuebnALOWG9T28X07tv9vjWknM1Nb+2mXPFo9jGnFul", + "+eZwPhZFihuc0WQvPYv/YF/bM5qkRgUfC53RJLZbY1B3RstocTW2oBjv4Cf/wyYFNRBAOJM4nNc9e+PU", + "8GuYgmLZJtj4580nyl457y5iA74Prt2iLHcXhqR2KZPmNmZl8uLvBCZwb04Ft4tri2Cx1o5ond4iVwqM", + "r5D8m/b6IabYRZmxUy8DdinYe/3WS472FnsB5ogi+JLuW5n41jKRiqN0d+apYJESUXLOojIxBgTusQsn", + "m1AJ2ppfT9XFSgwBgee0YfsubVvfpa3qDVMtJtf5Uimlsy14rVSEZVPpM/O81iAYR2HnNhqncGZVcZOJ", + "W4pq55z/uqjEFT32otBH7kt9yhbZweEdbBK2yFCCK9ajTddyoEPLYi6ewm60rp6NZz3iVcgqE7XkKpzh", + "ysJ8rfOT52hRcdLk9FBAdVsraYvKmCm8YKi2WlPyz4IRDzABMTGy44h+5XrsspeQmcMOK0WGvMEw5ncm", + "DKBLilDWcxc588PhcU2JMYYyoVZyWJlB4Ik7Hj/kBJOnleLcr4XiWJTswgcE6aAs+XGuWhZDaX5GSQh0", + "Bxamg7q8WYU6elhX1q6Vw0IOX4xyVacbSOIilltZvHWyuMwIVhUla9N1WZRWbaMTGQLy/FWZpWt1NJuf", + "1DrKsK0Ru8UMbeQ8S46u1KiiHsfeJq6sRImwXbu5Wr+7QIeYZj6DtG5VbmfaS5VtuFRJ96Z8qbKkf0JT", + "Pa2SdbNCac74hTOUtnTjjvjxuttawW0DdRYXlA+tRNi6AouqiFhJUUUrOVGbU6NHCJxHIjkMa2tR83XX", + "kmm0EqQqgA1hFt4vRAgnAn/7DghvfIlXxyibYugY0o4Vb+9ZkhJbHmbNWxbexmwAcRKIrap5fIGCKGHx", + "EPxyV7fc162wVNpcABXyhW34WwiUbE2VvgDezLIo/FdIRnzYVrS8nXXQLMuVwdMghmsPFNt8oJC7tBap", + "Ie7i957C+KHqwVgW1mkMlGhjJLIQdY6KW4ZUipCqWhsUGWkYPe/oyO1onfjbdiunkP/iqULEICYWeve3", + "bzn+4djYUIkczcxeo0Qfcmtbzt2+6zeV8RZx1nOpXO2epxqSC+/q2NtMN7x7ZZlhoq1EtfRRUz4Byr+d", + "5jhe9JJKIpofL5tniFRr8mgSRSqFdNp0kUq6SAUvuMZNlKt69HbJI3VwWxeZUzxIOYJpj6dbmVQyv0fl", + "R4bVB9QmAuen+s+62/EcJ9RqYEGmu3xZXmB9PWgqBnfYTBDbteh75fby3PxaOO+Xrn8p3M3T1OL8fMCu", + "OGpd1PwihDO0CvR+DV8P2Ogtc789c2e5Ea6U0hAcxmW82Xkcse1uHdobcmjfqrgPbLISZJvU1GRYncTB", + "MxDBNdkRIzZ2K292xpjgG9ZaFL+QRZFGxFuUzs5Vzfb99NYNa2yNKtZnz7H4BXlfpttvZcDKATwHmDiD", + "M5a0cgYdH8gdNCU/AZgMPGP2kw/HuuwnG4jca1JmQ5U8bWzNlt7YLyBL7K/z7WQhtrqZYC3tLJp3mY7J", + "gxOQ+KRzctjNiYpNJGZK5/60yOS8/LszfnHYBPpJxSfzK/FNmF3tZc/q7a1VJnpLx7Qs2+kAZwyIOytd", + "9lRZTO++Xqd6T8KRYRsMLGLUy1cl77qIp9/eHtUkXeJks4mbG3zgxmFQb5HQVs5f4TgDisRoOq0NnziN", + "w+Bdmyk7kzUy3Vjk0WmnkKQm8X5NcmDTwW3VyYt3KTNwRa7K8YszEfkwV5YyU+UzbJ82c/yyvsyZitrc", + "cO7MHDKWsGFbxaSxY0uaYE0GLVVLBz/pf/bkr3bFIMqqyvpqgBLOjpeGSFdvAiuH0c0Xh7Cs4qDdxDYv", + "Z7Gqgh5Nzbz5eYK4e+1WXbctyVy7HMCzxZy1JtXZqs1dcH03UtYrkA92+pvRgK2fW3W+19/et+fIbT5H", + "ysL4todI1n69J8itPt5S4CIQU6QZbnQLYPHGt6qPb0Pwad5ja2ETd6ebcgvk0IYJIAmGVsWNZNtFjrQj", + "1lccLm2Ae0CBZwUVa9gYpO8o8Oqh2XkPCkFz6IAJBbQUU/gEsHzipy6hc3x4fLR3SP93fXh4wv73fw24", + "F917dAI98XqAwD0KRce2Vh+FeAwnYQzXCfJnNsMqYa7A8gQFCM8Wh1n23yieVwX0SjG9Po9g2f32bv2B", + "RduxPdasJYpwPY5AFjhokywXOAI0qujy7K9mz7WMD97lco+tGd6a4Zs3w1vbsrUt3+RlAF6yPCoTQG0a", + "73r9voZSpZmep6B6iU/VY43XMG25iP9wJDu3XsRt9iKu71yUEsBOhUu0xlRrTO2MMZUtIxPVK/HNWtWd", + "Txk89dJuuHB7WcK0XofVWiUGC2C9dsnBz/TPvVKmk9qoJD3IDW2WHY9N0uDAmNlXi+qtDVfS724br1SM", + "VzLgqVlAgoE2aiKXVsKAO12tZ6e4b53quFXFux7XtF45YmcYpMkMXrM3NJX1PIETwCfzSxr7hzTXvMPu", + "pB+uPr2qr2D12QsqQdtopVHNNjSpDGLc/I2mf2wW5KlmTTbD34rFzZc/3LqUk0LQVVH5eh4xKrI450fW", + "y2NpEQiJbG8PlkyJYRK0UniTUljugLIBTeSv0W7YYKmm5uaoKoHf5UmzFb9W4lcYJHU28cpFLs9jvueG", + "SUBqQnRYG5kVSibgB48A+WDsQyZ9FXGjP41/hYTnScenbMadF711ybt2PHlfbrMWPHpzUuHk03rDDXf0", + "OSQtltIvz/4JhjE+cJM4htWcjfnpgDd0aLcS995gGH+F5FQMtka6ozM1pDMGcVsK5u1LwUA3iRF5YWLc", + "DcMHBHsJlV1/3lFRVXjclic3Se5s+zVkPEVklowPXOD7Y+A+GMn5NJxHPiSQ0/Qlnd/R6iM6ES+E8ZUN", + "fUlxeSqHLxD4h8PjmvsEV8zrleedQeCJqm9+yDdDW2UwFeuvBWTmcCcXmJ/DEn2YgNgsCkb062KIY12b", + "Y43Bs36cMegaIiwMpz5cD72xoX9xeuPoWzG9ZYj75egNBY+IQJvSkNIa5h2Y0W2lvukI16zvQMy1Ri2u", + "TmQVP+EjLDcmv8DWXrRWqyw7agF7GeVda06IOdo7AK4LI2L2vPXYd5x62MQkJWpTN5/36azHn8QH5xPV", + "ly6soD6+ch39tVEAWf1+hqTS3tvTVwxZnsGKmmb0ezP64n0666oQRgdfAX3xlbf0VVO/nSJpAfrywykK", + "zGR1Hk6xgwIHMN24X2FgnLOB1kNLTAXT8TdUY9XqHO2H0yn0HBS0x+etOj7n1TqlGttzsh9Ow4TUMEOY", + "EDtuCJO39/UIGg23rOJQS6Q1xiijHluyncP5GMZ4hqIGRyClk90xiKuQH1k38YxorQSun7T5eUhFUXsm", + "WuRMpGKwniQjgPFTGFdEInAxKSSpI9tXidQrOeb6bIzTGQim6UTbZGy4DDIvRVQrzndInHOyylO6BRPF", + "cEoFWVx16OMtcKVFksbprIttJBjbxDASee01107Y6ZKEbG0e7AP3YS03DCM68hZfMNSImoY3Do8wxgKE", + "yuK2op2MX8EwftTYiINgEn6F5A8x6EpLeyiQZhkdjvYP9w91OSOUsJE/0653FlU7risWWwiVqyDnW+jE", + "kCRxkENewc6mUioJAhRMsyme9+SQe2HEn6hms8lNe4LjWRg+7IkoooOf4geL93hUU4jW5Sgj/rv9Uzsx", + "kDmKJ51ow0E8lm/XJHytXnh7vVB8L6eSqTF0R7S4s2KOA4Fnm0OybCrL4lVzjLB7sG1ija3lm9UEv3Ho", + "eeybQA3FzFBMaJK6ad5QgZ10u1r23CL2ZD6B0hY15dGUN9kfrxaVrjXWBqcwy4epIkKwKuBUo+N3J9y0", + "ceCfWHHrDStFlJZe61CjuTqAlJnVlAqJO6vwdVUSMm+1M7S8BlcCQ0BOb5h0hcBAIlG2uUcslrzGIWs5", + "Tc9pgiGWYbaCNim+zLDKTJKGj1ulQmhwLtrK5w1NsnqkALavqzb/ukp3HFIoZsHHDd06C8ueExqYXO/h", + "lc+CL3ta3npr3lKfEC3DWDZmnz13NbMDt4LB1ld5miPD9qEzt7ryXLZp49BKIhTNw1YeGA3E5Zizxky0", + "Sq9PNymfRz9lvMf0psOoKRuk098GftaktOQJKVdQb2jxakN6wKZxmEQsT2gGgtwoIyis03f40qnN4bBm", + "IbFk7m55qdSm795Ca2KhfOGNBJfMK2OMDZEpEZpmelkowctWSq5rDbvsO4MJ827jhFIH9LqMq3xAICYp", + "TyHsTCBxZ9AzZZPOBP+WG1KCDBbMGvNmuWIUeBsliWlTw7SpYdaQGqaRaBayAVvcauU0uZVYFrE1O+SC", + "+RXk8pqlnAyYWs4UbOXdVpmAGSkuawIeH3hgig8IwA9WLzxoO4fMAHHG0A+DqQMcHEEXTZCbRlnQEUtC", + "5o/jMzBlb93ZVBYCBj4TGAfAd5Anw+XOel8NzOmB6T3ycKWsSQsdLMW1lmWvirmxCvCuPTnWsrLFHERD", + "xAZa1Y1g2346Q74Xc0YqIM/+tQ2btfVjFV7QiL1IH+MC/JBnb9bi4Cf9T10sDG3jjF840Ra5l45sm6id", + "jmPMCkch3M1LGY6EhpqUrbfVnhvSnin5PQHMbD6TKuXUXuIcs/IklZzF/rHHyn1Wq1FeQjQtq8Zh0PFa", + "n7azLqf2hhz3y5dSW79Iyfa6mWwR9NZKl22SLkYuX0LSFBPGMmnTuHwzA926cjMnTboafk+y67WbWY4x", + "XtxVlPVT0GC6W0KBC+0AbVxmtlBncIICLy00WFNnUJx01n6yWb/kGybBYndVRVpuTwZ5+4bhp3xTVC17", + "LIROFKKAWIqeOQoSAumJQv4VQ/DghU9BKo0aSKKvkFzRyXddDhUKYqf6ZPsLYWegyjeoiwI7QQHCs+2s", + "gc2pLUdqC4gmxietcKoQTnkMrU5E2bsxVaGzX2H/tAWtt7GgdS/nkNxQGVxOFMYCuFq0QBD7CGLi8ABC", + "C/DWaHU2BiYJCPJXoFB6aV1kpQzz3qO+AHN1YeX7+WorKxescbyt5ni38sGWDQ5543vkdXbJ4zriyqNx", + "fq72xsD6xmAxdatWeKxTu7l6n7WqV4nRaTXw9mvgVvlaUYMIOWlV786p3vUquaK0a5XckkouX1xaH5GH", + "K7VeMTHWGIIYxmlirK42VRbLtMT1UxL7nZNO5/Xu9f8FAAD//wE+28rVPQIA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/api/v1/server/oas/transformers/v2/tasks.go b/api/v1/server/oas/transformers/v2/tasks.go index 9cca7e71c2..1b9018a018 100644 --- a/api/v1/server/oas/transformers/v2/tasks.go +++ b/api/v1/server/oas/transformers/v2/tasks.go @@ -19,7 +19,7 @@ func jsonToMap(jsonBytes []byte) map[string]interface{} { return result } -func WorkflowRunRowToTaskSummaryUnit(task *timescalev2.ListWorkflowRunsRow) gen.V2TaskSummarySingle { +func ToTaskSummary(task *timescalev2.PopulateTaskRunDataRow) gen.V2TaskSummary { additionalMetadata := jsonToMap(task.AdditionalMetadata) var finishedAt *time.Time @@ -41,99 +41,66 @@ func WorkflowRunRowToTaskSummaryUnit(task *timescalev2.ListWorkflowRunsRow) gen. durationPtr = &duration } - output := jsonToMap(task.Output) - - return gen.V2TaskSummarySingle{ + return gen.V2TaskSummary{ Metadata: gen.APIResourceMeta{ Id: sqlchelpers.UUIDToStr(task.ExternalID), CreatedAt: task.InsertedAt.Time, UpdatedAt: task.InsertedAt.Time, }, - TaskId: int(task.RunID), - TaskInsertedAt: task.InsertedAt.Time, DisplayName: task.DisplayName, Duration: durationPtr, StartedAt: startedAt, FinishedAt: finishedAt, AdditionalMetadata: &additionalMetadata, - Status: gen.V2TaskStatus(task.ReadableStatus), + ErrorMessage: &task.ErrorMessage.String, + Status: gen.V2TaskStatus(task.Status), TenantId: uuid.MustParse(sqlchelpers.UUIDToStr(task.TenantID)), WorkflowId: uuid.MustParse(sqlchelpers.UUIDToStr(task.WorkflowID)), - Output: output, - ErrorMessage: &task.ErrorMessage.String, + TaskId: int(task.ID), + TaskInsertedAt: task.InsertedAt.Time, } } -func WorkflowRunChildToTaskSummaryUnit(task *timescalev2.ListDAGChildrenRow) gen.V2TaskSummarySingle { - additionalMetadata := jsonToMap(task.AdditionalMetadata) - - var finishedAt *time.Time +func ToTaskSummaryRows( + tasks []*timescalev2.PopulateTaskRunDataRow, +) []gen.V2TaskSummary { + toReturn := make([]gen.V2TaskSummary, len(tasks)) - if task.FinishedAt.Valid { - finishedAt = &task.FinishedAt.Time + for i, task := range tasks { + toReturn[i] = ToTaskSummary(task) } - var startedAt *time.Time - - if task.StartedAt.Valid { - startedAt = &task.StartedAt.Time - } + return toReturn +} - var durationPtr *int +func ToDagChildren( + tasks []*timescalev2.PopulateTaskRunDataRow, +) []gen.V2DagChildren { + dagIdToTasks := make(map[string][]gen.V2TaskSummary) - if task.FinishedAt.Valid && task.StartedAt.Valid { - duration := int(task.FinishedAt.Time.Sub(task.StartedAt.Time).Milliseconds()) - durationPtr = &duration + for _, task := range tasks { + dagId := sqlchelpers.UUIDToStr(task.DagExternalID) + dagIdToTasks[dagId] = append(dagIdToTasks[dagId], ToTaskSummary(task)) } - return gen.V2TaskSummarySingle{ - Metadata: gen.APIResourceMeta{ - Id: sqlchelpers.UUIDToStr(task.ExternalID), - CreatedAt: task.InsertedAt.Time, - UpdatedAt: task.InsertedAt.Time, - }, - TaskId: int(task.RunID), - TaskInsertedAt: task.InsertedAt.Time, - DisplayName: task.DisplayName, - Duration: durationPtr, - StartedAt: startedAt, - FinishedAt: finishedAt, - AdditionalMetadata: &additionalMetadata, - Status: gen.V2TaskStatus(task.ReadableStatus), - TenantId: uuid.MustParse(sqlchelpers.UUIDToStr(task.TenantID)), - WorkflowId: uuid.MustParse(sqlchelpers.UUIDToStr(task.WorkflowID)), - } -} - -func ToTaskSummary(task *olap.TaskRunDataRow) gen.V2TaskSummary { - parent := WorkflowRunRowToTaskSummaryUnit(task.Parent) + toReturn := make([]gen.V2DagChildren, 0, len(dagIdToTasks)) - children := make([]gen.V2TaskSummarySingle, len(task.Children)) - - for i, child := range task.Children { - children[i] = WorkflowRunChildToTaskSummaryUnit(child) + for dagId, tasks := range dagIdToTasks { + parsedUUID := types.UUID(uuid.MustParse(dagId)) + toReturn = append(toReturn, gen.V2DagChildren{ + DagId: &parsedUUID, + Children: &tasks, + }) } - return gen.V2TaskSummary{ - Metadata: gen.APIResourceMeta{ - Id: sqlchelpers.UUIDToStr(task.Parent.ExternalID), - CreatedAt: task.Parent.InsertedAt.Time, - UpdatedAt: task.Parent.InsertedAt.Time, - }, - Parent: parent, - Children: children, - } + return toReturn } func ToTaskSummaryMany( - tasks []*olap.TaskRunDataRow, + tasks []*timescalev2.PopulateTaskRunDataRow, total int, limit, offset int64, ) gen.V2TaskSummaryList { - toReturn := make([]gen.V2TaskSummary, len(tasks)) - - for i, task := range tasks { - toReturn[i] = ToTaskSummary(task) - } + toReturn := ToTaskSummaryRows(tasks) currentPage := (offset / limit) + 1 nextPage := currentPage + 1 diff --git a/api/v1/server/oas/transformers/v2/workflow_runs.go b/api/v1/server/oas/transformers/v2/workflow_runs.go new file mode 100644 index 0000000000..77583ebcd4 --- /dev/null +++ b/api/v1/server/oas/transformers/v2/workflow_runs.go @@ -0,0 +1,75 @@ +package transformers + +import ( + "math" + "time" + + "github.com/google/uuid" + "github.com/hatchet-dev/hatchet/api/v1/server/oas/gen" + "github.com/hatchet-dev/hatchet/pkg/repository" + "github.com/hatchet-dev/hatchet/pkg/repository/prisma/sqlchelpers" +) + +func ToWorkflowRun(task *repository.WorkflowRunData) gen.V2WorkflowRun { + additionalMetadata := jsonToMap(task.AdditionalMetadata) + + var finishedAt *time.Time + + if task.FinishedAt.Valid { + finishedAt = &task.FinishedAt.Time + } + + var startedAt *time.Time + + if task.StartedAt.Valid { + startedAt = &task.StartedAt.Time + } + + var durationPtr *int + + if task.FinishedAt.Valid && task.StartedAt.Valid { + duration := int(task.FinishedAt.Time.Sub(task.StartedAt.Time).Milliseconds()) + durationPtr = &duration + } + + return gen.V2WorkflowRun{ + Metadata: gen.APIResourceMeta{ + Id: sqlchelpers.UUIDToStr(task.ExternalID), + CreatedAt: task.InsertedAt.Time, + UpdatedAt: task.InsertedAt.Time, + }, + DisplayName: task.DisplayName, + Duration: durationPtr, + StartedAt: startedAt, + FinishedAt: finishedAt, + AdditionalMetadata: &additionalMetadata, + ErrorMessage: &task.ErrorMessage, + Status: gen.V2TaskStatus(task.ReadableStatus), + TenantId: uuid.MustParse(sqlchelpers.UUIDToStr(task.TenantID)), + WorkflowId: uuid.MustParse(sqlchelpers.UUIDToStr(task.WorkflowID)), + } +} + +func ToWorkflowRunMany( + tasks []*repository.WorkflowRunData, + total int, limit, offset int64, +) gen.V2WorkflowRunList { + toReturn := make([]gen.V2WorkflowRun, len(tasks)) + + for i, task := range tasks { + toReturn[i] = ToWorkflowRun(task) + } + + currentPage := (offset / limit) + 1 + nextPage := currentPage + 1 + numPages := int64(math.Ceil(float64(total) / float64(limit))) + + return gen.V2WorkflowRunList{ + Rows: toReturn, + Pagination: gen.PaginationResponse{ + CurrentPage: ¤tPage, + NextPage: &nextPage, + NumPages: &numPages, + }, + } +} diff --git a/api/v1/server/run/run.go b/api/v1/server/run/run.go index 1888f4fdd6..ee822bc142 100644 --- a/api/v1/server/run/run.go +++ b/api/v1/server/run/run.go @@ -27,6 +27,7 @@ import ( "github.com/hatchet-dev/hatchet/api/v1/server/handlers/tenants" "github.com/hatchet-dev/hatchet/api/v1/server/handlers/users" "github.com/hatchet-dev/hatchet/api/v1/server/handlers/v2/tasks" + workflowrunsv2 "github.com/hatchet-dev/hatchet/api/v1/server/handlers/v2/workflow-runs" webhookworker "github.com/hatchet-dev/hatchet/api/v1/server/handlers/webhook-worker" "github.com/hatchet-dev/hatchet/api/v1/server/handlers/workers" workflowruns "github.com/hatchet-dev/hatchet/api/v1/server/handlers/workflow-runs" @@ -57,6 +58,7 @@ type apiService struct { *monitoring.MonitoringService *info.InfoService *tasks.TasksService + *workflowrunsv2.V2WorkflowRunsService } func newAPIService(config *server.ServerConfig) *apiService { @@ -78,6 +80,7 @@ func newAPIService(config *server.ServerConfig) *apiService { MonitoringService: monitoring.NewMonitoringService(config), InfoService: info.NewInfoService(config), TasksService: tasks.NewTasksService(config), + V2WorkflowRunsService: workflowrunsv2.NewV2WorkflowRunsService(config), } } @@ -348,6 +351,19 @@ func (t *APIServer) registerSpec(g *echo.Group, spec *openapi3.T) (*populator.Po return task, sqlchelpers.UUIDToStr(task.TenantID), nil }) + // populatorMW.RegisterGetter("dag", func(config *server.ServerConfig, parentId, id string) (result interface{}, uniqueParentId string, err error) { + // ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + // defer cancel() + + // dag, err := config.OLAPRepository.ReadDAG(ctx, id) + + // if err != nil { + // return nil, "", err + // } + + // return dag, sqlchelpers.UUIDToStr(dag.TenantID), nil + // }) + authnMW := authn.NewAuthN(t.config) authzMW := authz.NewAuthZ(t.config) diff --git a/examples/loadtest/cli/do.go b/examples/loadtest/cli/do.go index 0a3a8f6002..7ee3b6c02d 100644 --- a/examples/loadtest/cli/do.go +++ b/examples/loadtest/cli/do.go @@ -94,12 +94,12 @@ func do(duration time.Duration, eventsPerSecond int, delay time.Duration, wait t log.Printf("ℹ️ final average duration per executed event: %s", finalDurationResult.avg) log.Printf("ℹ️ final average scheduling time per event: %s", finalScheduledResult.avg) - if emitted != executed { - log.Printf("⚠️ warning: emitted and executed counts do not match: %d != %d", emitted, executed) + if 2*emitted != executed { + log.Printf("⚠️ warning: emitted and executed counts do not match: %d != %d", 2*emitted, executed) } - if emitted != uniques { - return fmt.Errorf("❌ emitted and unique executed counts do not match: %d != %d", emitted, uniques) + if 2*emitted != uniques { + return fmt.Errorf("❌ emitted and unique executed counts do not match: %d != %d", 2*emitted, uniques) } log.Printf("✅ success") diff --git a/examples/loadtest/cli/run.go b/examples/loadtest/cli/run.go index ec8c074112..42e391ecff 100644 --- a/examples/loadtest/cli/run.go +++ b/examples/loadtest/cli/run.go @@ -50,56 +50,75 @@ func run(ctx context.Context, delay time.Duration, executions chan<- time.Durati concurrencyOpts = worker.Concurrency(getConcurrencyKey).MaxRuns(int32(concurrency)) } - err = w.On( - worker.Event("load-test:event"), + step := func(ctx worker.HatchetContext) (result *stepOneOutput, err error) { + var input Event + err = ctx.WorkflowInput(&input) + if err != nil { + return nil, err + } + + took := time.Since(input.CreatedAt) + l.Info().Msgf("executing %d took %s", input.ID, took) + + mx.Lock() + executions <- took + // detect duplicate in executed slice + var duplicate bool + // for i := 0; i < len(executed)-1; i++ { + // if executed[i] == input.ID { + // duplicate = true + // break + // } + // } + if duplicate { + l.Warn().Str("step-run-id", ctx.StepRunId()).Msgf("duplicate %d", input.ID) + } + if !duplicate { + uniques++ + } + count++ + executed = append(executed, input.ID) + mx.Unlock() + + time.Sleep(delay) + + if failureRate > 0 { + if rand.Float32() < failureRate { + return nil, fmt.Errorf("random failure") + } + } + + return &stepOneOutput{ + Message: "This ran at: " + time.Now().Format(time.RFC3339Nano), + }, nil + } + + err = w.RegisterWorkflow( + &worker.WorkflowJob{ + Name: "load-test-1", + Description: "Load testing", + On: worker.Event("load-test:event"), + Concurrency: concurrencyOpts, + // ScheduleTimeout: "30s", + Steps: []*worker.WorkflowStep{ + worker.Fn(step).SetName("step-one").SetTimeout("5m"), + }, + }, + ) + + if err != nil { + panic(err) + } + + err = w.RegisterWorkflow( &worker.WorkflowJob{ - Name: "load-test", + Name: "load-test-2", Description: "Load testing", + On: worker.Event("load-test:event"), Concurrency: concurrencyOpts, // ScheduleTimeout: "30s", Steps: []*worker.WorkflowStep{ - worker.Fn(func(ctx worker.HatchetContext) (result *stepOneOutput, err error) { - var input Event - err = ctx.WorkflowInput(&input) - if err != nil { - return nil, err - } - - took := time.Since(input.CreatedAt) - l.Info().Msgf("executing %d took %s", input.ID, took) - - mx.Lock() - executions <- took - // detect duplicate in executed slice - var duplicate bool - // for i := 0; i < len(executed)-1; i++ { - // if executed[i] == input.ID { - // duplicate = true - // break - // } - // } - if duplicate { - l.Warn().Str("step-run-id", ctx.StepRunId()).Msgf("duplicate %d", input.ID) - } - if !duplicate { - uniques++ - } - count++ - executed = append(executed, input.ID) - mx.Unlock() - - time.Sleep(delay) - - if failureRate > 0 { - if rand.Float32() < failureRate { - return nil, fmt.Errorf("random failure") - } - } - - return &stepOneOutput{ - Message: "This ran at: " + time.Now().Format(time.RFC3339Nano), - }, nil - }).SetName("step-one").SetTimeout("5m"), + worker.Fn(step).SetName("step-one").SetTimeout("5m"), }, }, ) diff --git a/frontend/app/src/components/molecules/data-table/data-table.tsx b/frontend/app/src/components/molecules/data-table/data-table.tsx index 2ccabe282d..3e2b7e6548 100644 --- a/frontend/app/src/components/molecules/data-table/data-table.tsx +++ b/frontend/app/src/components/molecules/data-table/data-table.tsx @@ -76,6 +76,7 @@ interface DataTableProps, TValue> { | undefined; manualSorting?: boolean; manualFiltering?: boolean; + getSubRows?: (row: TData) => TData[]; } interface ExtraDataTableProps { @@ -114,6 +115,7 @@ export function DataTable, TValue>({ card, manualSorting = true, manualFiltering = true, + getSubRows, }: DataTableProps & ExtraDataTableProps) { const [expanded, setExpanded] = React.useState({}); @@ -159,9 +161,10 @@ export function DataTable, TValue>({ getSortedRowModel: getSortedRowModel(), getFacetedRowModel: getFacetedRowModel(), getFacetedUniqueValues: getFacetedUniqueValues(), - getSubRows: (row) => row.subRows || [], + getSubRows: getSubRows, onExpandedChange: setExpanded, - getRowCanExpand: (row) => (row.original.subRows || []).length > 0, + // TODO: Figure this out + getRowCanExpand: (row) => row.subRows.length > 0, manualSorting, manualFiltering, manualPagination: true, @@ -222,8 +225,7 @@ export function DataTable, TValue>({ table.getRowModel().rows.map((row) => ( {getTableRow(row)} - {row.getIsExpanded() && - row.subRows.map((subRow) => getTableRow(subRow))} + {row.getIsExpanded() && row.subRows.map((r) => getTableRow(r))} )) ) : ( diff --git a/frontend/app/src/lib/api/generated/Api.ts b/frontend/app/src/lib/api/generated/Api.ts index 1b781f188c..fbe5a35edb 100644 --- a/frontend/app/src/lib/api/generated/Api.ts +++ b/frontend/app/src/lib/api/generated/Api.ts @@ -83,12 +83,14 @@ import { UserLoginRequest, UserRegisterRequest, UserTenantMembershipsList, + V2DagChildren, V2Task, V2TaskEventList, V2TaskPointMetrics, V2TaskRunMetrics, V2TaskStatus, V2TaskSummaryList, + V2WorkflowRunList, WebhookWorkerCreateRequest, WebhookWorkerCreated, WebhookWorkerListResponse, @@ -222,6 +224,86 @@ export class Api extends HttpClient + this.request({ + path: `/api/v2/dags/tasks`, + method: 'GET', + query: query, + secure: true, + format: 'json', + ...params, + }); + /** + * @description Lists workflow runs for a tenant. + * + * @tags Workflow Runs + * @name V2WorkflowRunList + * @summary List workflow runs + * @request GET:/api/v2/tenants/{tenant}/workflow-runs + * @secure + */ + v2WorkflowRunList = ( + tenant: string, + query: { + /** + * The number to skip + * @format int64 + */ + offset?: number; + /** + * The number to limit by + * @format int64 + */ + limit?: number; + /** A list of statuses to filter by */ + statuses?: V2TaskStatus[]; + /** + * The earliest date to filter by + * @format date-time + */ + since: string; + /** + * The latest date to filter by + * @format date-time + */ + until?: string; + /** Additional metadata k-v pairs to filter by */ + additional_metadata?: string[]; + /** The workflow ids to find runs for */ + workflow_ids?: string[]; + }, + params: RequestParams = {}, + ) => + this.request({ + path: `/api/v2/tenants/${tenant}/workflow-runs`, + method: 'GET', + query: query, + secure: true, + format: 'json', + ...params, + }); /** * @description Get a summary of task run metrics for a tenant * diff --git a/frontend/app/src/lib/api/generated/data-contracts.ts b/frontend/app/src/lib/api/generated/data-contracts.ts index 6080af17e4..d228aba397 100644 --- a/frontend/app/src/lib/api/generated/data-contracts.ts +++ b/frontend/app/src/lib/api/generated/data-contracts.ts @@ -61,7 +61,7 @@ export interface APIResourceMeta { updatedAt: string; } -export interface V2TaskSummarySingle { +export interface V2TaskSummary { metadata: APIResourceMeta; /** The ID of the task. */ taskId: number; @@ -103,13 +103,6 @@ export interface V2TaskSummarySingle { errorMessage?: string; } -export interface V2TaskSummary { - metadata: APIResourceMeta; - parent: V2TaskSummarySingle; - /** The list of child tasks */ - children: V2TaskSummarySingle[]; -} - export interface V2TaskSummaryList { pagination: PaginationResponse; /** The list of tasks */ @@ -230,6 +223,53 @@ export interface V2TaskEventList { rows?: V2TaskEvent[]; } +export interface V2DagChildren { + /** @format uuid */ + dagId?: string; + children?: V2TaskSummary[]; +} + +export interface V2WorkflowRun { + metadata: APIResourceMeta; + status: V2TaskStatus; + /** + * The timestamp the task run started. + * @format date-time + */ + startedAt?: string; + /** + * The timestamp the task run finished. + * @format date-time + */ + finishedAt?: string; + /** The duration of the task run, in milliseconds. */ + duration?: number; + /** + * The ID of the tenant. + * @format uuid + * @minLength 36 + * @maxLength 36 + * @example "bb214807-246e-43a5-a25d-41761d1cff9e" + */ + tenantId: string; + /** Additional metadata for the task run. */ + additionalMetadata?: object; + /** The display name of the task run. */ + displayName: string; + /** @format uuid */ + workflowId: string; + /** The output of the task run (for the latest run) */ + output: object; + /** The error message of the task run (for the latest run) */ + errorMessage?: string; +} + +export interface V2WorkflowRunList { + pagination: PaginationResponse; + /** The list of workflow runs */ + rows: V2WorkflowRun[]; +} + export interface V2TaskRunMetric { status: V2TaskStatus; count: number; diff --git a/frontend/app/src/lib/api/queries.ts b/frontend/app/src/lib/api/queries.ts index 89246775ab..bbd94ac864 100644 --- a/frontend/app/src/lib/api/queries.ts +++ b/frontend/app/src/lib/api/queries.ts @@ -9,6 +9,7 @@ type ListRateLimitsQuery = Parameters[1]; type ListLogLineQuery = Parameters[1]; type ListWorkflowRunsQuery = Parameters[1]; type GetTaskMetricsQuery = Parameters[1]; +type V2ListWorkflowRunsQuery = Parameters[1]; type V2ListTaskRunsQuery = Parameters[1]; type V2TaskGetPointMetricsQuery = Parameters< typeof api.v2TaskGetPointMetrics @@ -209,6 +210,12 @@ export const queries = createQueryKeyStore({ queryFn: async () => (await api.cronWorkflowList(tenant, query)).data, }), }, + v2WorkflowRuns: { + list: (tenant: string, query: V2ListWorkflowRunsQuery) => ({ + queryKey: ['v2:workflow-run:list', tenant, query], + queryFn: async () => (await api.v2WorkflowRunList(tenant, query)).data, + }), + }, v2Tasks: { list: (tenant: string, query: V2ListTaskRunsQuery) => ({ queryKey: ['v2-task:list', tenant, query], @@ -218,6 +225,16 @@ export const queries = createQueryKeyStore({ queryKey: ['v2-task:get', task], queryFn: async () => (await api.v2TaskGet(task)).data, }), + getByDagId: (tenant: string, dagIds: string[]) => ({ + queryKey: ['v2-task:get-by-dag-id', dagIds], + queryFn: async () => + ( + await api.v2DagListTasks({ + dag_ids: dagIds, + tenant, + }) + ).data, + }), }, v2TaskEvents: { list: (tenant: string, task: string, query: ListWorkflowRunsQuery) => ({ diff --git a/frontend/app/src/pages/main/index.tsx b/frontend/app/src/pages/main/index.tsx index 5579831163..7f98956f00 100644 --- a/frontend/app/src/pages/main/index.tsx +++ b/frontend/app/src/pages/main/index.tsx @@ -60,24 +60,8 @@ interface SidebarProps extends React.HTMLAttributes { currTenant: Tenant; } -function useSidebarButtonSelected() { - const location = useLocation(); - - function getSelectedAndOpenStates(prefix: string, to: string) { - return { - open: location.pathname.startsWith(prefix || to), - selected: !prefix && location.pathname === to, - }; - } - - return { - getSelectedAndOpenStates, - }; -} - function Sidebar({ className, memberships, currTenant }: SidebarProps) { const { sidebarOpen, setSidebarOpen } = useSidebar(); - const { getSelectedAndOpenStates } = useSidebarButtonSelected(); const meta = useCloudApiMeta(); const featureFlags = useCloudFeatureFlags(currTenant.metadata.id); @@ -125,8 +109,8 @@ function Sidebar({ className, memberships, currTenant }: SidebarProps) { } /> diff --git a/frontend/app/src/pages/main/tasks/$run/components/job-runs-columns.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/components/job-runs-columns.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/$run/components/job-runs-columns.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/components/job-runs-columns.tsx diff --git a/frontend/app/src/pages/main/tasks/$run/components/step-run-events.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/components/step-run-events.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/$run/components/step-run-events.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/components/step-run-events.tsx diff --git a/frontend/app/src/pages/main/tasks/$run/components/step-run-inputs.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/components/step-run-inputs.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/$run/components/step-run-inputs.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/components/step-run-inputs.tsx diff --git a/frontend/app/src/pages/main/tasks/$run/components/step-run-logs.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/components/step-run-logs.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/$run/components/step-run-logs.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/components/step-run-logs.tsx diff --git a/frontend/app/src/pages/main/tasks/$run/components/step-run-node.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/components/step-run-node.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/$run/components/step-run-node.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/components/step-run-node.tsx diff --git a/frontend/app/src/pages/main/tasks/$run/components/step-run-output.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/components/step-run-output.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/$run/components/step-run-output.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/components/step-run-output.tsx diff --git a/frontend/app/src/pages/main/tasks/$run/components/step-run-playground.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/components/step-run-playground.tsx similarity index 99% rename from frontend/app/src/pages/main/tasks/$run/components/step-run-playground.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/components/step-run-playground.tsx index 65175343e1..c6afb34450 100644 --- a/frontend/app/src/pages/main/tasks/$run/components/step-run-playground.tsx +++ b/frontend/app/src/pages/main/workflow-runs-v2/$run/components/step-run-playground.tsx @@ -22,7 +22,7 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'; import { StepRunLogs } from './step-run-logs'; import { RunStatus } from '../../components/run-statuses'; import { QuestionMarkCircleIcon, XMarkIcon } from '@heroicons/react/24/outline'; -import { TaskRunsTable as WorkflowRunsTable } from '../../components/task-runs-table'; +import { TaskRunsTable as WorkflowRunsTable } from '../../components/workflow-runs-table'; import { StepRunEvents } from './step-run-events'; import RelativeDate from '@/components/molecules/relative-date'; diff --git a/frontend/app/src/pages/main/tasks/$run/components/step-runs-worker-label-columns.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/components/step-runs-worker-label-columns.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/$run/components/step-runs-worker-label-columns.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/components/step-runs-worker-label-columns.tsx diff --git a/frontend/app/src/pages/main/tasks/$run/components/workflow-run-visualizer.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/components/workflow-run-visualizer.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/$run/components/workflow-run-visualizer.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/components/workflow-run-visualizer.tsx diff --git a/frontend/app/src/pages/main/tasks/$run/index.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/index.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/$run/index.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/index.tsx diff --git a/frontend/app/src/pages/main/tasks/$run/v2components/events-columns.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/events-columns.tsx similarity index 99% rename from frontend/app/src/pages/main/tasks/$run/v2components/events-columns.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/events-columns.tsx index 75a9cbd69a..1f07005885 100644 --- a/frontend/app/src/pages/main/tasks/$run/v2components/events-columns.tsx +++ b/frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/events-columns.tsx @@ -193,6 +193,8 @@ function mapEventTypeToTitle(eventType: V2TaskEventType | undefined): string { return 'Sent to worker'; case V2TaskEventType.QUEUED: return 'Queued'; + case V2TaskEventType.SKIPPED: + return 'Skipped'; case undefined: return 'Unknown'; default: diff --git a/frontend/app/src/pages/main/tasks/$run/v2components/header.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/header.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/$run/v2components/header.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/header.tsx diff --git a/frontend/app/src/pages/main/tasks/$run/v2components/mini-map.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/mini-map.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/$run/v2components/mini-map.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/mini-map.tsx diff --git a/frontend/app/src/pages/main/tasks/$run/v2components/step-run-detail/step-run-detail.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/step-run-detail/step-run-detail.tsx similarity index 99% rename from frontend/app/src/pages/main/tasks/$run/v2components/step-run-detail/step-run-detail.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/step-run-detail/step-run-detail.tsx index 2ae835ba29..8b4ba73712 100644 --- a/frontend/app/src/pages/main/tasks/$run/v2components/step-run-detail/step-run-detail.tsx +++ b/frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/step-run-detail/step-run-detail.tsx @@ -16,7 +16,7 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'; import { StepRunEvents } from '../step-run-events-for-workflow-run'; import { useOutletContext } from 'react-router-dom'; import { TenantContextType } from '@/lib/outlet'; -import { TaskRunsTable as WorkflowRunsTable } from '../../../components/task-runs-table'; +import { TaskRunsTable as WorkflowRunsTable } from '../../../components/workflow-runs-table'; import { useTenant } from '@/lib/atoms'; import { V2RunIndicator } from '../../../components/run-statuses'; import RelativeDate from '@/components/molecules/relative-date'; diff --git a/frontend/app/src/pages/main/tasks/$run/v2components/step-run-detail/step-run-error.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/step-run-detail/step-run-error.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/$run/v2components/step-run-detail/step-run-error.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/step-run-detail/step-run-error.tsx diff --git a/frontend/app/src/pages/main/tasks/$run/v2components/step-run-detail/step-run-logs.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/step-run-detail/step-run-logs.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/$run/v2components/step-run-detail/step-run-logs.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/step-run-detail/step-run-logs.tsx diff --git a/frontend/app/src/pages/main/tasks/$run/v2components/step-run-detail/step-run-output.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/step-run-detail/step-run-output.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/$run/v2components/step-run-detail/step-run-output.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/step-run-detail/step-run-output.tsx diff --git a/frontend/app/src/pages/main/tasks/$run/v2components/step-run-events-for-workflow-run.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/step-run-events-for-workflow-run.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/$run/v2components/step-run-events-for-workflow-run.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/step-run-events-for-workflow-run.tsx diff --git a/frontend/app/src/pages/main/tasks/$run/v2components/step-run-node.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/step-run-node.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/$run/v2components/step-run-node.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/step-run-node.tsx diff --git a/frontend/app/src/pages/main/tasks/$run/v2components/step-runs-worker-label-columns.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/step-runs-worker-label-columns.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/$run/v2components/step-runs-worker-label-columns.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/step-runs-worker-label-columns.tsx diff --git a/frontend/app/src/pages/main/tasks/$run/v2components/view-toggle.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/view-toggle.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/$run/v2components/view-toggle.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/view-toggle.tsx diff --git a/frontend/app/src/pages/main/tasks/$run/v2components/workflow-run-input.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/workflow-run-input.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/$run/v2components/workflow-run-input.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/workflow-run-input.tsx diff --git a/frontend/app/src/pages/main/tasks/$run/v2components/workflow-run-visualizer-v2.tsx b/frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/workflow-run-visualizer-v2.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/$run/v2components/workflow-run-visualizer-v2.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/$run/v2components/workflow-run-visualizer-v2.tsx diff --git a/frontend/app/src/pages/main/tasks/components/run-statuses.tsx b/frontend/app/src/pages/main/workflow-runs-v2/components/run-statuses.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/components/run-statuses.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/components/run-statuses.tsx diff --git a/frontend/app/src/pages/main/tasks/components/task-runs-metrics.tsx b/frontend/app/src/pages/main/workflow-runs-v2/components/task-runs-metrics.tsx similarity index 100% rename from frontend/app/src/pages/main/tasks/components/task-runs-metrics.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/components/task-runs-metrics.tsx diff --git a/frontend/app/src/pages/main/tasks/components/v2/task-runs-columns.tsx b/frontend/app/src/pages/main/workflow-runs-v2/components/v2/workflow-runs-columns.tsx similarity index 99% rename from frontend/app/src/pages/main/tasks/components/v2/task-runs-columns.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/components/v2/workflow-runs-columns.tsx index 58e9e02f95..caab4a3a78 100644 --- a/frontend/app/src/pages/main/tasks/components/v2/task-runs-columns.tsx +++ b/frontend/app/src/pages/main/workflow-runs-v2/components/v2/workflow-runs-columns.tsx @@ -7,7 +7,7 @@ import { } from '../../../events/components/additional-metadata'; import RelativeDate from '@/components/molecules/relative-date'; import { Checkbox } from '@/components/ui/checkbox'; -import { ListableWorkflowRun } from '../task-runs-table'; +import { ListableWorkflowRun } from '../workflow-runs-table'; import { ChevronDownIcon, ChevronRightIcon } from '@heroicons/react/24/outline'; import { Button } from '@/components/ui/button'; import { cn } from '@/lib/utils'; diff --git a/frontend/app/src/pages/main/tasks/components/task-runs-table.tsx b/frontend/app/src/pages/main/workflow-runs-v2/components/workflow-runs-table.tsx similarity index 95% rename from frontend/app/src/pages/main/tasks/components/task-runs-table.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/components/workflow-runs-table.tsx index e43f3298b1..c0b392199c 100644 --- a/frontend/app/src/pages/main/tasks/components/task-runs-table.tsx +++ b/frontend/app/src/pages/main/workflow-runs-v2/components/workflow-runs-table.tsx @@ -1,5 +1,5 @@ import { DataTable } from '@/components/molecules/data-table/data-table.tsx'; -import { columns } from './v2/task-runs-columns'; +import { columns } from './v2/workflow-runs-columns'; import { useEffect, useMemo, useState } from 'react'; import { ColumnFiltersState, @@ -14,7 +14,7 @@ import api, { queries, ReplayWorkflowRunsRequest, V2TaskStatus, - V2TaskSummarySingle, + V2WorkflowRun, } from '@/lib/api'; import { TenantContextType } from '@/lib/outlet'; import { useOutletContext, useSearchParams } from 'react-router-dom'; @@ -73,7 +73,7 @@ export interface TaskRunsTableProps { } // TODO: Clean this up -export type ListableWorkflowRun = V2TaskSummarySingle & { +export type ListableWorkflowRun = V2WorkflowRun & { workflowName: string | undefined; triggeredBy: string; workflowVersionId: string; @@ -94,7 +94,6 @@ export const getCreatedAfterFromTimeRange = (timeRange?: string) => { export function TaskRunsTable({ workflowId, - workerId, createdAfter: createdAfterProp, initColumnVisibility = {}, filterVisibility = {}, @@ -263,12 +262,12 @@ export function TaskRunsTable({ }, [columnFilters]); const listTasksQuery = useQuery({ - ...queries.v2Tasks.list(tenant.metadata.id, { + ...queries.v2WorkflowRuns.list(tenant.metadata.id, { offset, limit: pagination.pageSize, statuses, workflow_ids: workflow ? [workflow] : [], - worker_id: workerId, + // worker_id: workerId, since: createdAfter || new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(), @@ -280,6 +279,13 @@ export function TaskRunsTable({ refetchInterval, }); + const dagIds = listTasksQuery.data?.rows?.map((r) => r.metadata.id) || []; + + const { data: dagChildrenRaw } = useQuery({ + ...queries.v2Tasks.getByDagId(tenant.metadata.id, dagIds), + enabled: !!dagIds.length, + }); + const metricsQuery = useQuery({ ...queries.v2TaskRuns.metrics(tenant.metadata.id, { since: @@ -492,23 +498,27 @@ export function TaskRunsTable({ const data: ListableWorkflowRun[] = (listTasksQuery.data?.rows || []).map( (row) => ({ - ...row.parent, + ...row, workflowVersionId: 'first version', triggeredBy: 'manual', workflowName: workflowKeys?.rows?.find( - (r) => r.metadata.id == row.parent.workflowId, + (r) => r.metadata.id == row.workflowId, )?.name, - subRows: row.children.map((child) => ({ - ...child, - workflowVersionId: 'first version', - triggeredBy: 'manual', - workflowName: workflowKeys?.rows?.find( - (r2) => r2.metadata.id == child.workflowId, - )?.name, - })), }), ); + const dagChildren = (dagChildrenRaw || []).map((dag) => ({ + dagId: dag.dagId, + children: dag.children?.map((child) => ({ + ...child, + workflowVersionId: 'first version', + triggeredBy: 'manual', + workflowName: workflowKeys?.rows?.find( + (r2) => r2.metadata.id == child.workflowId, + )?.name, + })), + })); + return ( <> {showMetrics && ( @@ -673,6 +683,9 @@ export function TaskRunsTable({ setRowSelection={setRowSelection} pageCount={listTasksQuery.data?.pagination?.num_pages || 0} showColumnToggle={true} + getSubRows={(row) => + dagChildren.find((c) => c.dagId === row.metadata.id)?.children || [] + } /> ); diff --git a/frontend/app/src/pages/main/tasks/index.tsx b/frontend/app/src/pages/main/workflow-runs-v2/index.tsx similarity index 94% rename from frontend/app/src/pages/main/tasks/index.tsx rename to frontend/app/src/pages/main/workflow-runs-v2/index.tsx index ceb110cc37..dc935d27a9 100644 --- a/frontend/app/src/pages/main/tasks/index.tsx +++ b/frontend/app/src/pages/main/workflow-runs-v2/index.tsx @@ -1,5 +1,5 @@ import { Separator } from '@/components/ui/separator'; -import { TaskRunsTable } from './components/task-runs-table'; +import { TaskRunsTable } from './components/workflow-runs-table'; import { TriggerWorkflowForm } from '../workflows/$workflow/components/trigger-workflow-form'; import { useState } from 'react'; diff --git a/frontend/app/src/pages/main/workflow-runs/components/v2/workflow-runs-columns.tsx b/frontend/app/src/pages/main/workflow-runs/components/v2/workflow-runs-columns.tsx index 82774162f2..d3398fb148 100644 --- a/frontend/app/src/pages/main/workflow-runs/components/v2/workflow-runs-columns.tsx +++ b/frontend/app/src/pages/main/workflow-runs/components/v2/workflow-runs-columns.tsx @@ -45,7 +45,7 @@ export const columns: ( cell: ({ row }) => (
- {row.original.parent.displayName || row.original.metadata.id} + {row.original.displayName || row.original.metadata.id}
), @@ -57,7 +57,7 @@ export const columns: ( header: ({ column }) => ( ), - cell: ({ row }) => , + cell: ({ row }) => , enableSorting: false, enableHiding: false, }, @@ -67,7 +67,7 @@ export const columns: ( ), cell: ({ row }) => { - const workflowId = row.original?.parent.workflowId; + const workflowId = row.original?.workflowId; const workflowName = row.original.workflowName; return ( @@ -132,8 +132,8 @@ export const columns: ( cell: ({ row }) => { return (
- {row.original.parent.startedAt ? ( - + {row.original.startedAt ? ( + ) : ( 'N/A' )} @@ -153,8 +153,8 @@ export const columns: ( /> ), cell: ({ row }) => { - const finishedAt = row.original.parent.finishedAt ? ( - + const finishedAt = row.original.finishedAt ? ( + ) : ( 'N/A' ); @@ -174,9 +174,7 @@ export const columns: ( /> ), cell: ({ row }) => { - return ( -
{row.original.parent.duration}
- ); + return
{row.original.duration}
; }, enableSorting: true, enableHiding: true, @@ -187,13 +185,13 @@ export const columns: ( ), cell: ({ row }) => { - if (!row.original.parent.additionalMetadata) { + if (!row.original.additionalMetadata) { return
; } return ( ); diff --git a/frontend/app/src/pages/main/workflow-runs/components/workflow-runs-table.tsx b/frontend/app/src/pages/main/workflow-runs/components/workflow-runs-table.tsx index 2cdf43ff5b..e3b5042084 100644 --- a/frontend/app/src/pages/main/workflow-runs/components/workflow-runs-table.tsx +++ b/frontend/app/src/pages/main/workflow-runs/components/workflow-runs-table.tsx @@ -14,7 +14,7 @@ import api, { queries, ReplayWorkflowRunsRequest, V2TaskStatus, - V2TaskSummary, + V2WorkflowRun, } from '@/lib/api'; import { TenantContextType } from '@/lib/outlet'; import { useOutletContext, useSearchParams } from 'react-router-dom'; @@ -73,7 +73,7 @@ export interface WorkflowRunsTableProps { } // TODO: Clean this up -export type ListableWorkflowRun = V2TaskSummary & { +export type ListableWorkflowRun = V2WorkflowRun & { workflowName: string | undefined; triggeredBy: string; workflowVersionId: string; @@ -272,6 +272,8 @@ export function WorkflowRunsTable({ since: createdAfter || new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(), + additional_metadata: columnFilters.find((f) => f.id === 'Metadata') + ?.value as string[], }), placeholderData: (prev) => prev, refetchInterval, @@ -493,7 +495,7 @@ export function WorkflowRunsTable({ workflowVersionId: 'first version', triggeredBy: 'manual', workflowName: workflowKeys?.rows?.find( - (r) => r.metadata.id == row.parent.workflowId, + (r) => r.metadata.id == row.workflowId, )?.name, }), ); @@ -649,7 +651,7 @@ export function WorkflowRunsTable({ columnVisibility={columnVisibility} setColumnVisibility={setColumnVisibility} // TODO: This is a hack - fix this type - data={data as any} + data={data} filters={filters} actions={actions} sorting={sorting} diff --git a/frontend/app/src/router.tsx b/frontend/app/src/router.tsx index 5a2870e77c..5b8d707fab 100644 --- a/frontend/app/src/router.tsx +++ b/frontend/app/src/router.tsx @@ -151,13 +151,15 @@ const routes: RouteObject[] = [ }), }, { - path: '/tasks', + path: '/workflow-runs', lazy: async () => - import('./pages/main/tasks').then((res) => { - return { - Component: res.default, - }; - }), + import('./pages/main/workflow-runs-v2/index.tsx').then( + (res) => { + return { + Component: res.default, + }; + }, + ), }, { path: '/workflows', @@ -177,15 +179,6 @@ const routes: RouteObject[] = [ }; }), }, - { - path: '/workflow-runs', - lazy: async () => - import('./pages/main/workflow-runs').then((res) => { - return { - Component: res.default, - }; - }), - }, { path: '/workflow-runs/:run', lazy: async () => diff --git a/pkg/client/rest/gen.go b/pkg/client/rest/gen.go index 5cd6bff1e8..fb091d87c8 100644 --- a/pkg/client/rest/gen.go +++ b/pkg/client/rest/gen.go @@ -1143,6 +1143,12 @@ type UserTenantPublic struct { Name *string `json:"name,omitempty"` } +// V2DagChildren defines model for V2DagChildren. +type V2DagChildren struct { + Children *[]V2TaskSummary `json:"children,omitempty"` + DagId *openapi_types.UUID `json:"dagId,omitempty"` +} + // V2Task defines model for V2Task. type V2Task struct { // AdditionalMetadata Additional metadata for the task run. @@ -1229,10 +1235,38 @@ type V2TaskStatus string // V2TaskSummary defines model for V2TaskSummary. type V2TaskSummary struct { - // Children The list of child tasks - Children []V2TaskSummarySingle `json:"children"` - Metadata APIResourceMeta `json:"metadata"` - Parent V2TaskSummarySingle `json:"parent"` + // AdditionalMetadata Additional metadata for the task run. + AdditionalMetadata *map[string]interface{} `json:"additionalMetadata,omitempty"` + + // DisplayName The display name of the task run. + DisplayName string `json:"displayName"` + + // Duration The duration of the task run, in milliseconds. + Duration *int `json:"duration,omitempty"` + + // ErrorMessage The error message of the task run (for the latest run) + ErrorMessage *string `json:"errorMessage,omitempty"` + + // FinishedAt The timestamp the task run finished. + FinishedAt *time.Time `json:"finishedAt,omitempty"` + Metadata APIResourceMeta `json:"metadata"` + + // Output The output of the task run (for the latest run) + Output map[string]interface{} `json:"output"` + + // StartedAt The timestamp the task run started. + StartedAt *time.Time `json:"startedAt,omitempty"` + Status V2TaskStatus `json:"status"` + + // TaskId The ID of the task. + TaskId int `json:"taskId"` + + // TaskInsertedAt The timestamp the task was inserted. + TaskInsertedAt time.Time `json:"taskInsertedAt"` + + // TenantId The ID of the tenant. + TenantId openapi_types.UUID `json:"tenantId"` + WorkflowId openapi_types.UUID `json:"workflowId"` } // V2TaskSummaryList defines model for V2TaskSummaryList. @@ -1243,8 +1277,8 @@ type V2TaskSummaryList struct { Rows []V2TaskSummary `json:"rows"` } -// V2TaskSummarySingle defines model for V2TaskSummarySingle. -type V2TaskSummarySingle struct { +// V2WorkflowRun defines model for V2WorkflowRun. +type V2WorkflowRun struct { // AdditionalMetadata Additional metadata for the task run. AdditionalMetadata *map[string]interface{} `json:"additionalMetadata,omitempty"` @@ -1268,17 +1302,19 @@ type V2TaskSummarySingle struct { StartedAt *time.Time `json:"startedAt,omitempty"` Status V2TaskStatus `json:"status"` - // TaskId The ID of the task. - TaskId int `json:"taskId"` - - // TaskInsertedAt The timestamp the task was inserted. - TaskInsertedAt time.Time `json:"taskInsertedAt"` - // TenantId The ID of the tenant. TenantId openapi_types.UUID `json:"tenantId"` WorkflowId openapi_types.UUID `json:"workflowId"` } +// V2WorkflowRunList defines model for V2WorkflowRunList. +type V2WorkflowRunList struct { + Pagination PaginationResponse `json:"pagination"` + + // Rows The list of workflow runs + Rows []V2WorkflowRun `json:"rows"` +} + // WebhookWorker defines model for WebhookWorker. type WebhookWorker struct { Metadata APIResourceMeta `json:"metadata"` @@ -1898,6 +1934,15 @@ type WorkflowVersionGetParams struct { Version *openapi_types.UUID `form:"version,omitempty" json:"version,omitempty"` } +// V2DagListTasksParams defines parameters for V2DagListTasks. +type V2DagListTasksParams struct { + // DagIds The external id of the DAG + DagIds []openapi_types.UUID `form:"dag_ids" json:"dag_ids"` + + // Tenant The tenant id + Tenant openapi_types.UUID `form:"tenant" json:"tenant"` +} + // V2TaskEventListParams defines parameters for V2TaskEventList. type V2TaskEventListParams struct { // Offset The number to skip @@ -1952,6 +1997,30 @@ type V2TaskListParams struct { WorkerId *openapi_types.UUID `form:"worker_id,omitempty" json:"worker_id,omitempty"` } +// V2WorkflowRunListParams defines parameters for V2WorkflowRunList. +type V2WorkflowRunListParams struct { + // Offset The number to skip + Offset *int64 `form:"offset,omitempty" json:"offset,omitempty"` + + // Limit The number to limit by + Limit *int64 `form:"limit,omitempty" json:"limit,omitempty"` + + // Statuses A list of statuses to filter by + Statuses *[]V2TaskStatus `form:"statuses,omitempty" json:"statuses,omitempty"` + + // Since The earliest date to filter by + Since time.Time `form:"since" json:"since"` + + // Until The latest date to filter by + Until *time.Time `form:"until,omitempty" json:"until,omitempty"` + + // AdditionalMetadata Additional metadata k-v pairs to filter by + AdditionalMetadata *[]string `form:"additional_metadata,omitempty" json:"additional_metadata,omitempty"` + + // WorkflowIds The workflow ids to find runs for + WorkflowIds *[]openapi_types.UUID `form:"workflow_ids,omitempty" json:"workflow_ids,omitempty"` +} + // AlertEmailGroupUpdateJSONRequestBody defines body for AlertEmailGroupUpdate for application/json ContentType. type AlertEmailGroupUpdateJSONRequestBody = UpdateTenantAlertEmailGroupRequest @@ -2437,6 +2506,9 @@ type ClientInterface interface { // WorkflowVersionGet request WorkflowVersionGet(ctx context.Context, workflow openapi_types.UUID, params *WorkflowVersionGetParams, reqEditors ...RequestEditorFn) (*http.Response, error) + // V2DagListTasks request + V2DagListTasks(ctx context.Context, params *V2DagListTasksParams, reqEditors ...RequestEditorFn) (*http.Response, error) + // V2TaskGet request V2TaskGet(ctx context.Context, task openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -2451,6 +2523,9 @@ type ClientInterface interface { // V2TaskList request V2TaskList(ctx context.Context, tenant openapi_types.UUID, params *V2TaskListParams, reqEditors ...RequestEditorFn) (*http.Response, error) + + // V2WorkflowRunList request + V2WorkflowRunList(ctx context.Context, tenant openapi_types.UUID, params *V2WorkflowRunListParams, reqEditors ...RequestEditorFn) (*http.Response, error) } func (c *Client) LivenessGet(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) { @@ -3893,6 +3968,18 @@ func (c *Client) WorkflowVersionGet(ctx context.Context, workflow openapi_types. return c.Client.Do(req) } +func (c *Client) V2DagListTasks(ctx context.Context, params *V2DagListTasksParams, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewV2DagListTasksRequest(c.Server, params) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) V2TaskGet(ctx context.Context, task openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewV2TaskGetRequest(c.Server, task) if err != nil { @@ -3953,6 +4040,18 @@ func (c *Client) V2TaskList(ctx context.Context, tenant openapi_types.UUID, para return c.Client.Do(req) } +func (c *Client) V2WorkflowRunList(ctx context.Context, tenant openapi_types.UUID, params *V2WorkflowRunListParams, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewV2WorkflowRunListRequest(c.Server, tenant, params) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + // NewLivenessGetRequest generates requests for LivenessGet func NewLivenessGetRequest(server string) (*http.Request, error) { var err error @@ -8715,6 +8814,63 @@ func NewWorkflowVersionGetRequest(server string, workflow openapi_types.UUID, pa return req, nil } +// NewV2DagListTasksRequest generates requests for V2DagListTasks +func NewV2DagListTasksRequest(server string, params *V2DagListTasksParams) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/api/v2/dags/tasks") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + if params != nil { + queryValues := queryURL.Query() + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "dag_ids", runtime.ParamLocationQuery, params.DagIds); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "tenant", runtime.ParamLocationQuery, params.Tenant); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + queryURL.RawQuery = queryValues.Encode() + } + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + // NewV2TaskGetRequest generates requests for V2TaskGet func NewV2TaskGetRequest(server string, task openapi_types.UUID) (*http.Request, error) { var err error @@ -9125,6 +9281,154 @@ func NewV2TaskListRequest(server string, tenant openapi_types.UUID, params *V2Ta return req, nil } +// NewV2WorkflowRunListRequest generates requests for V2WorkflowRunList +func NewV2WorkflowRunListRequest(server string, tenant openapi_types.UUID, params *V2WorkflowRunListParams) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "tenant", runtime.ParamLocationPath, tenant) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/api/v2/tenants/%s/workflow-runs", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + if params != nil { + queryValues := queryURL.Query() + + if params.Offset != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "offset", runtime.ParamLocationQuery, *params.Offset); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + if params.Limit != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "limit", runtime.ParamLocationQuery, *params.Limit); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + if params.Statuses != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "statuses", runtime.ParamLocationQuery, *params.Statuses); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "since", runtime.ParamLocationQuery, params.Since); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + if params.Until != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "until", runtime.ParamLocationQuery, *params.Until); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + if params.AdditionalMetadata != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "additional_metadata", runtime.ParamLocationQuery, *params.AdditionalMetadata); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + if params.WorkflowIds != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "workflow_ids", runtime.ParamLocationQuery, *params.WorkflowIds); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + queryURL.RawQuery = queryValues.Encode() + } + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + func (c *Client) applyEditors(ctx context.Context, req *http.Request, additionalEditors []RequestEditorFn) error { for _, r := range c.RequestEditors { if err := r(ctx, req); err != nil { @@ -9502,6 +9806,9 @@ type ClientWithResponsesInterface interface { // WorkflowVersionGetWithResponse request WorkflowVersionGetWithResponse(ctx context.Context, workflow openapi_types.UUID, params *WorkflowVersionGetParams, reqEditors ...RequestEditorFn) (*WorkflowVersionGetResponse, error) + // V2DagListTasksWithResponse request + V2DagListTasksWithResponse(ctx context.Context, params *V2DagListTasksParams, reqEditors ...RequestEditorFn) (*V2DagListTasksResponse, error) + // V2TaskGetWithResponse request V2TaskGetWithResponse(ctx context.Context, task openapi_types.UUID, reqEditors ...RequestEditorFn) (*V2TaskGetResponse, error) @@ -9516,6 +9823,9 @@ type ClientWithResponsesInterface interface { // V2TaskListWithResponse request V2TaskListWithResponse(ctx context.Context, tenant openapi_types.UUID, params *V2TaskListParams, reqEditors ...RequestEditorFn) (*V2TaskListResponse, error) + + // V2WorkflowRunListWithResponse request + V2WorkflowRunListWithResponse(ctx context.Context, tenant openapi_types.UUID, params *V2WorkflowRunListParams, reqEditors ...RequestEditorFn) (*V2WorkflowRunListResponse, error) } type LivenessGetResponse struct { @@ -11777,6 +12087,30 @@ func (r WorkflowVersionGetResponse) StatusCode() int { return 0 } +type V2DagListTasksResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *[]V2DagChildren + JSON400 *APIErrors + JSON403 *APIErrors +} + +// Status returns HTTPResponse.Status +func (r V2DagListTasksResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r V2DagListTasksResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type V2TaskGetResponse struct { Body []byte HTTPResponse *http.Response @@ -11899,6 +12233,30 @@ func (r V2TaskListResponse) StatusCode() int { return 0 } +type V2WorkflowRunListResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *V2WorkflowRunList + JSON400 *APIErrors + JSON403 *APIErrors +} + +// Status returns HTTPResponse.Status +func (r V2WorkflowRunListResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r V2WorkflowRunListResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + // LivenessGetWithResponse request returning *LivenessGetResponse func (c *ClientWithResponses) LivenessGetWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*LivenessGetResponse, error) { rsp, err := c.LivenessGet(ctx, reqEditors...) @@ -12953,6 +13311,15 @@ func (c *ClientWithResponses) WorkflowVersionGetWithResponse(ctx context.Context return ParseWorkflowVersionGetResponse(rsp) } +// V2DagListTasksWithResponse request returning *V2DagListTasksResponse +func (c *ClientWithResponses) V2DagListTasksWithResponse(ctx context.Context, params *V2DagListTasksParams, reqEditors ...RequestEditorFn) (*V2DagListTasksResponse, error) { + rsp, err := c.V2DagListTasks(ctx, params, reqEditors...) + if err != nil { + return nil, err + } + return ParseV2DagListTasksResponse(rsp) +} + // V2TaskGetWithResponse request returning *V2TaskGetResponse func (c *ClientWithResponses) V2TaskGetWithResponse(ctx context.Context, task openapi_types.UUID, reqEditors ...RequestEditorFn) (*V2TaskGetResponse, error) { rsp, err := c.V2TaskGet(ctx, task, reqEditors...) @@ -12998,6 +13365,15 @@ func (c *ClientWithResponses) V2TaskListWithResponse(ctx context.Context, tenant return ParseV2TaskListResponse(rsp) } +// V2WorkflowRunListWithResponse request returning *V2WorkflowRunListResponse +func (c *ClientWithResponses) V2WorkflowRunListWithResponse(ctx context.Context, tenant openapi_types.UUID, params *V2WorkflowRunListParams, reqEditors ...RequestEditorFn) (*V2WorkflowRunListResponse, error) { + rsp, err := c.V2WorkflowRunList(ctx, tenant, params, reqEditors...) + if err != nil { + return nil, err + } + return ParseV2WorkflowRunListResponse(rsp) +} + // ParseLivenessGetResponse parses an HTTP response from a LivenessGetWithResponse call func ParseLivenessGetResponse(rsp *http.Response) (*LivenessGetResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -16719,6 +17095,46 @@ func ParseWorkflowVersionGetResponse(rsp *http.Response) (*WorkflowVersionGetRes return response, nil } +// ParseV2DagListTasksResponse parses an HTTP response from a V2DagListTasksWithResponse call +func ParseV2DagListTasksResponse(rsp *http.Response) (*V2DagListTasksResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &V2DagListTasksResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest []V2DagChildren + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest APIErrors + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 403: + var dest APIErrors + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON403 = &dest + + } + + return response, nil +} + // ParseV2TaskGetResponse parses an HTTP response from a V2TaskGetWithResponse call func ParseV2TaskGetResponse(rsp *http.Response) (*V2TaskGetResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -16932,3 +17348,43 @@ func ParseV2TaskListResponse(rsp *http.Response) (*V2TaskListResponse, error) { return response, nil } + +// ParseV2WorkflowRunListResponse parses an HTTP response from a V2WorkflowRunListWithResponse call +func ParseV2WorkflowRunListResponse(rsp *http.Response) (*V2WorkflowRunListResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &V2WorkflowRunListResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest V2WorkflowRunList + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest APIErrors + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 403: + var dest APIErrors + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON403 = &dest + + } + + return response, nil +} diff --git a/pkg/repository/olap.go b/pkg/repository/olap.go index 6e046410a5..1a3ad6e438 100644 --- a/pkg/repository/olap.go +++ b/pkg/repository/olap.go @@ -48,16 +48,50 @@ type ListTaskRunOpts struct { Offset int64 } +type ListWorkflowRunOpts struct { + CreatedAfter time.Time + + Statuses []gen.V2TaskStatus + + WorkflowIds []uuid.UUID + + StartedAfter time.Time + + FinishedBefore *time.Time + + AdditionalMetadata map[string]interface{} + + Limit int64 + + Offset int64 +} + type ReadTaskRunMetricsOpts struct { CreatedAfter time.Time WorkflowIds []uuid.UUID } +type WorkflowRunData struct { + TenantID pgtype.UUID `json:"tenant_id"` + InsertedAt pgtype.Timestamptz `json:"inserted_at"` + ExternalID pgtype.UUID `json:"external_id"` + ReadableStatus timescalev2.V2ReadableStatusOlap `json:"readable_status"` + Kind timescalev2.V2RunKind `json:"kind"` + WorkflowID pgtype.UUID `json:"workflow_id"` + DisplayName string `json:"display_name"` + AdditionalMetadata []byte `json:"additional_metadata"` + CreatedAt pgtype.Timestamptz `json:"created_at"` + StartedAt pgtype.Timestamptz `json:"started_at"` + FinishedAt pgtype.Timestamptz `json:"finished_at"` + ErrorMessage string `json:"error_message"` +} + type OLAPEventRepository interface { ReadTaskRun(ctx context.Context, taskExternalId string) (*timescalev2.V2TasksOlap, error) ReadTaskRunData(ctx context.Context, tenantId pgtype.UUID, taskId int64, taskInsertedAt pgtype.Timestamptz) (*timescalev2.PopulateSingleTaskRunDataRow, error) - ListTaskRuns(ctx context.Context, tenantId string, opts ListTaskRunOpts) ([]*olap.TaskRunDataRow, int, error) + ListTasks(ctx context.Context, tenantId string, opts ListTaskRunOpts) ([]*timescalev2.PopulateTaskRunDataRow, int, error) + ListWorkflowRuns(ctx context.Context, tenantId string, opts ListWorkflowRunOpts) ([]*WorkflowRunData, int, error) ListTaskRunEvents(ctx context.Context, tenantId string, taskId int64, taskInsertedAt pgtype.Timestamptz, limit, offset int64) ([]*timescalev2.ListTaskEventsRow, error) ReadTaskRunMetrics(ctx context.Context, tenantId string, opts ReadTaskRunMetricsOpts) ([]olap.TaskRunMetric, error) CreateTasks(ctx context.Context, tenantId string, tasks []*sqlcv2.V2Task) error @@ -66,6 +100,8 @@ type OLAPEventRepository interface { GetTaskPointMetrics(ctx context.Context, tenantId string, startTimestamp *time.Time, endTimestamp *time.Time, bucketInterval time.Duration) ([]*timescalev2.GetTaskPointMetricsRow, error) UpdateTaskStatuses(ctx context.Context, tenantId string) (bool, error) UpdateDAGStatuses(ctx context.Context, tenantId string) (bool, error) + ReadDAG(ctx context.Context, dagExternalId string) (*timescalev2.V2DagsOlap, error) + ListTasksByDAGId(ctx context.Context, tenantId string, dagIds []pgtype.UUID) ([]*timescalev2.PopulateTaskRunDataRow, error) } type olapEventRepository struct { @@ -303,7 +339,7 @@ func uniq[T comparable](arr []T) []T { return uniq } -func (r *olapEventRepository) ListTaskRuns(ctx context.Context, tenantId string, opts ListTaskRunOpts) ([]*olap.TaskRunDataRow, int, error) { +func (r *olapEventRepository) ListTasks(ctx context.Context, tenantId string, opts ListTaskRunOpts) ([]*timescalev2.PopulateTaskRunDataRow, int, error) { tx, err := r.pool.Begin(ctx) if err != nil { @@ -312,35 +348,49 @@ func (r *olapEventRepository) ListTaskRuns(ctx context.Context, tenantId string, defer tx.Rollback(ctx) + params := timescalev2.ListTasksParams{ + Tenantid: sqlchelpers.UUIDFromStr(tenantId), + Since: sqlchelpers.TimestamptzFromTime(opts.CreatedAfter), + Tasklimit: int32(opts.Limit), + Taskoffset: int32(opts.Offset), + } + + countParams := timescalev2.CountTasksParams{ + Tenantid: sqlchelpers.UUIDFromStr(tenantId), + Since: sqlchelpers.TimestamptzFromTime(opts.CreatedAfter), + } + statuses := make([]string, 0) + for _, status := range opts.Statuses { statuses = append(statuses, string(status)) } - var workflowIdParams []pgtype.UUID + if len(statuses) != 0 && len(statuses) != 5 { + params.Statuses = statuses + countParams.Statuses = statuses + } if len(opts.WorkflowIds) > 0 { - workflowIdParams = make([]pgtype.UUID, 0) + workflowIdParams := make([]pgtype.UUID, 0) for _, id := range opts.WorkflowIds { workflowIdParams = append(workflowIdParams, sqlchelpers.UUIDFromStr(id.String())) } - } - params := timescalev2.ListWorkflowRunsParams{ - WorkflowIds: workflowIdParams, - Statuses: statuses, - Since: sqlchelpers.TimestamptzFromTime(opts.CreatedAfter), - Listworkflowrunsoffset: int32(opts.Offset), - Listworkflowrunslimit: int32(opts.Limit), + params.WorkflowIds = workflowIdParams + countParams.WorkflowIds = workflowIdParams } until := opts.FinishedBefore + if until != nil { params.Until = sqlchelpers.TimestamptzFromTime(*until) + countParams.Until = sqlchelpers.TimestamptzFromTime(*until) } workerId := opts.WorkerId + if workerId != nil { params.WorkerId = sqlchelpers.UUIDFromStr(workerId.String()) } @@ -348,78 +398,282 @@ func (r *olapEventRepository) ListTaskRuns(ctx context.Context, tenantId string, for key, value := range opts.AdditionalMetadata { params.Keys = append(params.Keys, key) params.Values = append(params.Values, value.(string)) + countParams.Keys = append(countParams.Keys, key) + countParams.Values = append(countParams.Values, value.(string)) } - rows, err := r.queries.ListWorkflowRuns(ctx, r.pool, params) + rows, err := r.queries.ListTasks(ctx, tx, params) if err != nil { return nil, 0, err } - dagIds := make([]int64, 0) + taskIds := make([]int64, 0) + taskInsertedAts := make([]pgtype.Timestamptz, 0) for _, row := range rows { - dagIds = append(dagIds, row.DagID.Int64) + taskIds = append(taskIds, row.ID) + taskInsertedAts = append(taskInsertedAts, row.InsertedAt) } - dagIds = uniq(dagIds) - - children, err := r.queries.ListDAGChildren(ctx, r.pool, dagIds) + tasksWithData, err := r.queries.PopulateTaskRunData(ctx, tx, timescalev2.PopulateTaskRunDataParams{ + Taskids: taskIds, + Taskinsertedats: taskInsertedAts, + Tenantid: sqlchelpers.UUIDFromStr(tenantId), + }) if err != nil && !errors.Is(err, pgx.ErrNoRows) { return nil, 0, err } - if err := tx.Commit(context.Background()); err != nil { + count, err := r.queries.CountTasks(ctx, r.pool, countParams) + + if err != nil { + count = int64(len(tasksWithData)) + } + + if err := tx.Commit(ctx); err != nil { return nil, 0, err } - records := make([]*olap.TaskRunDataRow, 0) - for _, row := range rows { - parent := ×calev2.ListWorkflowRunsRow{ - TenantID: row.TenantID, - RunID: row.RunID, - InsertedAt: row.InsertedAt, - ExternalID: row.ExternalID, - WorkflowID: row.WorkflowID, - DisplayName: row.DisplayName, - AdditionalMetadata: row.AdditionalMetadata, - ReadableStatus: row.ReadableStatus, - FinishedAt: row.FinishedAt, - StartedAt: row.StartedAt, - Output: row.Output, - ErrorMessage: row.ErrorMessage, - } + return tasksWithData, int(count), nil +} - rowChildren := make([]*timescalev2.ListDAGChildrenRow, 0) +func (r *olapEventRepository) ListTasksByDAGId(ctx context.Context, tenantId string, dagids []pgtype.UUID) ([]*timescalev2.PopulateTaskRunDataRow, error) { + tx, err := r.pool.Begin(ctx) - for _, child := range children { - if child.DagID == row.DagID.Int64 { - rowChildren = append(rowChildren, child) - } + if err != nil { + return nil, err + } + + defer tx.Rollback(ctx) + + tasks, err := r.queries.ListTasksByDAGIds(ctx, tx, timescalev2.ListTasksByDAGIdsParams{ + Dagids: dagids, + Tenantid: sqlchelpers.UUIDFromStr(tenantId), + }) + + if err != nil { + return nil, err + } + + taskIds := make([]int64, 0) + taskInsertedAts := make([]pgtype.Timestamptz, 0) + + for _, row := range tasks { + taskIds = append(taskIds, row.TaskID) + taskInsertedAts = append(taskInsertedAts, row.TaskInsertedAt) + } + + tasksWithData, err := r.queries.PopulateTaskRunData(ctx, tx, timescalev2.PopulateTaskRunDataParams{ + Taskids: taskIds, + Taskinsertedats: taskInsertedAts, + Tenantid: sqlchelpers.UUIDFromStr(tenantId), + }) + + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return nil, err + } + + if err := tx.Commit(ctx); err != nil { + return nil, err + } + + return tasksWithData, nil +} + +func (r *olapEventRepository) ListWorkflowRuns(ctx context.Context, tenantId string, opts ListWorkflowRunOpts) ([]*WorkflowRunData, int, error) { + tx, err := r.pool.Begin(ctx) + + if err != nil { + return nil, 0, err + } + + defer tx.Rollback(ctx) + + params := timescalev2.FetchWorkflowRunIdsParams{ + Tenantid: sqlchelpers.UUIDFromStr(tenantId), + Since: sqlchelpers.TimestamptzFromTime(opts.CreatedAfter), + Listworkflowrunslimit: int32(opts.Limit), + Listworkflowrunsoffset: int32(opts.Offset), + } + + countParams := timescalev2.CountWorkflowRunsParams{ + Tenantid: sqlchelpers.UUIDFromStr(tenantId), + Since: sqlchelpers.TimestamptzFromTime(opts.CreatedAfter), + } + + statuses := make([]string, 0) + + for _, status := range opts.Statuses { + statuses = append(statuses, string(status)) + } + + if len(statuses) != 0 && len(statuses) != 5 { + params.Statuses = statuses + countParams.Statuses = statuses + } + + if len(opts.WorkflowIds) > 0 { + workflowIdParams := make([]pgtype.UUID, 0) + + for _, id := range opts.WorkflowIds { + workflowIdParams = append(workflowIdParams, sqlchelpers.UUIDFromStr(id.String())) } - record := &olap.TaskRunDataRow{ - Parent: parent, - Children: rowChildren, + params.WorkflowIds = workflowIdParams + countParams.WorkflowIds = workflowIdParams + } + + until := opts.FinishedBefore + + if until != nil { + params.Until = sqlchelpers.TimestamptzFromTime(*until) + countParams.Until = sqlchelpers.TimestamptzFromTime(*until) + } + + for key, value := range opts.AdditionalMetadata { + params.Keys = append(params.Keys, key) + params.Values = append(params.Values, value.(string)) + countParams.Keys = append(countParams.Keys, key) + countParams.Values = append(countParams.Values, value.(string)) + } + + workflowRunIds, err := r.queries.FetchWorkflowRunIds(ctx, tx, params) + + if err != nil { + return nil, 0, err + } + + runIdsWithDAGs := make([]int64, 0) + runInsertedAtsWithDAGs := make([]pgtype.Timestamptz, 0) + taskExternalIds := make([]pgtype.UUID, 0) + + for _, row := range workflowRunIds { + if row.Kind == timescalev2.V2RunKindDAG { + runIdsWithDAGs = append(runIdsWithDAGs, row.ID) + runInsertedAtsWithDAGs = append(runInsertedAtsWithDAGs, row.InsertedAt) + } else { + taskExternalIds = append(taskExternalIds, row.ExternalID) } + } + + populatedDAGs, err := r.queries.PopulateDAGMetadata(ctx, tx, timescalev2.PopulateDAGMetadataParams{ + Ids: runIdsWithDAGs, + Insertedats: runInsertedAtsWithDAGs, + Tenantid: sqlchelpers.UUIDFromStr(tenantId), + }) + + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return nil, 0, err + } + + dagsToPopulated := make(map[string]*timescalev2.PopulateDAGMetadataRow) + + for _, dag := range populatedDAGs { + externalId := sqlchelpers.UUIDToStr(dag.ExternalID) + + dagsToPopulated[externalId] = dag + } + + tasksWithIds, err := r.queries.ListTasksByExternalIds(ctx, tx, timescalev2.ListTasksByExternalIdsParams{ + Externalids: taskExternalIds, + Tenantid: sqlchelpers.UUIDFromStr(tenantId), + }) + + if err != nil { + return nil, 0, err + } - records = append(records, record) + taskIds := make([]int64, 0) + taskInsertedAts := make([]pgtype.Timestamptz, 0) + + for _, row := range tasksWithIds { + taskIds = append(taskIds, row.TaskID.Int64) + taskInsertedAts = append(taskInsertedAts, row.InsertedAt) } - countParams := timescalev2.CountRunsParams{ - WorkflowIds: workflowIdParams, - Statuses: statuses, - Since: sqlchelpers.TimestamptzFromTime(opts.CreatedAfter), + populatedTasks, err := r.queries.PopulateTaskRunData(ctx, tx, timescalev2.PopulateTaskRunDataParams{ + Taskids: taskIds, + Taskinsertedats: taskInsertedAts, + Tenantid: sqlchelpers.UUIDFromStr(tenantId), + }) + + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return nil, 0, err } - count, err := r.queries.CountRuns(ctx, r.pool, countParams) + tasksToPopulated := make(map[string]*timescalev2.PopulateTaskRunDataRow) + + for _, task := range populatedTasks { + externalId := sqlchelpers.UUIDToStr(task.ExternalID) + tasksToPopulated[externalId] = task + } + + count, err := r.queries.CountWorkflowRuns(ctx, r.pool, countParams) if err != nil { - count = int64(len(records)) + r.l.Error().Msgf("error counting workflow runs: %v", err) + count = int64(len(workflowRunIds)) } - return records, int(count), nil + if err := tx.Commit(ctx); err != nil { + return nil, 0, err + } + + res := make([]*WorkflowRunData, 0) + + for _, row := range workflowRunIds { + externalId := sqlchelpers.UUIDToStr(row.ExternalID) + + if row.Kind == timescalev2.V2RunKindDAG { + dag, ok := dagsToPopulated[externalId] + + if !ok { + r.l.Error().Msgf("could not find dag with external id %s", externalId) + continue + } + + res = append(res, &WorkflowRunData{ + TenantID: dag.TenantID, + InsertedAt: dag.InsertedAt, + ExternalID: dag.ExternalID, + WorkflowID: dag.WorkflowID, + DisplayName: dag.DisplayName, + ReadableStatus: dag.ReadableStatus, + AdditionalMetadata: dag.AdditionalMetadata, + CreatedAt: dag.CreatedAt, + StartedAt: dag.StartedAt, + FinishedAt: dag.FinishedAt, + ErrorMessage: dag.ErrorMessage.String, + Kind: timescalev2.V2RunKindDAG, + }) + } else { + task, ok := tasksToPopulated[externalId] + + if !ok { + r.l.Error().Msgf("could not find task with external id %s", externalId) + continue + } + + res = append(res, &WorkflowRunData{ + TenantID: task.TenantID, + InsertedAt: task.InsertedAt, + ExternalID: task.ExternalID, + WorkflowID: task.WorkflowID, + DisplayName: task.DisplayName, + ReadableStatus: task.Status, + AdditionalMetadata: task.AdditionalMetadata, + CreatedAt: task.InsertedAt, + StartedAt: task.StartedAt, + FinishedAt: task.FinishedAt, + ErrorMessage: task.ErrorMessage.String, + Kind: timescalev2.V2RunKindTASK, + }) + } + } + + return res, int(count), nil } func (r *olapEventRepository) ListTaskRunEvents(ctx context.Context, tenantId string, taskId int64, taskInsertedAt pgtype.Timestamptz, limit, offset int64) ([]*timescalev2.ListTaskEventsRow, error) { @@ -737,6 +991,10 @@ func (r *olapEventRepository) GetTaskPointMetrics(ctx context.Context, tenantId return rows, nil } +func (r *olapEventRepository) ReadDAG(ctx context.Context, dagExternalId string) (*timescalev2.V2DagsOlap, error) { + return r.queries.ReadDAGByExternalID(ctx, r.pool, sqlchelpers.UUIDFromStr(dagExternalId)) +} + func durationToPgInterval(d time.Duration) pgtype.Interval { // Convert the time.Duration to microseconds microseconds := d.Microseconds() diff --git a/pkg/repository/olap/types.go b/pkg/repository/olap/types.go index 70868357bf..389880a1f9 100644 --- a/pkg/repository/olap/types.go +++ b/pkg/repository/olap/types.go @@ -4,7 +4,6 @@ import ( "time" "github.com/google/uuid" - "github.com/hatchet-dev/hatchet/pkg/repository/v2/timescalev2" ) // type WorkflowRun struct { @@ -140,8 +139,3 @@ type TaskRunMetric struct { Status string `json:"status"` Count uint64 `json:"count"` } - -type TaskRunDataRow struct { - Parent *timescalev2.ListWorkflowRunsRow - Children []*timescalev2.ListDAGChildrenRow -} diff --git a/pkg/repository/v2/timescalev2/models.go b/pkg/repository/v2/timescalev2/models.go index b30e377322..85e95df64e 100644 --- a/pkg/repository/v2/timescalev2/models.go +++ b/pkg/repository/v2/timescalev2/models.go @@ -314,13 +314,14 @@ type V2LookupTable struct { } type V2RunsOlap struct { - TenantID pgtype.UUID `json:"tenant_id"` - ID int64 `json:"id"` - InsertedAt pgtype.Timestamptz `json:"inserted_at"` - ExternalID pgtype.UUID `json:"external_id"` - ReadableStatus V2ReadableStatusOlap `json:"readable_status"` - Kind V2RunKind `json:"kind"` - WorkflowID pgtype.UUID `json:"workflow_id"` + TenantID pgtype.UUID `json:"tenant_id"` + ID int64 `json:"id"` + InsertedAt pgtype.Timestamptz `json:"inserted_at"` + ExternalID pgtype.UUID `json:"external_id"` + ReadableStatus V2ReadableStatusOlap `json:"readable_status"` + Kind V2RunKind `json:"kind"` + WorkflowID pgtype.UUID `json:"workflow_id"` + AdditionalMetadata []byte `json:"additional_metadata"` } type V2StatusesOlap struct { diff --git a/pkg/repository/v2/timescalev2/queries.sql b/pkg/repository/v2/timescalev2/queries.sql index 09baf802a2..afe4c6c1f0 100644 --- a/pkg/repository/v2/timescalev2/queries.sql +++ b/pkg/repository/v2/timescalev2/queries.sql @@ -53,7 +53,8 @@ FROM get_v2_partitions_before_date( 'v2_runs_olap'::text, @date::date - ) AS p; + ) AS p +; -- name: CreateTasksOLAP :copyfrom INSERT INTO v2_tasks_olap ( @@ -207,6 +208,47 @@ JOIN v2_task_events_olap e ON (e.tenant_id, e.task_id, e.readable_status, e.retry_count) = (t.tenant_id, t.id, t.readable_status, t.latest_retry_count) ; +-- name: ListTasksByExternalIds :many +SELECT + tenant_id, + task_id, + inserted_at +FROM + v2_lookup_table +WHERE + external_id = ANY(@externalIds::uuid[]) + AND tenant_id = @tenantId::uuid; + +-- name: ListTasksByDAGIds :many +SELECT + dt.* +FROM + v2_lookup_table lt +JOIN + v2_dag_to_task_olap dt ON lt.dag_id = dt.dag_id +WHERE + lt.external_id = ANY(@dagIds::uuid[]) + AND tenant_id = @tenantId::uuid +; + +-- name: ReadDAGByExternalID :one +WITH lookup_task AS ( + SELECT + tenant_id, + dag_id, + inserted_at + FROM + v2_lookup_table + WHERE + external_id = @externalId::uuid +) +SELECT + d.* +FROM + v2_dags_olap d +JOIN + lookup_task lt ON lt.tenant_id = d.tenant_id AND lt.dag_id = d.id AND lt.inserted_at = d.inserted_at; + -- name: ListTaskEvents :many WITH aggregated_events AS ( SELECT @@ -253,12 +295,17 @@ ORDER BY a.time_first_seen DESC, t.event_timestamp DESC; -- name: ListTasks :many SELECT - * + id, + inserted_at FROM v2_tasks_olap WHERE tenant_id = @tenantId::uuid - AND inserted_at >= @insertedAfter::timestamptz + AND inserted_at >= @since::timestamptz + AND ( + sqlc.narg('until')::timestamptz IS NULL + OR inserted_at <= sqlc.narg('until')::timestamptz + ) AND ( sqlc.narg('statuses')::text[] IS NULL OR readable_status = ANY(cast(sqlc.narg('statuses')::text[] as v2_readable_status_olap[])) ) @@ -268,9 +315,57 @@ WHERE AND ( sqlc.narg('workerId')::uuid IS NULL OR latest_worker_id = sqlc.narg('workerId')::uuid ) + AND ( + sqlc.narg('keys')::text[] IS NULL + OR sqlc.narg('values')::text[] IS NULL + OR EXISTS ( + SELECT 1 FROM jsonb_each_text(additional_metadata) kv + JOIN LATERAL ( + SELECT unnest(sqlc.narg('keys')::text[]) AS k, + unnest(sqlc.narg('values')::text[]) AS v + ) AS u ON kv.key = u.k AND kv.value = u.v + ) + ) ORDER BY inserted_at DESC -LIMIT @taskLimit::integer; +LIMIT @taskLimit::integer +OFFSET @taskOffset::integer; + +-- name: CountTasks :one +SELECT + COUNT(*) +FROM + v2_tasks_olap +WHERE + tenant_id = @tenantId::uuid + AND inserted_at >= @since::timestamptz + AND ( + sqlc.narg('until')::timestamptz IS NULL + OR inserted_at <= sqlc.narg('until')::timestamptz + ) + AND ( + sqlc.narg('statuses')::text[] IS NULL OR readable_status = ANY(cast(sqlc.narg('statuses')::text[] as v2_readable_status_olap[])) + ) + AND ( + sqlc.narg('workflowIds')::uuid[] IS NULL OR workflow_id = ANY(sqlc.narg('workflowIds')::uuid[]) + ) + AND ( + sqlc.narg('workerId')::uuid IS NULL OR latest_worker_id = sqlc.narg('workerId')::uuid + ) + AND ( + sqlc.narg('keys')::text[] IS NULL + OR sqlc.narg('values')::text[] IS NULL + OR EXISTS ( + SELECT 1 FROM jsonb_each_text(additional_metadata) kv + JOIN LATERAL ( + SELECT unnest(sqlc.narg('keys')::text[]) AS k, + unnest(sqlc.narg('values')::text[]) AS v + ) AS u ON kv.key = u.k AND kv.value = u.v + ) + ) +ORDER BY + inserted_at DESC +LIMIT 20000; -- name: PopulateSingleTaskRunData :one WITH latest_retry_count AS ( @@ -359,16 +454,14 @@ WHERE -- name: PopulateTaskRunData :many WITH input AS ( SELECT - UNNEST(@tenantIds::uuid[]) AS tenant_id, UNNEST(@taskIds::bigint[]) AS id, - UNNEST(@taskInsertedAts::timestamptz[]) AS inserted_at, - UNNEST(@retryCounts::int[]) AS retry_count, - unnest(cast(@statuses::text[] as v2_readable_status_olap[])) AS status + UNNEST(@taskInsertedAts::timestamptz[]) AS inserted_at ), tasks AS ( SELECT DISTINCT ON(t.tenant_id, t.id, t.inserted_at) t.tenant_id, t.id, + d.external_id AS dag_external_id, t.inserted_at, t.queue, t.action_id, @@ -383,20 +476,51 @@ WITH input AS ( t.display_name, t.input, t.additional_metadata, - i.retry_count, - i.status + t.readable_status FROM v2_tasks_olap t JOIN - input i ON i.tenant_id = t.tenant_id AND i.id = t.id AND i.inserted_at = t.inserted_at + input i ON i.id = t.id AND i.inserted_at = t.inserted_at + LEFT JOIN + v2_dag_to_task_olap dtt ON dtt.task_id = t.id + LEFT JOIN + v2_dags_olap d ON d.id = dtt.dag_id AND d.tenant_id = t.tenant_id + + WHERE + t.tenant_id = @tenantId::uuid +), relevant_events AS ( + SELECT + e.* + FROM + v2_task_events_olap e + JOIN + tasks t ON t.id = e.task_id AND t.tenant_id = e.tenant_id AND t.inserted_at = e.task_inserted_at + WHERE + e.tenant_id = @tenantId::uuid + AND e.task_id = ANY(@taskIds::bigint[]) + AND e.task_inserted_at = ANY(@taskInsertedAts::timestamptz[]) +), max_retry_counts AS ( + SELECT + e.tenant_id, + e.task_id, + e.task_inserted_at, + MAX(e.retry_count) AS max_retry_count + FROM + relevant_events e + GROUP BY + e.tenant_id, e.task_id, e.task_inserted_at ), finished_ats AS ( SELECT e.task_id::bigint, MAX(e.event_timestamp) AS finished_at FROM - v2_task_events_olap e + relevant_events e JOIN - tasks t ON t.id = e.task_id AND t.tenant_id = e.tenant_id AND t.inserted_at = e.task_inserted_at AND t.retry_count = e.retry_count + max_retry_counts mrc ON + e.tenant_id = mrc.tenant_id + AND e.task_id = mrc.task_id + AND e.task_inserted_at = mrc.task_inserted_at + AND e.retry_count = mrc.max_retry_count WHERE e.readable_status = ANY(ARRAY['COMPLETED', 'FAILED', 'CANCELLED']::v2_readable_status_olap[]) GROUP BY e.task_id @@ -405,16 +529,37 @@ WITH input AS ( e.task_id::bigint, MAX(e.event_timestamp) AS started_at FROM - v2_task_events_olap e + relevant_events e JOIN - tasks t ON t.id = e.task_id AND t.tenant_id = e.tenant_id AND t.inserted_at = e.task_inserted_at AND t.retry_count = e.retry_count + max_retry_counts mrc ON + e.tenant_id = mrc.tenant_id + AND e.task_id = mrc.task_id + AND e.task_inserted_at = mrc.task_inserted_at + AND e.retry_count = mrc.max_retry_count WHERE e.event_type = 'STARTED' GROUP BY e.task_id +), error_message AS ( + SELECT + DISTINCT ON (e.task_id) e.task_id::bigint, + e.error_message + FROM + relevant_events e + JOIN + max_retry_counts mrc ON + e.tenant_id = mrc.tenant_id + AND e.task_id = mrc.task_id + AND e.task_inserted_at = mrc.task_inserted_at + AND e.retry_count = mrc.max_retry_count + WHERE + e.readable_status = 'FAILED' + ORDER BY + e.task_id, e.retry_count DESC ) SELECT t.tenant_id, t.id, + t.dag_external_id, t.inserted_at, t.external_id, t.queue, @@ -426,19 +571,20 @@ SELECT t.priority, t.sticky, t.display_name, - t.retry_count, t.additional_metadata, - t.status::v2_readable_status_olap as status, + t.readable_status::v2_readable_status_olap as status, f.finished_at::timestamptz as finished_at, - s.started_at::timestamptz as started_at + s.started_at::timestamptz as started_at, + e.error_message as error_message FROM tasks t LEFT JOIN finished_ats f ON f.task_id = t.id LEFT JOIN started_ats s ON s.task_id = t.id -ORDER BY t.inserted_at DESC, t.id DESC -LIMIT @taskLimit::int; +LEFT JOIN + error_message e ON e.task_id = t.id +ORDER BY t.inserted_at DESC, t.id DESC; -- name: GetTaskPointMetrics :many SELECT @@ -520,8 +666,19 @@ WITH locked_events AS ( updatable_events e WHERE (t.tenant_id, t.id, t.inserted_at) = (e.tenant_id, e.task_id, e.task_inserted_at) - AND e.retry_count >= t.latest_retry_count - AND e.max_readable_status > t.readable_status + AND + ( + -- if the retry count is greater than the latest retry count, update the status + ( + e.retry_count > t.latest_retry_count + AND e.max_readable_status != t.readable_status + ) OR + -- if the retry count is equal to the latest retry count, update the status if the status is greater + ( + e.retry_count = t.latest_retry_count + AND e.max_readable_status > t.readable_status + ) + ) RETURNING t.tenant_id, t.id, t.inserted_at ), events_to_requeue AS ( @@ -696,163 +853,129 @@ FROM locked_events; --- name: ListWorkflowRuns :many -WITH tasks AS ( - SELECT - r.id AS run_id, - r.tenant_id, - r.inserted_at, - r.external_id, - d.id AS dag_id, - t.id AS task_id, - r.readable_status, - r.kind, - r.workflow_id, - COALESCE(t.display_name, d.display_name) AS display_name, - COALESCE(d.input, t.input) AS input, - COALESCE(d.additional_metadata, t.additional_metadata) AS additional_metadata, - t.latest_retry_count - FROM v2_runs_olap r - LEFT JOIN v2_dags_olap d ON (r.tenant_id, r.external_id, r.inserted_at) = (d.tenant_id, d.external_id, d.inserted_at) - LEFT JOIN v2_tasks_olap t ON (r.tenant_id, r.external_id, r.inserted_at) = (t.tenant_id, t.external_id, t.inserted_at) - WHERE - ( - (r.kind = 'TASK' AND d.id IS NULL) - OR r.kind = 'DAG' - ) - AND ( - sqlc.narg('workflowIds')::uuid[] IS NULL - OR r.workflow_id = ANY(sqlc.narg('workflowIds')::uuid[]) - ) - AND ( - sqlc.narg('statuses')::text[] IS NULL - OR r.readable_status = ANY(cast(sqlc.narg('statuses')::text[] as v2_readable_status_olap[])) - ) - AND r.inserted_at >= @since::timestamptz - AND ( - sqlc.narg('until')::timestamptz IS NULL - OR r.inserted_at <= sqlc.narg('until')::timestamptz - ) - AND ( - sqlc.narg('keys')::text[] IS NULL - OR sqlc.narg('values')::text[] IS NULL - OR COALESCE(d.additional_metadata, t.additional_metadata) IS NULL - OR EXISTS ( - SELECT 1 FROM jsonb_each_text(COALESCE(d.additional_metadata, t.additional_metadata)) kv - JOIN LATERAL ( - SELECT unnest(sqlc.narg('keys')::text[]) AS k, - unnest(sqlc.narg('values')::text[]) AS v - ) AS u ON kv.key = u.k AND kv.value = u.v - ) - ) - AND ( - sqlc.narg('workerId')::uuid IS NULL - OR t.latest_worker_id = sqlc.narg('workerId')::uuid +-- name: FetchWorkflowRunIds :many +SELECT id, inserted_at, kind, external_id +FROM v2_runs_olap +WHERE + tenant_id = @tenantId::uuid + AND ( + sqlc.narg('workflowIds')::uuid[] IS NULL + OR workflow_id = ANY(sqlc.narg('workflowIds')::uuid[]) + ) + AND ( + sqlc.narg('statuses')::text[] IS NULL + OR readable_status = ANY(cast(sqlc.narg('statuses')::text[] as v2_readable_status_olap[])) + ) + AND inserted_at >= @since::timestamptz + AND ( + sqlc.narg('until')::timestamptz IS NULL + OR inserted_at <= sqlc.narg('until')::timestamptz + ) + AND ( + sqlc.narg('keys')::text[] IS NULL + OR sqlc.narg('values')::text[] IS NULL + OR EXISTS ( + SELECT 1 FROM jsonb_each_text(additional_metadata) kv + JOIN LATERAL ( + SELECT unnest(sqlc.narg('keys')::text[]) AS k, + unnest(sqlc.narg('values')::text[]) AS v + ) AS u ON kv.key = u.k AND kv.value = u.v ) - LIMIT @listWorkflowRunsLimit::integer - OFFSET @listWorkflowRunsOffset::integer -), metadata AS ( - SELECT - t.run_id, - MIN(e.inserted_at)::timestamptz AS created_at, - MIN(e.inserted_at) FILTER (WHERE e.readable_status = 'RUNNING')::timestamptz AS started_at, - MAX(e.inserted_at) FILTER (WHERE e.readable_status IN ('COMPLETED', 'CANCELLED', 'FAILED'))::timestamptz AS finished_at - FROM tasks t - JOIN v2_task_events_olap e ON (e.tenant_id, e.task_id, e.retry_count) = (t.tenant_id, t.task_id, t.latest_retry_count) - GROUP BY t.run_id -) - -SELECT - t.*, - COALESCE(m.created_at, t.inserted_at) AS created_at, - m.started_at, - m.finished_at, - e.output, - e.error_message -FROM tasks t -LEFT JOIN metadata m ON t.run_id = m.run_id --- NOTE: This is a bug - it only populates errors for tasks, but not dags or their children --- NOTE: JOIN v2_tasks_olap t ON (d.tenant_id, d.id) = (t.tenant_id, t.dag_id) is not going to be performant because the task's dag_id is not indexed. Use v2_dag_to_task_olap which indexes the dag_id which we can use instead -LEFT JOIN v2_task_events_olap e ON (e.tenant_id, e.task_id, e.retry_count, e.readable_status) = (t.tenant_id, t.task_id, t.latest_retry_count, t.readable_status) -ORDER BY t.inserted_at DESC + ) +ORDER BY inserted_at DESC, id DESC +LIMIT @listWorkflowRunsLimit::integer +OFFSET @listWorkflowRunsOffset::integer ; --- name: CountRuns :one +-- name: CountWorkflowRuns :one SELECT COUNT(*) -FROM v2_runs_olap r -LEFT JOIN v2_dags_olap d ON (r.tenant_id, r.external_id, r.inserted_at) = (d.tenant_id, d.external_id, d.inserted_at) -LEFT JOIN v2_tasks_olap t ON (r.tenant_id, r.external_id, r.inserted_at) = (t.tenant_id, t.external_id, t.inserted_at) +FROM v2_runs_olap WHERE - ( - (r.kind = 'TASK' AND d.id IS NULL) - OR r.kind = 'DAG' - ) + tenant_id = @tenantId::uuid AND ( sqlc.narg('workflowIds')::uuid[] IS NULL - OR r.workflow_id = ANY(sqlc.narg('workflowIds')::uuid[]) + OR workflow_id = ANY(sqlc.narg('workflowIds')::uuid[]) ) AND ( sqlc.narg('statuses')::text[] IS NULL - OR r.readable_status = ANY(cast(sqlc.narg('statuses')::text[] as v2_readable_status_olap[])) + OR readable_status = ANY(cast(sqlc.narg('statuses')::text[] as v2_readable_status_olap[])) ) - AND r.inserted_at >= @since::timestamptz + AND inserted_at >= @since::timestamptz AND ( sqlc.narg('until')::timestamptz IS NULL - OR r.inserted_at <= sqlc.narg('until')::timestamptz + OR inserted_at <= sqlc.narg('until')::timestamptz ) AND ( sqlc.narg('keys')::text[] IS NULL OR sqlc.narg('values')::text[] IS NULL - OR COALESCE(d.additional_metadata, t.additional_metadata) IS NULL OR EXISTS ( - SELECT 1 FROM jsonb_each_text(COALESCE(d.additional_metadata, t.additional_metadata)) kv + SELECT 1 FROM jsonb_each_text(additional_metadata) kv JOIN LATERAL ( SELECT unnest(sqlc.narg('keys')::text[]) AS k, unnest(sqlc.narg('values')::text[]) AS v ) AS u ON kv.key = u.k AND kv.value = u.v ) ) - AND ( - sqlc.narg('workerId')::uuid IS NULL - OR t.latest_worker_id = sqlc.narg('workerId')::uuid - ) -; +LIMIT 20000; --- name: ListDAGChildren :many -WITH tasks AS ( +-- name: PopulateDAGMetadata :many +WITH input AS ( SELECT + UNNEST(@ids::bigint[]) AS id, + UNNEST(@insertedAts::timestamptz[]) AS inserted_at +), runs AS ( + SELECT + d.id AS dag_id, r.id AS run_id, r.tenant_id, r.inserted_at, - t.external_id, - d.id AS dag_id, - t.id AS task_id, - t.readable_status, + r.external_id, + r.readable_status, r.kind, r.workflow_id, - t.display_name, - t.input, - t.additional_metadata, - t.latest_retry_count + d.display_name, + d.input, + d.additional_metadata FROM v2_runs_olap r JOIN v2_dags_olap d ON (r.tenant_id, r.external_id, r.inserted_at) = (d.tenant_id, d.external_id, d.inserted_at) - JOIN v2_tasks_olap t ON (d.tenant_id, d.id) = (t.tenant_id, t.dag_id) WHERE - kind = 'DAG' - AND (sqlc.narg('dagIds')::bigint[] IS NULL OR d.id = ANY(sqlc.narg('dagIds')::bigint[])) -), timers AS ( + (r.inserted_at, r.id) IN (SELECT inserted_at, id FROM input) + AND r.tenant_id = @tenantId::uuid + AND r.kind = 'DAG' +), relevant_events AS ( + SELECT + r.run_id, + e.* + FROM runs r + JOIN v2_dag_to_task_olap dt ON r.dag_id = dt.dag_id -- Do I need to join by `inserted_at` here too? + JOIN v2_task_events_olap e ON e.task_id = dt.task_id -- Do I need to join by `inserted_at` here too? +), metadata AS ( SELECT - t.task_id, + e.run_id, MIN(e.inserted_at)::timestamptz AS created_at, MIN(e.inserted_at) FILTER (WHERE e.readable_status = 'RUNNING')::timestamptz AS started_at, MAX(e.inserted_at) FILTER (WHERE e.readable_status IN ('COMPLETED', 'CANCELLED', 'FAILED'))::timestamptz AS finished_at - FROM tasks t - JOIN v2_task_events_olap e ON (e.tenant_id, e.task_id, e.retry_count) = (t.tenant_id, t.task_id, t.latest_retry_count) - GROUP BY t.task_id + FROM + relevant_events e + GROUP BY e.run_id +), error_message AS ( + SELECT + DISTINCT ON (e.run_id) e.run_id::bigint, + e.error_message + FROM + relevant_events e + WHERE + e.readable_status = 'FAILED' + ORDER BY + e.run_id, e.retry_count DESC ) - -SELECT t.*, COALESCE(ti.created_at, t.inserted_at) AS created_at, ti.started_at, ti.finished_at -FROM tasks t -LEFT JOIN timers ti ON t.task_id = ti.task_id -ORDER BY t.inserted_at DESC -; +SELECT + r.*, + m.created_at, + m.started_at, + m.finished_at, + e.error_message +FROM runs r +LEFT JOIN metadata m ON r.run_id = m.run_id +LEFT JOIN error_message e ON r.run_id = e.run_id +ORDER BY r.inserted_at DESC, r.run_id DESC; \ No newline at end of file diff --git a/pkg/repository/v2/timescalev2/queries.sql.go b/pkg/repository/v2/timescalev2/queries.sql.go index 0cbde3ff03..c33d176496 100644 --- a/pkg/repository/v2/timescalev2/queries.sql.go +++ b/pkg/repository/v2/timescalev2/queries.sql.go @@ -11,66 +11,121 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) -const countRuns = `-- name: CountRuns :one -SELECT COUNT(*) -FROM v2_runs_olap r -LEFT JOIN v2_dags_olap d ON (r.tenant_id, r.external_id, r.inserted_at) = (d.tenant_id, d.external_id, d.inserted_at) -LEFT JOIN v2_tasks_olap t ON (r.tenant_id, r.external_id, r.inserted_at) = (t.tenant_id, t.external_id, t.inserted_at) +const countTasks = `-- name: CountTasks :one +SELECT + COUNT(*) +FROM + v2_tasks_olap WHERE - ( - (r.kind = 'TASK' AND d.id IS NULL) - OR r.kind = 'DAG' + tenant_id = $1::uuid + AND inserted_at >= $2::timestamptz + AND ( + $3::timestamptz IS NULL + OR inserted_at <= $3::timestamptz ) AND ( - $1::uuid[] IS NULL - OR r.workflow_id = ANY($1::uuid[]) + $4::text[] IS NULL OR readable_status = ANY(cast($4::text[] as v2_readable_status_olap[])) ) AND ( - $2::text[] IS NULL - OR r.readable_status = ANY(cast($2::text[] as v2_readable_status_olap[])) + $5::uuid[] IS NULL OR workflow_id = ANY($5::uuid[]) ) - AND r.inserted_at >= $3::timestamptz AND ( - $4::timestamptz IS NULL - OR r.inserted_at <= $4::timestamptz + $6::uuid IS NULL OR latest_worker_id = $6::uuid ) AND ( - $5::text[] IS NULL - OR $6::text[] IS NULL - OR COALESCE(d.additional_metadata, t.additional_metadata) IS NULL + $7::text[] IS NULL + OR $8::text[] IS NULL OR EXISTS ( - SELECT 1 FROM jsonb_each_text(COALESCE(d.additional_metadata, t.additional_metadata)) kv + SELECT 1 FROM jsonb_each_text(additional_metadata) kv JOIN LATERAL ( - SELECT unnest($5::text[]) AS k, - unnest($6::text[]) AS v + SELECT unnest($7::text[]) AS k, + unnest($8::text[]) AS v ) AS u ON kv.key = u.k AND kv.value = u.v ) ) +ORDER BY + inserted_at DESC +LIMIT 20000 +` + +type CountTasksParams struct { + Tenantid pgtype.UUID `json:"tenantid"` + Since pgtype.Timestamptz `json:"since"` + Until pgtype.Timestamptz `json:"until"` + Statuses []string `json:"statuses"` + WorkflowIds []pgtype.UUID `json:"workflowIds"` + WorkerId pgtype.UUID `json:"workerId"` + Keys []string `json:"keys"` + Values []string `json:"values"` +} + +func (q *Queries) CountTasks(ctx context.Context, db DBTX, arg CountTasksParams) (int64, error) { + row := db.QueryRow(ctx, countTasks, + arg.Tenantid, + arg.Since, + arg.Until, + arg.Statuses, + arg.WorkflowIds, + arg.WorkerId, + arg.Keys, + arg.Values, + ) + var count int64 + err := row.Scan(&count) + return count, err +} + +const countWorkflowRuns = `-- name: CountWorkflowRuns :one +SELECT COUNT(*) +FROM v2_runs_olap +WHERE + tenant_id = $1::uuid + AND ( + $2::uuid[] IS NULL + OR workflow_id = ANY($2::uuid[]) + ) + AND ( + $3::text[] IS NULL + OR readable_status = ANY(cast($3::text[] as v2_readable_status_olap[])) + ) + AND inserted_at >= $4::timestamptz + AND ( + $5::timestamptz IS NULL + OR inserted_at <= $5::timestamptz + ) AND ( - $7::uuid IS NULL - OR t.latest_worker_id = $7::uuid + $6::text[] IS NULL + OR $7::text[] IS NULL + OR EXISTS ( + SELECT 1 FROM jsonb_each_text(additional_metadata) kv + JOIN LATERAL ( + SELECT unnest($6::text[]) AS k, + unnest($7::text[]) AS v + ) AS u ON kv.key = u.k AND kv.value = u.v + ) ) +LIMIT 20000 ` -type CountRunsParams struct { +type CountWorkflowRunsParams struct { + Tenantid pgtype.UUID `json:"tenantid"` WorkflowIds []pgtype.UUID `json:"workflowIds"` Statuses []string `json:"statuses"` Since pgtype.Timestamptz `json:"since"` Until pgtype.Timestamptz `json:"until"` Keys []string `json:"keys"` Values []string `json:"values"` - WorkerId pgtype.UUID `json:"workerId"` } -func (q *Queries) CountRuns(ctx context.Context, db DBTX, arg CountRunsParams) (int64, error) { - row := db.QueryRow(ctx, countRuns, +func (q *Queries) CountWorkflowRuns(ctx context.Context, db DBTX, arg CountWorkflowRunsParams) (int64, error) { + row := db.QueryRow(ctx, countWorkflowRuns, + arg.Tenantid, arg.WorkflowIds, arg.Statuses, arg.Since, arg.Until, arg.Keys, arg.Values, - arg.WorkerId, ) var count int64 err := row.Scan(&count) @@ -196,6 +251,94 @@ type CreateTasksOLAPParams struct { DagInsertedAt pgtype.Timestamptz `json:"dag_inserted_at"` } +const fetchWorkflowRunIds = `-- name: FetchWorkflowRunIds :many +SELECT id, inserted_at, kind, external_id +FROM v2_runs_olap +WHERE + tenant_id = $1::uuid + AND ( + $2::uuid[] IS NULL + OR workflow_id = ANY($2::uuid[]) + ) + AND ( + $3::text[] IS NULL + OR readable_status = ANY(cast($3::text[] as v2_readable_status_olap[])) + ) + AND inserted_at >= $4::timestamptz + AND ( + $5::timestamptz IS NULL + OR inserted_at <= $5::timestamptz + ) + AND ( + $6::text[] IS NULL + OR $7::text[] IS NULL + OR EXISTS ( + SELECT 1 FROM jsonb_each_text(additional_metadata) kv + JOIN LATERAL ( + SELECT unnest($6::text[]) AS k, + unnest($7::text[]) AS v + ) AS u ON kv.key = u.k AND kv.value = u.v + ) + ) +ORDER BY inserted_at DESC, id DESC +LIMIT $9::integer +OFFSET $8::integer +` + +type FetchWorkflowRunIdsParams struct { + Tenantid pgtype.UUID `json:"tenantid"` + WorkflowIds []pgtype.UUID `json:"workflowIds"` + Statuses []string `json:"statuses"` + Since pgtype.Timestamptz `json:"since"` + Until pgtype.Timestamptz `json:"until"` + Keys []string `json:"keys"` + Values []string `json:"values"` + Listworkflowrunsoffset int32 `json:"listworkflowrunsoffset"` + Listworkflowrunslimit int32 `json:"listworkflowrunslimit"` +} + +type FetchWorkflowRunIdsRow struct { + ID int64 `json:"id"` + InsertedAt pgtype.Timestamptz `json:"inserted_at"` + Kind V2RunKind `json:"kind"` + ExternalID pgtype.UUID `json:"external_id"` +} + +func (q *Queries) FetchWorkflowRunIds(ctx context.Context, db DBTX, arg FetchWorkflowRunIdsParams) ([]*FetchWorkflowRunIdsRow, error) { + rows, err := db.Query(ctx, fetchWorkflowRunIds, + arg.Tenantid, + arg.WorkflowIds, + arg.Statuses, + arg.Since, + arg.Until, + arg.Keys, + arg.Values, + arg.Listworkflowrunsoffset, + arg.Listworkflowrunslimit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*FetchWorkflowRunIdsRow + for rows.Next() { + var i FetchWorkflowRunIdsRow + if err := rows.Scan( + &i.ID, + &i.InsertedAt, + &i.Kind, + &i.ExternalID, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getTaskPointMetrics = `-- name: GetTaskPointMetrics :many SELECT time_bucket(COALESCE($1::interval, '1 minute'), bucket)::timestamptz as bucket_2, @@ -294,101 +437,6 @@ func (q *Queries) GetTenantStatusMetrics(ctx context.Context, db DBTX, arg GetTe return &i, err } -const listDAGChildren = `-- name: ListDAGChildren :many -WITH tasks AS ( - SELECT - r.id AS run_id, - r.tenant_id, - r.inserted_at, - t.external_id, - d.id AS dag_id, - t.id AS task_id, - t.readable_status, - r.kind, - r.workflow_id, - t.display_name, - t.input, - t.additional_metadata, - t.latest_retry_count - FROM v2_runs_olap r - JOIN v2_dags_olap d ON (r.tenant_id, r.external_id, r.inserted_at) = (d.tenant_id, d.external_id, d.inserted_at) - JOIN v2_tasks_olap t ON (d.tenant_id, d.id) = (t.tenant_id, t.dag_id) - WHERE - kind = 'DAG' - AND ($1::bigint[] IS NULL OR d.id = ANY($1::bigint[])) -), timers AS ( - SELECT - t.task_id, - MIN(e.inserted_at)::timestamptz AS created_at, - MIN(e.inserted_at) FILTER (WHERE e.readable_status = 'RUNNING')::timestamptz AS started_at, - MAX(e.inserted_at) FILTER (WHERE e.readable_status IN ('COMPLETED', 'CANCELLED', 'FAILED'))::timestamptz AS finished_at - FROM tasks t - JOIN v2_task_events_olap e ON (e.tenant_id, e.task_id, e.retry_count) = (t.tenant_id, t.task_id, t.latest_retry_count) - GROUP BY t.task_id -) - -SELECT t.run_id, t.tenant_id, t.inserted_at, t.external_id, t.dag_id, t.task_id, t.readable_status, t.kind, t.workflow_id, t.display_name, t.input, t.additional_metadata, t.latest_retry_count, COALESCE(ti.created_at, t.inserted_at) AS created_at, ti.started_at, ti.finished_at -FROM tasks t -LEFT JOIN timers ti ON t.task_id = ti.task_id -ORDER BY t.inserted_at DESC -` - -type ListDAGChildrenRow struct { - RunID int64 `json:"run_id"` - TenantID pgtype.UUID `json:"tenant_id"` - InsertedAt pgtype.Timestamptz `json:"inserted_at"` - ExternalID pgtype.UUID `json:"external_id"` - DagID int64 `json:"dag_id"` - TaskID int64 `json:"task_id"` - ReadableStatus V2ReadableStatusOlap `json:"readable_status"` - Kind V2RunKind `json:"kind"` - WorkflowID pgtype.UUID `json:"workflow_id"` - DisplayName string `json:"display_name"` - Input []byte `json:"input"` - AdditionalMetadata []byte `json:"additional_metadata"` - LatestRetryCount int32 `json:"latest_retry_count"` - CreatedAt pgtype.Timestamptz `json:"created_at"` - StartedAt pgtype.Timestamptz `json:"started_at"` - FinishedAt pgtype.Timestamptz `json:"finished_at"` -} - -func (q *Queries) ListDAGChildren(ctx context.Context, db DBTX, dagids []int64) ([]*ListDAGChildrenRow, error) { - rows, err := db.Query(ctx, listDAGChildren, dagids) - if err != nil { - return nil, err - } - defer rows.Close() - var items []*ListDAGChildrenRow - for rows.Next() { - var i ListDAGChildrenRow - if err := rows.Scan( - &i.RunID, - &i.TenantID, - &i.InsertedAt, - &i.ExternalID, - &i.DagID, - &i.TaskID, - &i.ReadableStatus, - &i.Kind, - &i.WorkflowID, - &i.DisplayName, - &i.Input, - &i.AdditionalMetadata, - &i.LatestRetryCount, - &i.CreatedAt, - &i.StartedAt, - &i.FinishedAt, - ); err != nil { - return nil, err - } - items = append(items, &i) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const listOLAPDAGPartitionsBeforeDate = `-- name: ListOLAPDAGPartitionsBeforeDate :many SELECT p::text AS partition_name @@ -588,73 +636,123 @@ func (q *Queries) ListTaskEvents(ctx context.Context, db DBTX, arg ListTaskEvent const listTasks = `-- name: ListTasks :many SELECT - tenant_id, id, inserted_at, external_id, queue, action_id, step_id, workflow_id, schedule_timeout, step_timeout, priority, sticky, desired_worker_id, display_name, input, additional_metadata, readable_status, latest_retry_count, latest_worker_id, dag_id, dag_inserted_at + id, + inserted_at FROM v2_tasks_olap WHERE tenant_id = $1::uuid AND inserted_at >= $2::timestamptz AND ( - $3::text[] IS NULL OR readable_status = ANY(cast($3::text[] as v2_readable_status_olap[])) + $3::timestamptz IS NULL + OR inserted_at <= $3::timestamptz + ) + AND ( + $4::text[] IS NULL OR readable_status = ANY(cast($4::text[] as v2_readable_status_olap[])) ) AND ( - $4::uuid[] IS NULL OR workflow_id = ANY($4::uuid[]) + $5::uuid[] IS NULL OR workflow_id = ANY($5::uuid[]) ) AND ( - $5::uuid IS NULL OR latest_worker_id = $5::uuid + $6::uuid IS NULL OR latest_worker_id = $6::uuid + ) + AND ( + $7::text[] IS NULL + OR $8::text[] IS NULL + OR EXISTS ( + SELECT 1 FROM jsonb_each_text(additional_metadata) kv + JOIN LATERAL ( + SELECT unnest($7::text[]) AS k, + unnest($8::text[]) AS v + ) AS u ON kv.key = u.k AND kv.value = u.v + ) ) ORDER BY inserted_at DESC -LIMIT $6::integer +LIMIT $10::integer +OFFSET $9::integer ` type ListTasksParams struct { - Tenantid pgtype.UUID `json:"tenantid"` - Insertedafter pgtype.Timestamptz `json:"insertedafter"` - Statuses []string `json:"statuses"` - WorkflowIds []pgtype.UUID `json:"workflowIds"` - WorkerId pgtype.UUID `json:"workerId"` - Tasklimit int32 `json:"tasklimit"` + Tenantid pgtype.UUID `json:"tenantid"` + Since pgtype.Timestamptz `json:"since"` + Until pgtype.Timestamptz `json:"until"` + Statuses []string `json:"statuses"` + WorkflowIds []pgtype.UUID `json:"workflowIds"` + WorkerId pgtype.UUID `json:"workerId"` + Keys []string `json:"keys"` + Values []string `json:"values"` + Taskoffset int32 `json:"taskoffset"` + Tasklimit int32 `json:"tasklimit"` +} + +type ListTasksRow struct { + ID int64 `json:"id"` + InsertedAt pgtype.Timestamptz `json:"inserted_at"` } -func (q *Queries) ListTasks(ctx context.Context, db DBTX, arg ListTasksParams) ([]*V2TasksOlap, error) { +func (q *Queries) ListTasks(ctx context.Context, db DBTX, arg ListTasksParams) ([]*ListTasksRow, error) { rows, err := db.Query(ctx, listTasks, arg.Tenantid, - arg.Insertedafter, + arg.Since, + arg.Until, arg.Statuses, arg.WorkflowIds, arg.WorkerId, + arg.Keys, + arg.Values, + arg.Taskoffset, arg.Tasklimit, ) if err != nil { return nil, err } defer rows.Close() - var items []*V2TasksOlap + var items []*ListTasksRow + for rows.Next() { + var i ListTasksRow + if err := rows.Scan(&i.ID, &i.InsertedAt); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listTasksByDAGIds = `-- name: ListTasksByDAGIds :many +SELECT + dt.dag_id, dt.dag_inserted_at, dt.task_id, dt.task_inserted_at +FROM + v2_lookup_table lt +JOIN + v2_dag_to_task_olap dt ON lt.dag_id = dt.dag_id +WHERE + lt.external_id = ANY($1::uuid[]) + AND tenant_id = $2::uuid +` + +type ListTasksByDAGIdsParams struct { + Dagids []pgtype.UUID `json:"dagids"` + Tenantid pgtype.UUID `json:"tenantid"` +} + +func (q *Queries) ListTasksByDAGIds(ctx context.Context, db DBTX, arg ListTasksByDAGIdsParams) ([]*V2DagToTaskOlap, error) { + rows, err := db.Query(ctx, listTasksByDAGIds, arg.Dagids, arg.Tenantid) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*V2DagToTaskOlap for rows.Next() { - var i V2TasksOlap + var i V2DagToTaskOlap if err := rows.Scan( - &i.TenantID, - &i.ID, - &i.InsertedAt, - &i.ExternalID, - &i.Queue, - &i.ActionID, - &i.StepID, - &i.WorkflowID, - &i.ScheduleTimeout, - &i.StepTimeout, - &i.Priority, - &i.Sticky, - &i.DesiredWorkerID, - &i.DisplayName, - &i.Input, - &i.AdditionalMetadata, - &i.ReadableStatus, - &i.LatestRetryCount, - &i.LatestWorkerID, &i.DagID, &i.DagInsertedAt, + &i.TaskID, + &i.TaskInsertedAt, ); err != nil { return nil, err } @@ -666,157 +764,160 @@ func (q *Queries) ListTasks(ctx context.Context, db DBTX, arg ListTasksParams) ( return items, nil } -const listWorkflowRuns = `-- name: ListWorkflowRuns :many -WITH tasks AS ( +const listTasksByExternalIds = `-- name: ListTasksByExternalIds :many +SELECT + tenant_id, + task_id, + inserted_at +FROM + v2_lookup_table +WHERE + external_id = ANY($1::uuid[]) + AND tenant_id = $2::uuid +` + +type ListTasksByExternalIdsParams struct { + Externalids []pgtype.UUID `json:"externalids"` + Tenantid pgtype.UUID `json:"tenantid"` +} + +type ListTasksByExternalIdsRow struct { + TenantID pgtype.UUID `json:"tenant_id"` + TaskID pgtype.Int8 `json:"task_id"` + InsertedAt pgtype.Timestamptz `json:"inserted_at"` +} + +func (q *Queries) ListTasksByExternalIds(ctx context.Context, db DBTX, arg ListTasksByExternalIdsParams) ([]*ListTasksByExternalIdsRow, error) { + rows, err := db.Query(ctx, listTasksByExternalIds, arg.Externalids, arg.Tenantid) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*ListTasksByExternalIdsRow + for rows.Next() { + var i ListTasksByExternalIdsRow + if err := rows.Scan(&i.TenantID, &i.TaskID, &i.InsertedAt); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const populateDAGMetadata = `-- name: PopulateDAGMetadata :many +WITH input AS ( + SELECT + UNNEST($1::bigint[]) AS id, + UNNEST($2::timestamptz[]) AS inserted_at +), runs AS ( SELECT + d.id AS dag_id, r.id AS run_id, r.tenant_id, r.inserted_at, r.external_id, - d.id AS dag_id, - t.id AS task_id, r.readable_status, r.kind, r.workflow_id, - COALESCE(t.display_name, d.display_name) AS display_name, - COALESCE(d.input, t.input) AS input, - COALESCE(d.additional_metadata, t.additional_metadata) AS additional_metadata, - t.latest_retry_count + d.display_name, + d.input, + d.additional_metadata FROM v2_runs_olap r - LEFT JOIN v2_dags_olap d ON (r.tenant_id, r.external_id, r.inserted_at) = (d.tenant_id, d.external_id, d.inserted_at) - LEFT JOIN v2_tasks_olap t ON (r.tenant_id, r.external_id, r.inserted_at) = (t.tenant_id, t.external_id, t.inserted_at) + JOIN v2_dags_olap d ON (r.tenant_id, r.external_id, r.inserted_at) = (d.tenant_id, d.external_id, d.inserted_at) WHERE - ( - (r.kind = 'TASK' AND d.id IS NULL) - OR r.kind = 'DAG' - ) - AND ( - $1::uuid[] IS NULL - OR r.workflow_id = ANY($1::uuid[]) - ) - AND ( - $2::text[] IS NULL - OR r.readable_status = ANY(cast($2::text[] as v2_readable_status_olap[])) - ) - AND r.inserted_at >= $3::timestamptz - AND ( - $4::timestamptz IS NULL - OR r.inserted_at <= $4::timestamptz - ) - AND ( - $5::text[] IS NULL - OR $6::text[] IS NULL - OR COALESCE(d.additional_metadata, t.additional_metadata) IS NULL - OR EXISTS ( - SELECT 1 FROM jsonb_each_text(COALESCE(d.additional_metadata, t.additional_metadata)) kv - JOIN LATERAL ( - SELECT unnest($5::text[]) AS k, - unnest($6::text[]) AS v - ) AS u ON kv.key = u.k AND kv.value = u.v - ) - ) - AND ( - $7::uuid IS NULL - OR t.latest_worker_id = $7::uuid - ) - LIMIT $9::integer - OFFSET $8::integer + (r.inserted_at, r.id) IN (SELECT inserted_at, id FROM input) + AND r.tenant_id = $3::uuid + AND r.kind = 'DAG' +), relevant_events AS ( + SELECT + r.run_id, + e.tenant_id, e.id, e.inserted_at, e.task_id, e.task_inserted_at, e.event_type, e.workflow_id, e.event_timestamp, e.readable_status, e.retry_count, e.error_message, e.output, e.worker_id, e.additional__event_data, e.additional__event_message + FROM runs r + JOIN v2_dag_to_task_olap dt ON r.dag_id = dt.dag_id -- Do I need to join by ` + "`" + `inserted_at` + "`" + ` here too? + JOIN v2_task_events_olap e ON e.task_id = dt.task_id -- Do I need to join by ` + "`" + `inserted_at` + "`" + ` here too? ), metadata AS ( SELECT - t.run_id, + e.run_id, MIN(e.inserted_at)::timestamptz AS created_at, MIN(e.inserted_at) FILTER (WHERE e.readable_status = 'RUNNING')::timestamptz AS started_at, MAX(e.inserted_at) FILTER (WHERE e.readable_status IN ('COMPLETED', 'CANCELLED', 'FAILED'))::timestamptz AS finished_at - FROM tasks t - JOIN v2_task_events_olap e ON (e.tenant_id, e.task_id, e.retry_count) = (t.tenant_id, t.task_id, t.latest_retry_count) - GROUP BY t.run_id + FROM + relevant_events e + GROUP BY e.run_id +), error_message AS ( + SELECT + DISTINCT ON (e.run_id) e.run_id::bigint, + e.error_message + FROM + relevant_events e + WHERE + e.readable_status = 'FAILED' + ORDER BY + e.run_id, e.retry_count DESC ) - SELECT - t.run_id, t.tenant_id, t.inserted_at, t.external_id, t.dag_id, t.task_id, t.readable_status, t.kind, t.workflow_id, t.display_name, t.input, t.additional_metadata, t.latest_retry_count, - COALESCE(m.created_at, t.inserted_at) AS created_at, + r.dag_id, r.run_id, r.tenant_id, r.inserted_at, r.external_id, r.readable_status, r.kind, r.workflow_id, r.display_name, r.input, r.additional_metadata, + m.created_at, m.started_at, m.finished_at, - e.output, e.error_message -FROM tasks t -LEFT JOIN metadata m ON t.run_id = m.run_id -LEFT JOIN v2_task_events_olap e ON (e.tenant_id, e.task_id, e.retry_count, e.readable_status) = (t.tenant_id, t.task_id, t.latest_retry_count, t.readable_status) -ORDER BY t.inserted_at DESC +FROM runs r +LEFT JOIN metadata m ON r.run_id = m.run_id +LEFT JOIN error_message e ON r.run_id = e.run_id +ORDER BY r.inserted_at DESC, r.run_id DESC ` -type ListWorkflowRunsParams struct { - WorkflowIds []pgtype.UUID `json:"workflowIds"` - Statuses []string `json:"statuses"` - Since pgtype.Timestamptz `json:"since"` - Until pgtype.Timestamptz `json:"until"` - Keys []string `json:"keys"` - Values []string `json:"values"` - WorkerId pgtype.UUID `json:"workerId"` - Listworkflowrunsoffset int32 `json:"listworkflowrunsoffset"` - Listworkflowrunslimit int32 `json:"listworkflowrunslimit"` +type PopulateDAGMetadataParams struct { + Ids []int64 `json:"ids"` + Insertedats []pgtype.Timestamptz `json:"insertedats"` + Tenantid pgtype.UUID `json:"tenantid"` } -type ListWorkflowRunsRow struct { +type PopulateDAGMetadataRow struct { + DagID int64 `json:"dag_id"` RunID int64 `json:"run_id"` TenantID pgtype.UUID `json:"tenant_id"` InsertedAt pgtype.Timestamptz `json:"inserted_at"` ExternalID pgtype.UUID `json:"external_id"` - DagID pgtype.Int8 `json:"dag_id"` - TaskID pgtype.Int8 `json:"task_id"` ReadableStatus V2ReadableStatusOlap `json:"readable_status"` Kind V2RunKind `json:"kind"` WorkflowID pgtype.UUID `json:"workflow_id"` DisplayName string `json:"display_name"` Input []byte `json:"input"` AdditionalMetadata []byte `json:"additional_metadata"` - LatestRetryCount pgtype.Int4 `json:"latest_retry_count"` CreatedAt pgtype.Timestamptz `json:"created_at"` StartedAt pgtype.Timestamptz `json:"started_at"` FinishedAt pgtype.Timestamptz `json:"finished_at"` - Output []byte `json:"output"` ErrorMessage pgtype.Text `json:"error_message"` } -// NOTE: This is a bug - it only populates errors for tasks, but not dags or their children -// NOTE: JOIN v2_tasks_olap t ON (d.tenant_id, d.id) = (t.tenant_id, t.dag_id) is not going to be performant because the task's dag_id is not indexed. Use v2_dag_to_task_olap which indexes the dag_id which we can use instead -func (q *Queries) ListWorkflowRuns(ctx context.Context, db DBTX, arg ListWorkflowRunsParams) ([]*ListWorkflowRunsRow, error) { - rows, err := db.Query(ctx, listWorkflowRuns, - arg.WorkflowIds, - arg.Statuses, - arg.Since, - arg.Until, - arg.Keys, - arg.Values, - arg.WorkerId, - arg.Listworkflowrunsoffset, - arg.Listworkflowrunslimit, - ) +func (q *Queries) PopulateDAGMetadata(ctx context.Context, db DBTX, arg PopulateDAGMetadataParams) ([]*PopulateDAGMetadataRow, error) { + rows, err := db.Query(ctx, populateDAGMetadata, arg.Ids, arg.Insertedats, arg.Tenantid) if err != nil { return nil, err } defer rows.Close() - var items []*ListWorkflowRunsRow + var items []*PopulateDAGMetadataRow for rows.Next() { - var i ListWorkflowRunsRow + var i PopulateDAGMetadataRow if err := rows.Scan( + &i.DagID, &i.RunID, &i.TenantID, &i.InsertedAt, &i.ExternalID, - &i.DagID, - &i.TaskID, &i.ReadableStatus, &i.Kind, &i.WorkflowID, &i.DisplayName, &i.Input, &i.AdditionalMetadata, - &i.LatestRetryCount, &i.CreatedAt, &i.StartedAt, &i.FinishedAt, - &i.Output, &i.ErrorMessage, ); err != nil { return nil, err @@ -986,16 +1087,14 @@ func (q *Queries) PopulateSingleTaskRunData(ctx context.Context, db DBTX, arg Po const populateTaskRunData = `-- name: PopulateTaskRunData :many WITH input AS ( SELECT - UNNEST($2::uuid[]) AS tenant_id, - UNNEST($3::bigint[]) AS id, - UNNEST($4::timestamptz[]) AS inserted_at, - UNNEST($5::int[]) AS retry_count, - unnest(cast($6::text[] as v2_readable_status_olap[])) AS status + UNNEST($1::bigint[]) AS id, + UNNEST($2::timestamptz[]) AS inserted_at ), tasks AS ( SELECT DISTINCT ON(t.tenant_id, t.id, t.inserted_at) t.tenant_id, t.id, + d.external_id AS dag_external_id, t.inserted_at, t.queue, t.action_id, @@ -1010,20 +1109,51 @@ WITH input AS ( t.display_name, t.input, t.additional_metadata, - i.retry_count, - i.status + t.readable_status FROM v2_tasks_olap t JOIN - input i ON i.tenant_id = t.tenant_id AND i.id = t.id AND i.inserted_at = t.inserted_at + input i ON i.id = t.id AND i.inserted_at = t.inserted_at + LEFT JOIN + v2_dag_to_task_olap dtt ON dtt.task_id = t.id + LEFT JOIN + v2_dags_olap d ON d.id = dtt.dag_id AND d.tenant_id = t.tenant_id + + WHERE + t.tenant_id = $3::uuid +), relevant_events AS ( + SELECT + e.tenant_id, e.id, e.inserted_at, e.task_id, e.task_inserted_at, e.event_type, e.workflow_id, e.event_timestamp, e.readable_status, e.retry_count, e.error_message, e.output, e.worker_id, e.additional__event_data, e.additional__event_message + FROM + v2_task_events_olap e + JOIN + tasks t ON t.id = e.task_id AND t.tenant_id = e.tenant_id AND t.inserted_at = e.task_inserted_at + WHERE + e.tenant_id = $3::uuid + AND e.task_id = ANY($1::bigint[]) + AND e.task_inserted_at = ANY($2::timestamptz[]) +), max_retry_counts AS ( + SELECT + e.tenant_id, + e.task_id, + e.task_inserted_at, + MAX(e.retry_count) AS max_retry_count + FROM + relevant_events e + GROUP BY + e.tenant_id, e.task_id, e.task_inserted_at ), finished_ats AS ( SELECT e.task_id::bigint, MAX(e.event_timestamp) AS finished_at FROM - v2_task_events_olap e + relevant_events e JOIN - tasks t ON t.id = e.task_id AND t.tenant_id = e.tenant_id AND t.inserted_at = e.task_inserted_at AND t.retry_count = e.retry_count + max_retry_counts mrc ON + e.tenant_id = mrc.tenant_id + AND e.task_id = mrc.task_id + AND e.task_inserted_at = mrc.task_inserted_at + AND e.retry_count = mrc.max_retry_count WHERE e.readable_status = ANY(ARRAY['COMPLETED', 'FAILED', 'CANCELLED']::v2_readable_status_olap[]) GROUP BY e.task_id @@ -1032,16 +1162,37 @@ WITH input AS ( e.task_id::bigint, MAX(e.event_timestamp) AS started_at FROM - v2_task_events_olap e + relevant_events e JOIN - tasks t ON t.id = e.task_id AND t.tenant_id = e.tenant_id AND t.inserted_at = e.task_inserted_at AND t.retry_count = e.retry_count + max_retry_counts mrc ON + e.tenant_id = mrc.tenant_id + AND e.task_id = mrc.task_id + AND e.task_inserted_at = mrc.task_inserted_at + AND e.retry_count = mrc.max_retry_count WHERE e.event_type = 'STARTED' GROUP BY e.task_id +), error_message AS ( + SELECT + DISTINCT ON (e.task_id) e.task_id::bigint, + e.error_message + FROM + relevant_events e + JOIN + max_retry_counts mrc ON + e.tenant_id = mrc.tenant_id + AND e.task_id = mrc.task_id + AND e.task_inserted_at = mrc.task_inserted_at + AND e.retry_count = mrc.max_retry_count + WHERE + e.readable_status = 'FAILED' + ORDER BY + e.task_id, e.retry_count DESC ) SELECT t.tenant_id, t.id, + t.dag_external_id, t.inserted_at, t.external_id, t.queue, @@ -1053,33 +1204,32 @@ SELECT t.priority, t.sticky, t.display_name, - t.retry_count, t.additional_metadata, - t.status::v2_readable_status_olap as status, + t.readable_status::v2_readable_status_olap as status, f.finished_at::timestamptz as finished_at, - s.started_at::timestamptz as started_at + s.started_at::timestamptz as started_at, + e.error_message as error_message FROM tasks t LEFT JOIN finished_ats f ON f.task_id = t.id LEFT JOIN started_ats s ON s.task_id = t.id +LEFT JOIN + error_message e ON e.task_id = t.id ORDER BY t.inserted_at DESC, t.id DESC -LIMIT $1::int ` type PopulateTaskRunDataParams struct { - Tasklimit int32 `json:"tasklimit"` - Tenantids []pgtype.UUID `json:"tenantids"` Taskids []int64 `json:"taskids"` Taskinsertedats []pgtype.Timestamptz `json:"taskinsertedats"` - Retrycounts []int32 `json:"retrycounts"` - Statuses []string `json:"statuses"` + Tenantid pgtype.UUID `json:"tenantid"` } type PopulateTaskRunDataRow struct { TenantID pgtype.UUID `json:"tenant_id"` ID int64 `json:"id"` + DagExternalID pgtype.UUID `json:"dag_external_id"` InsertedAt pgtype.Timestamptz `json:"inserted_at"` ExternalID pgtype.UUID `json:"external_id"` Queue string `json:"queue"` @@ -1091,22 +1241,15 @@ type PopulateTaskRunDataRow struct { Priority pgtype.Int4 `json:"priority"` Sticky V2StickyStrategyOlap `json:"sticky"` DisplayName string `json:"display_name"` - RetryCount interface{} `json:"retry_count"` AdditionalMetadata []byte `json:"additional_metadata"` Status V2ReadableStatusOlap `json:"status"` FinishedAt pgtype.Timestamptz `json:"finished_at"` StartedAt pgtype.Timestamptz `json:"started_at"` + ErrorMessage pgtype.Text `json:"error_message"` } func (q *Queries) PopulateTaskRunData(ctx context.Context, db DBTX, arg PopulateTaskRunDataParams) ([]*PopulateTaskRunDataRow, error) { - rows, err := db.Query(ctx, populateTaskRunData, - arg.Tasklimit, - arg.Tenantids, - arg.Taskids, - arg.Taskinsertedats, - arg.Retrycounts, - arg.Statuses, - ) + rows, err := db.Query(ctx, populateTaskRunData, arg.Taskids, arg.Taskinsertedats, arg.Tenantid) if err != nil { return nil, err } @@ -1117,6 +1260,7 @@ func (q *Queries) PopulateTaskRunData(ctx context.Context, db DBTX, arg Populate if err := rows.Scan( &i.TenantID, &i.ID, + &i.DagExternalID, &i.InsertedAt, &i.ExternalID, &i.Queue, @@ -1128,11 +1272,11 @@ func (q *Queries) PopulateTaskRunData(ctx context.Context, db DBTX, arg Populate &i.Priority, &i.Sticky, &i.DisplayName, - &i.RetryCount, &i.AdditionalMetadata, &i.Status, &i.FinishedAt, &i.StartedAt, + &i.ErrorMessage, ); err != nil { return nil, err } @@ -1144,6 +1288,43 @@ func (q *Queries) PopulateTaskRunData(ctx context.Context, db DBTX, arg Populate return items, nil } +const readDAGByExternalID = `-- name: ReadDAGByExternalID :one +WITH lookup_task AS ( + SELECT + tenant_id, + dag_id, + inserted_at + FROM + v2_lookup_table + WHERE + external_id = $1::uuid +) +SELECT + d.id, d.inserted_at, d.tenant_id, d.external_id, d.display_name, d.workflow_id, d.workflow_version_id, d.readable_status, d.input, d.additional_metadata +FROM + v2_dags_olap d +JOIN + lookup_task lt ON lt.tenant_id = d.tenant_id AND lt.dag_id = d.id AND lt.inserted_at = d.inserted_at +` + +func (q *Queries) ReadDAGByExternalID(ctx context.Context, db DBTX, externalid pgtype.UUID) (*V2DagsOlap, error) { + row := db.QueryRow(ctx, readDAGByExternalID, externalid) + var i V2DagsOlap + err := row.Scan( + &i.ID, + &i.InsertedAt, + &i.TenantID, + &i.ExternalID, + &i.DisplayName, + &i.WorkflowID, + &i.WorkflowVersionID, + &i.ReadableStatus, + &i.Input, + &i.AdditionalMetadata, + ) + return &i, err +} + const readTaskByExternalID = `-- name: ReadTaskByExternalID :one WITH lookup_task AS ( SELECT @@ -1418,8 +1599,19 @@ WITH locked_events AS ( updatable_events e WHERE (t.tenant_id, t.id, t.inserted_at) = (e.tenant_id, e.task_id, e.task_inserted_at) - AND e.retry_count >= t.latest_retry_count - AND e.max_readable_status > t.readable_status + AND + ( + -- if the retry count is greater than the latest retry count, update the status + ( + e.retry_count > t.latest_retry_count + AND e.max_readable_status != t.readable_status + ) OR + -- if the retry count is equal to the latest retry count, update the status if the status is greater + ( + e.retry_count = t.latest_retry_count + AND e.max_readable_status > t.readable_status + ) + ) RETURNING t.tenant_id, t.id, t.inserted_at ), events_to_requeue AS ( diff --git a/sql/schema/timescale.sql b/sql/schema/timescale.sql index 4c1620014b..6e3e0833c8 100644 --- a/sql/schema/timescale.sql +++ b/sql/schema/timescale.sql @@ -199,6 +199,7 @@ CREATE TABLE v2_runs_olap ( readable_status v2_readable_status_olap NOT NULL DEFAULT 'QUEUED', kind v2_run_kind NOT NULL, workflow_id UUID NOT NULL, + additional_metadata JSONB, PRIMARY KEY (inserted_at, id, readable_status, kind) ) PARTITION BY RANGE(inserted_at); @@ -421,7 +422,8 @@ BEGIN external_id, readable_status, kind, - workflow_id + workflow_id, + additional_metadata ) SELECT tenant_id, @@ -430,7 +432,8 @@ BEGIN external_id, readable_status, 'TASK', - workflow_id + workflow_id, + additional_metadata FROM new_rows WHERE dag_id IS NULL; @@ -522,7 +525,8 @@ BEGIN external_id, readable_status, kind, - workflow_id + workflow_id, + additional_metadata ) SELECT tenant_id, @@ -531,7 +535,8 @@ BEGIN external_id, readable_status, 'DAG', - workflow_id + workflow_id, + additional_metadata FROM new_rows; INSERT INTO v2_lookup_table (