From c8cbe35195d2e8f867fea3822a7ecc9289a44d3c Mon Sep 17 00:00:00 2001 From: Denis Gukov Date: Sun, 15 Jun 2025 17:19:53 +0500 Subject: [PATCH 1/4] perf(templates): improve templates loading performance --- api/projects/templates.go | 7 +++++-- api/projects/views.go | 6 +++++- db/Store.go | 2 +- db/bolt/template.go | 2 +- db/sql/project.go | 7 ------- db/sql/task.go | 17 ++++++++++++++++- db/sql/template.go | 23 +++++++++++++++-------- services/project/backup.go | 2 +- services/tasks/TaskRunner.go | 7 ++++--- web/src/views/project/Templates.vue | 7 +------ 10 files changed, 49 insertions(+), 31 deletions(-) diff --git a/api/projects/templates.go b/api/projects/templates.go index 74d459574..8c5b74690 100644 --- a/api/projects/templates.go +++ b/api/projects/templates.go @@ -2,9 +2,10 @@ package projects import ( "fmt" - "github.com/semaphoreui/semaphore/util" "net/http" + "github.com/semaphoreui/semaphore/util" + "github.com/gorilla/context" "github.com/semaphoreui/semaphore/api/helpers" "github.com/semaphoreui/semaphore/db" @@ -56,7 +57,9 @@ func GetTemplates(w http.ResponseWriter, r *http.Request) { app := db.TemplateApp(r.URL.Query().Get("app")) filter.App = &app } - templates, err := helpers.Store(r).GetTemplates(project.ID, filter, helpers.QueryParams(r.URL)) + + q := helpers.QueryParams(r.URL) + templates, err := helpers.Store(r).GetTemplates(project.ID, filter, q, false) if err != nil { helpers.WriteError(w, err) diff --git a/api/projects/views.go b/api/projects/views.go index b73b72a6d..491e58d77 100644 --- a/api/projects/views.go +++ b/api/projects/views.go @@ -35,7 +35,11 @@ func GetViewTemplates(w http.ResponseWriter, r *http.Request) { project := context.Get(r, "project").(db.Project) view := context.Get(r, "view").(db.View) - templates, err := helpers.Store(r).GetTemplates(project.ID, db.TemplateFilter{ViewID: &view.ID}, helpers.QueryParams(r.URL)) + templates, err := helpers.Store(r).GetTemplates( + project.ID, + db.TemplateFilter{ViewID: &view.ID}, + helpers.QueryParams(r.URL), + false) if err != nil { helpers.WriteError(w, err) diff --git a/db/Store.go b/db/Store.go index bf315d851..290cfaa94 100644 --- a/db/Store.go +++ b/db/Store.go @@ -252,7 +252,7 @@ type ProjectStore interface { // TemplateManager handles template-related operations type TemplateManager interface { - GetTemplates(projectID int, filter TemplateFilter, params RetrieveQueryParams) ([]Template, error) + GetTemplates(projectID int, filter TemplateFilter, params RetrieveQueryParams, loadVaults bool) ([]Template, error) GetTemplateRefs(projectID int, templateID int) (ObjectReferrers, error) CreateTemplate(template Template) (Template, error) UpdateTemplate(template Template) error diff --git a/db/bolt/template.go b/db/bolt/template.go index 6809beee4..c4b4fc40d 100644 --- a/db/bolt/template.go +++ b/db/bolt/template.go @@ -69,7 +69,7 @@ func (d *BoltDb) SetTemplateDescription(projectID int, templateID int, descripti return err } -func (d *BoltDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.RetrieveQueryParams) (templates []db.Template, err error) { +func (d *BoltDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.RetrieveQueryParams, loadVaults bool) (templates []db.Template, err error) { var ftr = func(tpl any) bool { template := tpl.(db.Template) var res = true diff --git a/db/sql/project.go b/db/sql/project.go index b1d351eb6..96414f2d4 100644 --- a/db/sql/project.go +++ b/db/sql/project.go @@ -74,13 +74,6 @@ func (d *SqlDb) GetProject(projectID int) (project db.Project, err error) { func (d *SqlDb) DeleteProject(projectID int) error { - //tpls, err := d.GetTemplates(projectID, db.TemplateFilter{}, db.RetrieveQueryParams{}) - // - //if err != nil { - // return err - //} - // TODO: sort projects - tx, err := d.sql.Begin() if err != nil { diff --git a/db/sql/task.go b/db/sql/task.go index 9162b506f..bd2ae19dc 100644 --- a/db/sql/task.go +++ b/db/sql/task.go @@ -2,6 +2,7 @@ package sql import ( "encoding/json" + "fmt" "github.com/Masterminds/squirrel" "github.com/semaphoreui/semaphore/db" "math/rand" @@ -231,6 +232,13 @@ func (d *SqlDb) CreateTaskOutput(output db.TaskOutput) (db.TaskOutput, error) { } func (d *SqlDb) getTasks(projectID int, templateID *int, taskIDs []int, params db.RetrieveQueryParams, tasks *[]db.TaskWithTpl) (err error) { + start := time.Now() + + if taskIDs != nil && len(taskIDs) == 0 { + tasks = &[]db.TaskWithTpl{} + return nil + } + fields := "task.*" fields += ", tpl.playbook as tpl_playbook" + ", `user`.name as user_name" + @@ -250,7 +258,7 @@ func (d *SqlDb) getTasks(projectID int, templateID *int, taskIDs []int, params d q = q.Where("tpl.project_id=? AND task.template_id=?", projectID, templateID) } - if len(taskIDs) > 0 { + if taskIDs != nil { q = q.Where(squirrel.Eq{"task.id": taskIDs}) } @@ -262,6 +270,10 @@ func (d *SqlDb) getTasks(projectID int, templateID *int, taskIDs []int, params d _, err = d.selectAll(tasks, query, args...) + duration := time.Since(start) + fmt.Println("TASK: SQL query: ", duration.Milliseconds()) + start = time.Now() + for i := range *tasks { err = (*tasks)[i].Fill(d) if err != nil { @@ -269,6 +281,9 @@ func (d *SqlDb) getTasks(projectID int, templateID *int, taskIDs []int, params d } } + duration = time.Since(start) + fmt.Println("TASK: Filling: ", duration.Milliseconds()) + return } diff --git a/db/sql/template.go b/db/sql/template.go index 004678290..53d395648 100644 --- a/db/sql/template.go +++ b/db/sql/template.go @@ -150,7 +150,7 @@ func (d *SqlDb) SetTemplateDescription(projectID int, templateID int, descriptio return } -func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.RetrieveQueryParams) (templates []db.Template, err error) { +func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.RetrieveQueryParams, loadVaults bool) (templates []db.Template, err error) { pp, err := params.Validate(db.TemplateProps) if err != nil { @@ -230,6 +230,14 @@ func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db. OrderBy("pt.name " + order) } + if params.Count > 0 { + q = q.Limit(uint64(params.Count)) + } + + if params.Offset > 0 { + q = q.Offset(uint64(params.Offset)) + } + query, args, err := q.ToSql() if err != nil { @@ -253,6 +261,7 @@ func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db. } var tasks []db.TaskWithTpl + err = d.getTasks(projectID, nil, taskIDs, db.RetrieveQueryParams{}, &tasks) if err != nil { @@ -265,10 +274,6 @@ func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db. if tpl.LastTaskID != nil { for _, tsk := range tasks { if tsk.ID == *tpl.LastTaskID { - err = tsk.Fill(d) - if err != nil { - return - } template.LastTask = &tsk break } @@ -283,9 +288,11 @@ func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db. return } - template.Vaults, err = d.GetTemplateVaults(projectID, template.ID) - if err != nil { - return + if loadVaults { + template.Vaults, err = d.GetTemplateVaults(projectID, template.ID) + if err != nil { + return + } } templates = append(templates, template) diff --git a/services/project/backup.go b/services/project/backup.go index 87fdfd016..7eba0891a 100644 --- a/services/project/backup.go +++ b/services/project/backup.go @@ -125,7 +125,7 @@ func (b *BackupDB) makeUniqueNames() { func (b *BackupDB) load(projectID int, store db.Store) (err error) { - b.templates, err = store.GetTemplates(projectID, db.TemplateFilter{}, db.RetrieveQueryParams{}) + b.templates, err = store.GetTemplates(projectID, db.TemplateFilter{}, db.RetrieveQueryParams{}, true) if err != nil { return } diff --git a/services/tasks/TaskRunner.go b/services/tasks/TaskRunner.go index eec209b61..c86f0e8ed 100644 --- a/services/tasks/TaskRunner.go +++ b/services/tasks/TaskRunner.go @@ -3,13 +3,14 @@ package tasks import ( "encoding/json" "errors" - "github.com/semaphoreui/semaphore/pkg/tz" - "github.com/semaphoreui/semaphore/services/tasks/hooks" "os" "strconv" "strings" "sync" + "github.com/semaphoreui/semaphore/pkg/tz" + "github.com/semaphoreui/semaphore/services/tasks/hooks" + "github.com/semaphoreui/semaphore/api/sockets" "github.com/semaphoreui/semaphore/db" "github.com/semaphoreui/semaphore/pkg/task_logger" @@ -213,7 +214,7 @@ func (t *TaskRunner) run() { tpls, err := t.pool.store.GetTemplates(t.Task.ProjectID, db.TemplateFilter{ BuildTemplateID: &t.Task.TemplateID, AutorunOnly: true, - }, db.RetrieveQueryParams{}) + }, db.RetrieveQueryParams{}, true) if err != nil { t.Log("Running app failed: " + err.Error()) diff --git a/web/src/views/project/Templates.vue b/web/src/views/project/Templates.vue index 77eb00b38..f7bd0ef45 100644 --- a/web/src/views/project/Templates.vue +++ b/web/src/views/project/Templates.vue @@ -129,13 +129,12 @@ --> - - - Date: Fri, 11 Jul 2025 16:08:28 +0500 Subject: [PATCH 3/4] chore: remove unused import --- db/sql/task.go | 1 - 1 file changed, 1 deletion(-) diff --git a/db/sql/task.go b/db/sql/task.go index cfc87e9a4..ebf874ad1 100644 --- a/db/sql/task.go +++ b/db/sql/task.go @@ -2,7 +2,6 @@ package sql import ( "encoding/json" - "fmt" "github.com/Masterminds/squirrel" "github.com/semaphoreui/semaphore/db" "math/rand" From 510e68a0fc2d0bec45e5fd611638dbdee07f915c Mon Sep 17 00:00:00 2001 From: Denis Gukov Date: Fri, 11 Jul 2025 16:54:11 +0500 Subject: [PATCH 4/4] Update db/sql/task.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- db/sql/task.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/sql/task.go b/db/sql/task.go index ebf874ad1..c7ee538ee 100644 --- a/db/sql/task.go +++ b/db/sql/task.go @@ -232,7 +232,7 @@ func (d *SqlDb) CreateTaskOutput(output db.TaskOutput) (db.TaskOutput, error) { func (d *SqlDb) getTasks(projectID int, templateID *int, taskIDs []int, params db.RetrieveQueryParams, tasks *[]db.TaskWithTpl) (err error) { if taskIDs != nil && len(taskIDs) == 0 { - tasks = &[]db.TaskWithTpl{} + *tasks = []db.TaskWithTpl{} return nil }