Skip to content

Commit 98fb55e

Browse files
fiftinfiftin
committed
perf(templates): improve templates loading performance
1 parent 1b8c2f4 commit 98fb55e

File tree

10 files changed

+49
-31
lines changed

10 files changed

+49
-31
lines changed

‎api/projects/templates.go

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,10 @@ package projects
22

33
import (
44
"fmt"
5-
"github.com/semaphoreui/semaphore/util"
65
"net/http"
76

7+
"github.com/semaphoreui/semaphore/util"
8+
89
"github.com/gorilla/context"
910
"github.com/semaphoreui/semaphore/api/helpers"
1011
"github.com/semaphoreui/semaphore/db"
@@ -56,7 +57,9 @@ func GetTemplates(w http.ResponseWriter, r *http.Request) {
5657
app := db.TemplateApp(r.URL.Query().Get("app"))
5758
filter.App = &app
5859
}
59-
templates, err := helpers.Store(r).GetTemplates(project.ID, filter, helpers.QueryParams(r.URL))
60+
61+
q := helpers.QueryParams(r.URL)
62+
templates, err := helpers.Store(r).GetTemplates(project.ID, filter, q, false)
6063

6164
if err != nil {
6265
helpers.WriteError(w, err)

‎api/projects/views.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,11 @@ func GetViewTemplates(w http.ResponseWriter, r *http.Request) {
3535
project := context.Get(r, "project").(db.Project)
3636
view := context.Get(r, "view").(db.View)
3737

38-
templates, err := helpers.Store(r).GetTemplates(project.ID, db.TemplateFilter{ViewID: &view.ID}, helpers.QueryParams(r.URL))
38+
templates, err := helpers.Store(r).GetTemplates(
39+
project.ID,
40+
db.TemplateFilter{ViewID: &view.ID},
41+
helpers.QueryParams(r.URL),
42+
false)
3943

4044
if err != nil {
4145
helpers.WriteError(w, err)

‎db/Store.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,7 @@ type ProjectStore interface {
252252

253253
// TemplateManager handles template-related operations
254254
type TemplateManager interface {
255-
GetTemplates(projectID int, filter TemplateFilter, params RetrieveQueryParams) ([]Template, error)
255+
GetTemplates(projectID int, filter TemplateFilter, params RetrieveQueryParams, loadVaults bool) ([]Template, error)
256256
GetTemplateRefs(projectID int, templateID int) (ObjectReferrers, error)
257257
CreateTemplate(template Template) (Template, error)
258258
UpdateTemplate(template Template) error

‎db/bolt/template.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ func (d *BoltDb) SetTemplateDescription(projectID int, templateID int, descripti
6969
return err
7070
}
7171

72-
func (d *BoltDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.RetrieveQueryParams) (templates []db.Template, err error) {
72+
func (d *BoltDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.RetrieveQueryParams, loadVaults bool) (templates []db.Template, err error) {
7373
var ftr = func(tpl any) bool {
7474
template := tpl.(db.Template)
7575
var res = true

‎db/sql/project.go

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -74,13 +74,6 @@ func (d *SqlDb) GetProject(projectID int) (project db.Project, err error) {
7474

7575
func (d *SqlDb) DeleteProject(projectID int) error {
7676

77-
//tpls, err := d.GetTemplates(projectID, db.TemplateFilter{}, db.RetrieveQueryParams{})
78-
//
79-
//if err != nil {
80-
// return err
81-
//}
82-
// TODO: sort projects
83-
8477
tx, err := d.sql.Begin()
8578

8679
if err != nil {

‎db/sql/task.go

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package sql
22

33
import (
44
"encoding/json"
5+
"fmt"
56
"github.com/Masterminds/squirrel"
67
"github.com/semaphoreui/semaphore/db"
78
"math/rand"
@@ -231,6 +232,13 @@ func (d *SqlDb) CreateTaskOutput(output db.TaskOutput) (db.TaskOutput, error) {
231232
}
232233

233234
func (d *SqlDb) getTasks(projectID int, templateID *int, taskIDs []int, params db.RetrieveQueryParams, tasks *[]db.TaskWithTpl) (err error) {
235+
start := time.Now()
236+
237+
if taskIDs != nil && len(taskIDs) == 0 {
238+
tasks = &[]db.TaskWithTpl{}
239+
return nil
240+
}
241+
234242
fields := "task.*"
235243
fields += ", tpl.playbook as tpl_playbook" +
236244
", `user`.name as user_name" +
@@ -250,7 +258,7 @@ func (d *SqlDb) getTasks(projectID int, templateID *int, taskIDs []int, params d
250258
q = q.Where("tpl.project_id=? AND task.template_id=?", projectID, templateID)
251259
}
252260

253-
if len(taskIDs) > 0 {
261+
if taskIDs != nil {
254262
q = q.Where(squirrel.Eq{"task.id": taskIDs})
255263
}
256264

@@ -262,13 +270,20 @@ func (d *SqlDb) getTasks(projectID int, templateID *int, taskIDs []int, params d
262270

263271
_, err = d.selectAll(tasks, query, args...)
264272

273+
duration := time.Since(start)
274+
fmt.Println("TASK: SQL query: ", duration.Milliseconds())
275+
start = time.Now()
276+
265277
for i := range *tasks {
266278
err = (*tasks)[i].Fill(d)
267279
if err != nil {
268280
return
269281
}
270282
}
271283

284+
duration = time.Since(start)
285+
fmt.Println("TASK: Filling: ", duration.Milliseconds())
286+
272287
return
273288
}
274289

‎db/sql/template.go

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ func (d *SqlDb) SetTemplateDescription(projectID int, templateID int, descriptio
150150
return
151151
}
152152

153-
func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.RetrieveQueryParams) (templates []db.Template, err error) {
153+
func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.RetrieveQueryParams, loadVaults bool) (templates []db.Template, err error) {
154154

155155
pp, err := params.Validate(db.TemplateProps)
156156
if err != nil {
@@ -230,6 +230,14 @@ func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.
230230
OrderBy("pt.name " + order)
231231
}
232232

233+
if params.Count > 0 {
234+
q = q.Limit(uint64(params.Count))
235+
}
236+
237+
if params.Offset > 0 {
238+
q = q.Offset(uint64(params.Offset))
239+
}
240+
233241
query, args, err := q.ToSql()
234242

235243
if err != nil {
@@ -253,6 +261,7 @@ func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.
253261
}
254262

255263
var tasks []db.TaskWithTpl
264+
256265
err = d.getTasks(projectID, nil, taskIDs, db.RetrieveQueryParams{}, &tasks)
257266

258267
if err != nil {
@@ -265,10 +274,6 @@ func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.
265274
if tpl.LastTaskID != nil {
266275
for _, tsk := range tasks {
267276
if tsk.ID == *tpl.LastTaskID {
268-
err = tsk.Fill(d)
269-
if err != nil {
270-
return
271-
}
272277
template.LastTask = &tsk
273278
break
274279
}
@@ -283,9 +288,11 @@ func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.
283288
return
284289
}
285290

286-
template.Vaults, err = d.GetTemplateVaults(projectID, template.ID)
287-
if err != nil {
288-
return
291+
if loadVaults {
292+
template.Vaults, err = d.GetTemplateVaults(projectID, template.ID)
293+
if err != nil {
294+
return
295+
}
289296
}
290297

291298
templates = append(templates, template)

‎services/project/backup.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ func (b *BackupDB) makeUniqueNames() {
125125

126126
func (b *BackupDB) load(projectID int, store db.Store) (err error) {
127127

128-
b.templates, err = store.GetTemplates(projectID, db.TemplateFilter{}, db.RetrieveQueryParams{})
128+
b.templates, err = store.GetTemplates(projectID, db.TemplateFilter{}, db.RetrieveQueryParams{}, true)
129129
if err != nil {
130130
return
131131
}

‎services/tasks/TaskRunner.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,14 @@ package tasks
33
import (
44
"encoding/json"
55
"errors"
6-
"github.com/semaphoreui/semaphore/pkg/tz"
7-
"github.com/semaphoreui/semaphore/services/tasks/hooks"
86
"os"
97
"strconv"
108
"strings"
119
"sync"
1210

11+
"github.com/semaphoreui/semaphore/pkg/tz"
12+
"github.com/semaphoreui/semaphore/services/tasks/hooks"
13+
1314
"github.com/semaphoreui/semaphore/api/sockets"
1415
"github.com/semaphoreui/semaphore/db"
1516
"github.com/semaphoreui/semaphore/pkg/task_logger"
@@ -213,7 +214,7 @@ func (t *TaskRunner) run() {
213214
tpls, err := t.pool.store.GetTemplates(t.Task.ProjectID, db.TemplateFilter{
214215
BuildTemplateID: &t.Task.TemplateID,
215216
AutorunOnly: true,
216-
}, db.RetrieveQueryParams{})
217+
}, db.RetrieveQueryParams{}, true)
217218

218219
if err != nil {
219220
t.Log("Running app failed: " + err.Error())

‎web/src/views/project/Templates.vue

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -129,13 +129,12 @@
129129
<v-divider style="margin-top: -1px;"/>
130130

131131
<v-data-table
132-
hide-default-footer
133132
class="mt-4 templates-table"
134133
single-expand
135134
show-expand
136135
:headers="filteredHeaders"
137136
:items="items"
138-
:items-per-page="Number.MAX_VALUE"
137+
:items-per-page="50"
139138
:expanded.sync="openedItems"
140139
:style="{
141140
opacity: viewItemsLoading ? 0.3 : 1,
@@ -149,10 +148,6 @@
149148
{{ getAppIcon(item.app) }}
150149
</v-icon>
151150

152-
<!-- <v-icon class="mr-3" small>-->
153-
<!-- {{ TEMPLATE_TYPE_ICONS[item.type] }}-->
154-
<!-- </v-icon>-->
155-
156151
<router-link
157152
:to="viewId
158153
? `/project/${projectId}/views/${viewId}/templates/${item.id}`

0 commit comments

Comments
 (0)