Skip to content

perf(templates): improve templates loading performance #3099

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: develop
Choose a base branch
from
+36 −29
Open

perf(templates): improve templates loading performance #3099

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c8cbe35
perf(templates): improve templates loading performance
fiftin Jun 15, 2025
82f1718
chore(templates): remove debugging logs
fiftin Jun 16, 2025
01724e4
fix: merge conflict
fiftin Jul 11, 2025
de4aa27
chore: remove unused import
fiftin Jul 11, 2025
510e68a
Update db/sql/task.go
fiftin Jul 11, 2025
fd2c831
Merge branch 'develop' into get_template_perf
fiftin Aug 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions api/projects/templates.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@ package projects

import (
"fmt"
"github.com/semaphoreui/semaphore/util"
"net/http"

"github.com/semaphoreui/semaphore/api/helpers"
"github.com/semaphoreui/semaphore/db"
"github.com/semaphoreui/semaphore/util"
)

// TemplatesMiddleware ensures a template exists and loads it to the context
Expand Down Expand Up @@ -55,7 +55,9 @@ func GetTemplates(w http.ResponseWriter, r *http.Request) {
app := db.TemplateApp(r.URL.Query().Get("app"))
filter.App = &app
}
templates, err := helpers.Store(r).GetTemplates(project.ID, filter, helpers.QueryParams(r.URL))

q := helpers.QueryParams(r.URL)
templates, err := helpers.Store(r).GetTemplates(project.ID, filter, q, false)

if err != nil {
helpers.WriteError(w, err)
Expand Down
6 changes: 5 additions & 1 deletion api/projects/views.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,11 @@ func GetViewTemplates(w http.ResponseWriter, r *http.Request) {
project := helpers.GetFromContext(r, "project").(db.Project)
view := helpers.GetFromContext(r, "view").(db.View)

templates, err := helpers.Store(r).GetTemplates(project.ID, db.TemplateFilter{ViewID: &view.ID}, helpers.QueryParams(r.URL))
templates, err := helpers.Store(r).GetTemplates(
project.ID,
db.TemplateFilter{ViewID: &view.ID},
helpers.QueryParams(r.URL),
false)

if err != nil {
helpers.WriteError(w, err)
Expand Down
2 changes: 1 addition & 1 deletion db/Store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ type ProjectStore interface {

// TemplateManager handles template-related operations
type TemplateManager interface {
GetTemplates(projectID int, filter TemplateFilter, params RetrieveQueryParams) ([]Template, error)
GetTemplates(projectID int, filter TemplateFilter, params RetrieveQueryParams, loadVaults bool) ([]Template, error)
GetTemplateRefs(projectID int, templateID int) (ObjectReferrers, error)
CreateTemplate(template Template) (Template, error)
UpdateTemplate(template Template) error
Expand Down
2 changes: 1 addition & 1 deletion db/bolt/template.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ func (d *BoltDb) SetTemplateDescription(projectID int, templateID int, descripti
return err
}

func (d *BoltDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.RetrieveQueryParams) (templates []db.Template, err error) {
func (d *BoltDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.RetrieveQueryParams, loadVaults bool) (templates []db.Template, err error) {
var ftr = func(tpl any) bool {
template := tpl.(db.Template)
var res = true
Expand Down
7 changes: 0 additions & 7 deletions db/sql/project.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,13 +74,6 @@ func (d *SqlDb) GetProject(projectID int) (project db.Project, err error) {

func (d *SqlDb) DeleteProject(projectID int) error {

//tpls, err := d.GetTemplates(projectID, db.TemplateFilter{}, db.RetrieveQueryParams{})
//
//if err != nil {
// return err
//}
// TODO: sort projects

tx, err := d.sql.Begin()

if err != nil {
Expand Down
8 changes: 7 additions & 1 deletion db/sql/task.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,12 @@ func (d *SqlDb) CreateTaskOutput(output db.TaskOutput) (db.TaskOutput, error) {
}

func (d *SqlDb) getTasks(projectID int, templateID *int, taskIDs []int, params db.RetrieveQueryParams, tasks *[]db.TaskWithTpl) (err error) {

if taskIDs != nil && len(taskIDs) == 0 {
*tasks = []db.TaskWithTpl{}
return nil
}

fields := "task.*"
fields += ", tpl.playbook as tpl_playbook" +
", `user`.name as user_name" +
Expand All @@ -249,7 +255,7 @@ func (d *SqlDb) getTasks(projectID int, templateID *int, taskIDs []int, params d
q = q.Where("tpl.project_id=? AND task.template_id=?", projectID, templateID)
}

if len(taskIDs) > 0 {
if taskIDs != nil {
q = q.Where(squirrel.Eq{"task.id": taskIDs})
}

Expand Down
23 changes: 15 additions & 8 deletions db/sql/template.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ func (d *SqlDb) SetTemplateDescription(projectID int, templateID int, descriptio
return
}

func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.RetrieveQueryParams) (templates []db.Template, err error) {
func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.RetrieveQueryParams, loadVaults bool) (templates []db.Template, err error) {

pp, err := params.Validate(db.TemplateProps)
if err != nil {
Expand Down Expand Up @@ -230,6 +230,14 @@ func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.
OrderBy("pt.name " + order)
}

if params.Count > 0 {
q = q.Limit(uint64(params.Count))
}

if params.Offset > 0 {
q = q.Offset(uint64(params.Offset))
}

query, args, err := q.ToSql()

if err != nil {
Expand All @@ -253,6 +261,7 @@ func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.
}

var tasks []db.TaskWithTpl

err = d.getTasks(projectID, nil, taskIDs, db.RetrieveQueryParams{}, &tasks)

if err != nil {
Expand All @@ -265,10 +274,6 @@ func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.
if tpl.LastTaskID != nil {
for _, tsk := range tasks {
if tsk.ID == *tpl.LastTaskID {
err = tsk.Fill(d)
if err != nil {
return
}
template.LastTask = &tsk
break
}
Expand All @@ -283,9 +288,11 @@ func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.
return
}

template.Vaults, err = d.GetTemplateVaults(projectID, template.ID)
if err != nil {
return
if loadVaults {
template.Vaults, err = d.GetTemplateVaults(projectID, template.ID)
if err != nil {
return
}
}

templates = append(templates, template)
Expand Down
2 changes: 1 addition & 1 deletion services/project/backup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ func (b *BackupDB) makeUniqueNames() {

func (b *BackupDB) load(projectID int, store db.Store) (err error) {

b.templates, err = store.GetTemplates(projectID, db.TemplateFilter{}, db.RetrieveQueryParams{})
b.templates, err = store.GetTemplates(projectID, db.TemplateFilter{}, db.RetrieveQueryParams{}, true)
if err != nil {
return
}
Expand Down
7 changes: 4 additions & 3 deletions services/tasks/TaskRunner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,14 @@ package tasks
import (
"encoding/json"
"errors"
"github.com/semaphoreui/semaphore/pkg/tz"
"github.com/semaphoreui/semaphore/services/tasks/hooks"
"os"
"strconv"
"strings"
"sync"

"github.com/semaphoreui/semaphore/pkg/tz"
"github.com/semaphoreui/semaphore/services/tasks/hooks"

"github.com/semaphoreui/semaphore/api/sockets"
"github.com/semaphoreui/semaphore/db"
"github.com/semaphoreui/semaphore/pkg/task_logger"
Expand Down Expand Up @@ -213,7 +214,7 @@ func (t *TaskRunner) run() {
tpls, err := t.pool.store.GetTemplates(t.Task.ProjectID, db.TemplateFilter{
BuildTemplateID: &t.Task.TemplateID,
AutorunOnly: true,
}, db.RetrieveQueryParams{})
}, db.RetrieveQueryParams{}, true)

if err != nil {
t.Log("Running app failed: " + err.Error())
Expand Down
7 changes: 1 addition & 6 deletions web/src/views/project/Templates.vue
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,13 +129,12 @@
<v-divider style="margin-top: -1px;"/>

<v-data-table
hide-default-footer
class="mt-4 templates-table"
single-expand
show-expand
:headers="filteredHeaders"
:items="items"
:items-per-page="Number.MAX_VALUE"
:items-per-page="100"
Copy link
Preview

Copilot AI Jul 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[nitpick] Consider extracting the magic number 100 into a constant or prop to make the page size configurable and improve readability.

Copilot uses AI. Check for mistakes.

:expanded.sync="openedItems"
:style="{
opacity: viewItemsLoading ? 0.3 : 1,
Expand All @@ -149,10 +148,6 @@
{{ getAppIcon(item.app) }}
</v-icon>

<!-- <v-icon class="mr-3" small>-->
<!-- {{ TEMPLATE_TYPE_ICONS[item.type] }}-->
<!-- </v-icon>-->

<router-link
:to="viewId
? `/project/${projectId}/views/${viewId}/templates/${item.id}`
Expand Down
Loading