Skip to content

Add webhook payload size optimization options #35129

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion models/migrations/migrations.go
Original file line number Diff line number Diff line change
Expand Up @@ -383,9 +383,9 @@ func prepareMigrationTasks() []*migration {
newMigration(318, "Add anonymous_access_mode for repo_unit", v1_24.AddRepoUnitAnonymousAccessMode),
newMigration(319, "Add ExclusiveOrder to Label table", v1_24.AddExclusiveOrderColumnToLabelTable),
newMigration(320, "Migrate two_factor_policy to login_source table", v1_24.MigrateSkipTwoFactor),

// Gitea 1.24.0 ends at database version 321
newMigration(321, "Use LONGTEXT for some columns and fix review_state.updated_files column", v1_25.UseLongTextInSomeColumnsAndFixBugs),
newMigration(322, "Add webhook payload optimization JSON field", v1_25.AddWebhookPayloadOptimizationColumns),
}
return preparedMigrations
}
Expand Down
22 changes: 22 additions & 0 deletions models/migrations/v1_25/v322.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
// Copyright 2025 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package v1_25

import (
"xorm.io/xorm"
)

func AddWebhookPayloadOptimizationColumns(x *xorm.Engine) error {
type Webhook struct {
MetaSettings string `xorm:"meta_settings TEXT"`
}
_, err := x.SyncWithOptions(
xorm.SyncOptions{
IgnoreConstrains: true,
IgnoreIndices: true,
},
new(Webhook),
)
return err
}
127 changes: 127 additions & 0 deletions models/webhook/webhook.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,38 @@ import (
"xorm.io/builder"
)

// MetaSettings represents the metadata settings for webhook
type MetaSettings struct {
PayloadOptimization *PayloadOptimizationConfig `json:"payload_optimization,omitempty"` // Payload optimization configuration
}

// PayloadOptimizationConfig represents the configuration for webhook payload optimization
type PayloadOptimizationConfig struct {
Files *PayloadOptimizationItem `json:"files,omitempty"` // Files optimization config
Commits *PayloadOptimizationItem `json:"commits,omitempty"` // Commits optimization config
}

// PayloadOptimizationItem represents a single optimization item configuration
type PayloadOptimizationItem struct {
Enable bool `json:"enable"` // Whether to enable optimization for this item
Limit int `json:"limit"` // 0: trim all (none kept), >0: keep N items (forward order), <0: keep N items (reverse order)
}

// DefaultMetaSettings returns the default webhook meta settings
func DefaultMetaSettings() *MetaSettings {
return &MetaSettings{
PayloadOptimization: DefaultPayloadOptimizationConfig(),
}
}

// DefaultPayloadOptimizationConfig returns the default payload optimization configuration
func DefaultPayloadOptimizationConfig() *PayloadOptimizationConfig {
return &PayloadOptimizationConfig{
Files: &PayloadOptimizationItem{Enable: false, Limit: 0},
Commits: &PayloadOptimizationItem{Enable: false, Limit: 0},
}
}

// ErrWebhookNotExist represents a "WebhookNotExist" kind of error.
type ErrWebhookNotExist struct {
ID int64
Expand Down Expand Up @@ -139,6 +171,9 @@ type Webhook struct {
// HeaderAuthorizationEncrypted should be accessed using HeaderAuthorization() and SetHeaderAuthorization()
HeaderAuthorizationEncrypted string `xorm:"TEXT"`

// Webhook metadata settings (JSON format)
MetaSettings string `xorm:"meta_settings TEXT"` // JSON: webhook metadata configuration

CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
}
Expand Down Expand Up @@ -346,3 +381,95 @@ func DeleteWebhookByOwnerID(ctx context.Context, ownerID, id int64) error {
}
return DeleteWebhookByID(ctx, id)
}

// GetMetaSettings returns the webhook meta settings
func (w *Webhook) GetMetaSettings() *MetaSettings {
if w.MetaSettings == "" {
return DefaultMetaSettings()
}

var settings MetaSettings
if err := json.Unmarshal([]byte(w.MetaSettings), &settings); err != nil {
log.Error("Failed to unmarshal webhook meta settings: %v", err)
return DefaultMetaSettings()
}

// Ensure payload optimization config is initialized
if settings.PayloadOptimization == nil {
settings.PayloadOptimization = DefaultPayloadOptimizationConfig()
}

return &settings
}

// GetPayloadOptimizationConfig returns the payload optimization configuration
func (w *Webhook) GetPayloadOptimizationConfig() *PayloadOptimizationConfig {
return w.GetMetaSettings().PayloadOptimization
}

// SetMetaSettings sets the webhook meta settings
func (w *Webhook) SetMetaSettings(settings *MetaSettings) error {
if settings == nil {
settings = DefaultMetaSettings()
}

data, err := json.Marshal(settings)
if err != nil {
return fmt.Errorf("failed to marshal webhook meta settings: %w", err)
}

w.MetaSettings = string(data)
return nil
}

// SetPayloadOptimizationConfig sets the payload optimization configuration
func (w *Webhook) SetPayloadOptimizationConfig(config *PayloadOptimizationConfig) error {
settings := w.GetMetaSettings()
if config == nil {
config = DefaultPayloadOptimizationConfig()
}
settings.PayloadOptimization = config
return w.SetMetaSettings(settings)
}

// IsPayloadOptimizationEnabled returns whether payload optimization is enabled
func (w *Webhook) IsPayloadOptimizationEnabled() bool {
config := w.GetPayloadOptimizationConfig()
return config.Files.Enable || config.Commits.Enable
}

// GetPayloadOptimizationLimit returns the payload optimization limit
func (w *Webhook) GetPayloadOptimizationLimit() int {
config := w.GetPayloadOptimizationConfig()
if config.Files.Enable {
return config.Files.Limit
}
if config.Commits.Enable {
return config.Commits.Limit
}
return 0
}

// IsFilesOptimizationEnabled returns whether files optimization is enabled
func (w *Webhook) IsFilesOptimizationEnabled() bool {
config := w.GetPayloadOptimizationConfig()
return config.Files.Enable
}

// GetFilesOptimizationLimit returns the files optimization limit
func (w *Webhook) GetFilesOptimizationLimit() int {
config := w.GetPayloadOptimizationConfig()
return config.Files.Limit
}

// IsCommitsOptimizationEnabled returns whether commits optimization is enabled
func (w *Webhook) IsCommitsOptimizationEnabled() bool {
config := w.GetPayloadOptimizationConfig()
return config.Commits.Enable
}

// GetCommitsOptimizationLimit returns the commits optimization limit
func (w *Webhook) GetCommitsOptimizationLimit() int {
config := w.GetPayloadOptimizationConfig()
return config.Commits.Limit
}
60 changes: 60 additions & 0 deletions models/webhook/webhook_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -330,3 +330,63 @@ func TestCleanupHookTaskTable_OlderThan_LeavesTaskEarlierThanAgeToDelete(t *test
assert.NoError(t, CleanupHookTaskTable(t.Context(), OlderThan, 168*time.Hour, 0))
unittest.AssertExistsAndLoadBean(t, hookTask)
}

func TestWebhookPayloadOptimization(t *testing.T) {
webhook := &Webhook{}

// Test default configuration
config := webhook.GetPayloadOptimizationConfig()
assert.False(t, config.Files.Enable)
assert.Equal(t, 0, config.Files.Limit)
assert.False(t, config.Commits.Enable)
assert.Equal(t, 0, config.Commits.Limit)

// Test setting configuration via meta settings
metaSettings := &MetaSettings{
PayloadOptimization: &PayloadOptimizationConfig{
Files: &PayloadOptimizationItem{
Enable: true,
Limit: 5,
},
Commits: &PayloadOptimizationItem{
Enable: true,
Limit: -3,
},
},
}
webhook.SetMetaSettings(metaSettings)

// Test getting configuration
config = webhook.GetPayloadOptimizationConfig()
assert.True(t, config.Files.Enable)
assert.Equal(t, 5, config.Files.Limit)
assert.True(t, config.Commits.Enable)
assert.Equal(t, -3, config.Commits.Limit)

// Test individual methods
assert.True(t, webhook.IsFilesOptimizationEnabled())
assert.Equal(t, 5, webhook.GetFilesOptimizationLimit())
assert.True(t, webhook.IsCommitsOptimizationEnabled())
assert.Equal(t, -3, webhook.GetCommitsOptimizationLimit())
assert.True(t, webhook.IsPayloadOptimizationEnabled())

// Test backward compatibility with direct payload optimization config setting
newConfig := &PayloadOptimizationConfig{
Files: &PayloadOptimizationItem{
Enable: false,
Limit: 10,
},
Commits: &PayloadOptimizationItem{
Enable: false,
Limit: 20,
},
}
webhook.SetPayloadOptimizationConfig(newConfig)

// Verify the config is properly set through meta settings
config = webhook.GetPayloadOptimizationConfig()
assert.False(t, config.Files.Enable)
assert.Equal(t, 10, config.Files.Limit)
assert.False(t, config.Commits.Enable)
assert.Equal(t, 20, config.Commits.Limit)
}
8 changes: 7 additions & 1 deletion modules/structs/hook.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ type Hook struct {
Events []string `json:"events"`
AuthorizationHeader string `json:"authorization_header"`
Active bool `json:"active"`
// MetaSettings webhook metadata settings including payload optimization
MetaSettings map[string]any `json:"meta_settings"`
// swagger:strfmt date-time
Updated time.Time `json:"updated_at"`
// swagger:strfmt date-time
Expand All @@ -48,6 +50,8 @@ type CreateHookOption struct {
Events []string `json:"events"`
BranchFilter string `json:"branch_filter" binding:"GlobPattern"`
AuthorizationHeader string `json:"authorization_header"`
// Webhook metadata settings including payload optimization
MetaSettings map[string]any `json:"meta_settings"` // {"payload_optimization": {"files": {"enable": bool, "limit": int}, "commits": {"enable": bool, "limit": int}}}
// default: false
Active bool `json:"active"`
}
Expand All @@ -58,7 +62,9 @@ type EditHookOption struct {
Events []string `json:"events"`
BranchFilter string `json:"branch_filter" binding:"GlobPattern"`
AuthorizationHeader string `json:"authorization_header"`
Active *bool `json:"active"`
// Webhook metadata settings including payload optimization
MetaSettings *map[string]any `json:"meta_settings"` // {"payload_optimization": {"files": {"enable": bool, "limit": int}, "commits": {"enable": bool, "limit": int}}}
Active *bool `json:"active"`
}

// Payloader payload is some part of one hook
Expand Down
9 changes: 8 additions & 1 deletion options/locale/locale_en-US.ini
Original file line number Diff line number Diff line change
Expand Up @@ -2425,6 +2425,13 @@ settings.event_package = Package
settings.event_package_desc = Package created or deleted in a repository.
settings.branch_filter = Branch filter
settings.branch_filter_desc = Branch whitelist for push, branch creation and branch deletion events, specified as glob pattern. If empty or <code>*</code>, events for all branches are reported. See <a href="%[1]s">%[2]s</a> documentation for syntax. Examples: <code>master</code>, <code>{master,release*}</code>.
settings.payload_optimization = Payload Size Optimization
settings.payload_optimization_files = Files
settings.payload_optimization_commits = Commits
settings.payload_optimization_enable = Enable optimization
settings.payload_optimization_enable_desc = Enable payload size optimization for this item
settings.payload_optimization_limit = Limit
settings.payload_optimization_limit_desc = 0: trim all (none kept), >0: keep N items (forward order), <0: keep N items (reverse order)
settings.authorization_header = Authorization Header
settings.authorization_header_desc = Will be included as authorization header for requests when present. Examples: %s.
settings.active = Active
Expand Down Expand Up @@ -3283,7 +3290,7 @@ auths.tip.github = Register a new OAuth application on %s
auths.tip.gitlab_new = Register a new application on %s
auths.tip.google_plus = Obtain OAuth2 client credentials from the Google API console at %s
auths.tip.openid_connect = Use the OpenID Connect Discovery URL "https://{server}/.well-known/openid-configuration" to specify the endpoints
auths.tip.twitter = Go to %s, create an application and ensure that the Allow this application to be used to Sign in with Twitter option is enabled
auths.tip.twitter = Go to %s, create an application and ensure that the "Allow this application to be used to Sign in with Twitter" option is enabled
auths.tip.discord = Register a new application on %s
auths.tip.gitea = Register a new OAuth2 application. Guide can be found at %s
auths.tip.yandex = Create a new application at %s. Select following permissions from the "Yandex.Passport API" section: "Access to email address", "Access to user avatar" and "Access to username, first name and surname, gender"
Expand Down
Loading