Skip to content

Commit abd7531

Browse files
ayu-devtronayu-devtron
authored
Merge pull request #6243 from devtron-labs/pg-enable-metrics
misc: pg enable metrics
2 parents 366bf84 + a9cd284 commit abd7531

24 files changed

+732
-567
lines changed

‎env_gen.md

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -200,9 +200,10 @@
200200
| PARALLELISM_LIMIT_FOR_TAG_PROCESSING | | |
201201
| PG_ADDR | 127.0.0.1 | |
202202
| PG_DATABASE | orchestrator | |
203-
| PG_EXPORT_PROM_METRICS | false | |
203+
| PG_EXPORT_PROM_METRICS | true | |
204+
| PG_LOG_ALL_FAILURE_QUERIES | true | |
204205
| PG_LOG_ALL_QUERY | false | |
205-
| PG_LOG_QUERY | true | |
206+
| PG_LOG_SLOW_QUERY | true | |
206207
| PG_PASSWORD | {password} | |
207208
| PG_PORT | 5432 | |
208209
| PG_QUERY_DUR_THRESHOLD | 5000 | |

‎go.mod

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -271,7 +271,7 @@ require (
271271
k8s.io/klog/v2 v2.110.1 // indirect
272272
k8s.io/kube-aggregator v0.29.6 // indirect
273273
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
274-
mellium.im/sasl v0.3.1 // indirect
274+
mellium.im/sasl v0.3.2 // indirect
275275
oras.land/oras-go/v2 v2.3.0 // indirect
276276
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
277277
sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
@@ -288,8 +288,8 @@ require gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
288288

289289
replace (
290290
github.com/argoproj/argo-workflows/v3 v3.5.10 => github.com/devtron-labs/argo-workflows/v3 v3.5.13
291-
github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250111030602-92e4c3e15f98
292-
github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250111030602-92e4c3e15f98
291+
github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250112200957-a668cc3673da
292+
github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250112200957-a668cc3673da
293293
github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127
294294
github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5
295295
k8s.io/api => k8s.io/api v0.29.7

‎go.sum

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -792,10 +792,10 @@ github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzq
792792
github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
793793
github.com/devtron-labs/argo-workflows/v3 v3.5.13 h1:3pINq0gXOSeTw2z/vYe+j80lRpSN5Rp/8mfQORh8SmU=
794794
github.com/devtron-labs/argo-workflows/v3 v3.5.13/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA=
795-
github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250111030602-92e4c3e15f98 h1:JsIuIbPBT0FmO+RxuNNyXEaxRpBDfWf0hjHvjuX57Q8=
796-
github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250111030602-92e4c3e15f98/go.mod h1:5lv4Wfj5ERhhvDGXe2IeES6qxjvUVCcohaRwKnWBMNo=
797-
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250111030602-92e4c3e15f98 h1:TP5AAayaxEyvbr1Yf9rHBQJpaBz0Tl/LStgnLJId+TQ=
798-
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250111030602-92e4c3e15f98/go.mod h1:GXADr0W94UqtDJe6/3possrPj6fD/3IUNISfYfevWqg=
795+
github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250112200957-a668cc3673da h1:xGvqLLRlPfoLq+mvIoN0tOLLkWYrsEh2vhCORb3EDfY=
796+
github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250112200957-a668cc3673da/go.mod h1:5lv4Wfj5ERhhvDGXe2IeES6qxjvUVCcohaRwKnWBMNo=
797+
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250112200957-a668cc3673da h1:hTYetUW3nH8bhiZu+s7GUg+Neh9P5CWWlZGigwCAzHA=
798+
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250112200957-a668cc3673da/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA=
799799
github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU=
800800
github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y=
801801
github.com/devtron-labs/protos v0.0.3-0.20240802105333-92ee9bb85d80 h1:xwbTeijNTf4/j1v+tSfwVqwLVnReas/NqEKeQHvSTys=
@@ -2493,8 +2493,8 @@ launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80
24932493
launchpad.net/xmlpath v0.0.0-20130614043138-000000000004/go.mod h1:vqyExLOM3qBx7mvYRkoxjSCF945s0mbe7YynlKYXtsA=
24942494
lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
24952495
lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
2496-
mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo=
2497-
mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw=
2496+
mellium.im/sasl v0.3.2 h1:PT6Xp7ccn9XaXAnJ03FcEjmAn7kK1x7aoXV6F+Vmrl0=
2497+
mellium.im/sasl v0.3.2/go.mod h1:NKXDi1zkr+BlMHLQjY3ofYuU4KSPFxknb8mfEu6SveY=
24982498
modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
24992499
modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
25002500
modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=

‎pkg/sql/connection.go

Lines changed: 20 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@
1717
package sql
1818

1919
import (
20-
"github.com/devtron-labs/devtron/internal/middleware"
20+
"github.com/devtron-labs/common-lib/utils"
21+
"github.com/devtron-labs/common-lib/utils/bean"
2122
"go.uber.org/zap"
2223
"reflect"
2324
"time"
@@ -27,24 +28,29 @@ import (
2728
)
2829

2930
type Config struct {
30-
Addr string `env:"PG_ADDR" envDefault:"127.0.0.1"`
31-
Port string `env:"PG_PORT" envDefault:"5432"`
32-
User string `env:"PG_USER" envDefault:""`
33-
Password string `env:"PG_PASSWORD" envDefault:"" secretData:"-"`
34-
Database string `env:"PG_DATABASE" envDefault:"orchestrator"`
35-
CasbinDatabase string `env:"CASBIN_DATABASE" envDefault:"casbin"`
31+
Addr string `env:"PG_ADDR" envDefault:"127.0.0.1"`
32+
Port string `env:"PG_PORT" envDefault:"5432"`
33+
User string `env:"PG_USER" envDefault:""`
34+
Password string `env:"PG_PASSWORD" envDefault:"" secretData:"-"`
35+
Database string `env:"PG_DATABASE" envDefault:"orchestrator"`
36+
CasbinDatabase string `env:"CASBIN_DATABASE" envDefault:"casbin"`
3637
ApplicationName string `env:"APP" envDefault:"orchestrator" envDescription:"Application name"`
37-
LogQuery bool `env:"PG_LOG_QUERY" envDefault:"true"`
38-
LogAllQuery bool `env:"PG_LOG_ALL_QUERY" envDefault:"false"`
39-
ExportPromMetrics bool `env:"PG_EXPORT_PROM_METRICS" envDefault:"false"`
40-
QueryDurationThreshold int64 `env:"PG_QUERY_DUR_THRESHOLD" envDefault:"5000"`
41-
ReadTimeout int64 `env:"PG_READ_TIMEOUT" envDefault:"30"`
42-
WriteTimeout int64 `env:"PG_WRITE_TIMEOUT" envDefault:"30"`
38+
ReadTimeout int64 `env:"PG_READ_TIMEOUT" envDefault:"30"`
39+
WriteTimeout int64 `env:"PG_WRITE_TIMEOUT" envDefault:"30"`
40+
bean.PgQueryMonitoringConfig
4341
}
4442

4543
func GetConfig() (*Config, error) {
4644
cfg := &Config{}
4745
err := env.Parse(cfg)
46+
if err != nil {
47+
return cfg, err
48+
}
49+
monitoringCfg, err := bean.GetPgQueryMonitoringConfig(cfg.ApplicationName)
50+
if err != nil {
51+
return cfg, err
52+
}
53+
cfg.PgQueryMonitoringConfig = monitoringCfg
4854
return cfg, err
4955
}
5056

@@ -71,28 +77,7 @@ func NewDbConnection(cfg *Config, logger *zap.SugaredLogger) (*pg.DB, error) {
7177
}
7278

7379
// --------------
74-
dbConnection.OnQueryProcessed(func(event *pg.QueryProcessedEvent) {
75-
queryDuration := time.Since(event.StartTime)
76-
77-
// Expose prom metrics
78-
if cfg.ExportPromMetrics {
79-
middleware.PgQueryDuration.WithLabelValues("value").Observe(queryDuration.Seconds())
80-
}
81-
82-
query, err := event.FormattedQuery()
83-
if err != nil {
84-
logger.Errorw("Error formatting query",
85-
"err", err)
86-
return
87-
}
88-
89-
// Log pg query if enabled
90-
if cfg.LogAllQuery || (cfg.LogQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold) {
91-
logger.Debugw("query time",
92-
"duration", queryDuration.Seconds(),
93-
"query", query)
94-
}
95-
})
80+
dbConnection.OnQueryProcessed(utils.GetPGPostQueryProcessor(cfg.PgQueryMonitoringConfig))
9681
return dbConnection, err
9782
}
9883

‎vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go

Lines changed: 57 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go

Lines changed: 23 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/mellium.im/sasl/CHANGELOG.md

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,18 @@
22

33
All notable changes to this project will be documented in this file.
44

5+
## v0.3.2 — 2024-09-11
6+
7+
### Added
8+
9+
- Support for SASL ANONYMOUS
10+
- Support for the server side of SCRAM
11+
12+
### Fixed
13+
14+
- Support for fast XOR removed from the repo and now uses the upstream version
15+
shipped with the Go tool chain (which supports more architectures)
16+
517

618
## v0.3.1 — 2022-12-28
719

‎vendor/mellium.im/sasl/anonymous.go

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
// Copyright 2024 The Mellium Contributors.
2+
// Use of this source code is governed by the BSD 2-clause
3+
// license that can be found in the LICENSE file.
4+
5+
package sasl
6+
7+
var anonymous = Mechanism{
8+
Name: "ANONYMOUS",
9+
Start: func(*Negotiator) (bool, []byte, interface{}, error) {
10+
// Per XEP-0175 we do not send any trace data.
11+
return false, nil, nil, nil
12+
},
13+
Next: func(m *Negotiator, _ []byte, _ interface{}) (_ bool, _ []byte, _ interface{}, err error) {
14+
if m.State()&Receiving != Receiving || m.State()&AuthTextSent != AuthTextSent {
15+
err = ErrTooManySteps
16+
}
17+
return
18+
},
19+
}

‎vendor/mellium.im/sasl/mechanism.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@ var (
4343
// ScramSha1 is a Mechanism that implements the SCRAM-SHA-1 authentication
4444
// mechanism defined in RFC 5802.
4545
ScramSha1 Mechanism = scram("SCRAM-SHA-1", sha1.New)
46+
47+
// Anonymous is a Mechanism that implements the ANONYMOUS
48+
// authentication mechanism as defined by RFC 4505.
49+
Anonymous Mechanism = anonymous
4650
)
4751

4852
// Mechanism represents a SASL mechanism that can be used by a Client or Server

‎vendor/mellium.im/sasl/negotiator.go

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ package sasl
77
import (
88
"crypto/rand"
99
"crypto/tls"
10+
"fmt"
1011
"strings"
1112
)
1213

@@ -85,18 +86,28 @@ func NewServer(m Mechanism, permissions func(*Negotiator) bool, opts ...Option)
8586
return machine
8687
}
8788

89+
// SaltedCredentialsFetcher defines function that fetches user information using
90+
// Username and optional Identity from storage.
91+
//
92+
// Function should return saltedPassword as well as salt and iterations used
93+
// to generate saltedPassword for specified SCRAM MechanismName. If there is no
94+
// such Username or Username is not authorized to take Identity function should
95+
// return ErrAuthn as an error.
96+
type SaltedCredentialsFetcher func(Username, Identity []byte, MechanismName string) (salt []byte, saltedPassword []byte, iterations int64, err error)
97+
8898
// A Negotiator represents a SASL client or server state machine that can
8999
// attempt to negotiate auth. Negotiators should not be used from multiple
90100
// goroutines, and must be reset between negotiation attempts.
91101
type Negotiator struct {
92-
tlsState *tls.ConnectionState
93-
remoteMechanisms []string
94-
credentials func() (Username, Password, Identity []byte)
95-
permissions func(*Negotiator) bool
96-
mechanism Mechanism
97-
state State
98-
nonce []byte
99-
cache interface{}
102+
tlsState *tls.ConnectionState
103+
remoteMechanisms []string
104+
credentials func() (Username, Password, Identity []byte) // client only
105+
saltedCredentials SaltedCredentialsFetcher // server only
106+
permissions func(*Negotiator) bool
107+
mechanism Mechanism
108+
state State
109+
nonce []byte
110+
cache interface{}
100111
}
101112

102113
// Nonce returns a unique nonce that is reset for each negotiation attempt. It
@@ -159,14 +170,25 @@ func (c *Negotiator) Reset() {
159170
}
160171

161172
// Credentials returns a username, and password for authentication and optional
162-
// identity for authorization.
173+
// identity for authorization. Used in client negotiator.
163174
func (c *Negotiator) Credentials() (username, password, identity []byte) {
164175
if c.credentials != nil {
165176
return c.credentials()
166177
}
167178
return
168179
}
169180

181+
// SaltedCredentials returns a salt, saltedPassword and iteration count for a
182+
// given username and optional identity for client authorization. Refer to
183+
// SaltedCredentialsFetcher documentation for details.
184+
// Used in server negotiator.
185+
func (c *Negotiator) SaltedCredentials(username, identity []byte) (salt []byte, saltedPassword []byte, iterations int64, err error) {
186+
if c.saltedCredentials != nil {
187+
return c.saltedCredentials(username, identity, c.mechanism.Name)
188+
}
189+
return nil, nil, 0, fmt.Errorf("sasl: salted credentials not provided")
190+
}
191+
170192
// Permissions is the callback used by the server to authenticate the user.
171193
func (c *Negotiator) Permissions(opts ...Option) bool {
172194
if c.permissions != nil {

0 commit comments

Comments
 (0)