diff --git a/README.md b/README.md index 6fa99404..10186b4f 100644 --- a/README.md +++ b/README.md @@ -19,9 +19,11 @@ Needs postgresql DSN, if no database is selected in the DSN, it will use all dat If you are having issues with the connector, please check the following: - Specified database - - User needs permission to access the database, this will cause missing resources. -- Not specified database - - User needs permission for each database, otherwise it will not be able to read the resources for that database. + - User needs permission to access the database, this will cause missing resources. +- Sync all databases + - Use `--dsn` and `--sync-all-databases` flag for postgres db `postgres://user:password@localhost:5432/postgres` + - Needs access to `"pg_catalog"."pg_database"` to get all databases + - Needs permission for each database to get all resources ## brew @@ -85,23 +87,28 @@ Usage: Available Commands: capabilities Get connector capabilities completion Generate the autocompletion script for the specified shell + config Get the connector config schema help Help about any command Flags: - --client-id string The client ID used to authenticate with ConductorOne ($BATON_CLIENT_ID) - --client-secret string The client secret used to authenticate with ConductorOne ($BATON_CLIENT_SECRET) - --dsn string required: The DSN to connect to the database ($BATON_DSN) - -f, --file string The path to the c1z file to sync with ($BATON_FILE) (default "sync.c1z") - -h, --help help for baton-postgresql - --include-columns Include column privileges when syncing. This can result in large amounts of data ($BATON_INCLUDE_COLUMNS) - --include-large-objects Include large objects when syncing. This can result in large amounts of data ($BATON_INCLUDE_LARGE_OBJECTS) - --log-format string The output format for logs: json, console ($BATON_LOG_FORMAT) (default "json") - --log-level string The log level: debug, info, warn, error ($BATON_LOG_LEVEL) (default "info") - -p, --provisioning This must be set in order for provisioning actions to be enabled ($BATON_PROVISIONING) - --schemas strings The schemas to include in the sync ($BATON_SCHEMAS) (default [public]) - --skip-full-sync This must be set to skip a full sync ($BATON_SKIP_FULL_SYNC) - --ticketing This must be set to enable ticketing support ($BATON_TICKETING) - -v, --version version for baton-postgresql + --client-id string The client ID used to authenticate with ConductorOne ($BATON_CLIENT_ID) + --client-secret string The client secret used to authenticate with ConductorOne ($BATON_CLIENT_SECRET) + --dsn string required: The DSN to connect to the database ($BATON_DSN) + --external-resource-c1z string The path to the c1z file to sync external baton resources with ($BATON_EXTERNAL_RESOURCE_C1Z) + --external-resource-entitlement-id-filter string The entitlement that external users, groups must have access to sync external baton resources ($BATON_EXTERNAL_RESOURCE_ENTITLEMENT_ID_FILTER) + -f, --file string The path to the c1z file to sync with ($BATON_FILE) (default "sync.c1z") + -h, --help help for baton-postgresql + --include-columns Include column privileges when syncing. This can result in large amounts of data ($BATON_INCLUDE_COLUMNS) + --include-large-objects Include large objects when syncing. This can result in large amounts of data ($BATON_INCLUDE_LARGE_OBJECTS) + --log-format string The output format for logs: json, console ($BATON_LOG_FORMAT) (default "json") + --log-level string The log level: debug, info, warn, error ($BATON_LOG_LEVEL) (default "info") + --otel-collector-endpoint string The endpoint of the OpenTelemetry collector to send observability data to (used for both tracing and logging if specific endpoints are not provided) ($BATON_OTEL_COLLECTOR_ENDPOINT) + -p, --provisioning This must be set in order for provisioning actions to be enabled ($BATON_PROVISIONING) + --schemas strings The schemas to include in the sync ($BATON_SCHEMAS) (default [public]) + --skip-full-sync This must be set to skip a full sync ($BATON_SKIP_FULL_SYNC) + --sync-all-databases Sync all databases. This can result in large amounts of data ($BATON_SYNC_ALL_DATABASES) + --ticketing This must be set to enable ticketing support ($BATON_TICKETING) + -v, --version version for baton-postgresql Use "baton-postgresql [command] --help" for more information about a command. ``` diff --git a/cmd/baton-postgresql/main.go b/cmd/baton-postgresql/main.go index e3e469e4..da0d7766 100644 --- a/cmd/baton-postgresql/main.go +++ b/cmd/baton-postgresql/main.go @@ -38,7 +38,7 @@ func main() { func getConnector(ctx context.Context, pgc *cfg.Postgresql) (types.ConnectorServer, error) { l := ctxzap.Extract(ctx) - cb, err := connector.New(ctx, pgc.Dsn, pgc.Schemas, pgc.IncludeColumns, pgc.IncludeLargeObjects) + cb, err := connector.New(ctx, pgc.Dsn, pgc.Schemas, pgc.IncludeColumns, pgc.IncludeLargeObjects, pgc.SyncAllDatabases) if err != nil { l.Error("error creating connector", zap.Error(err)) return nil, err diff --git a/pkg/config/conf.gen.go b/pkg/config/conf.gen.go index 3bdbf1f6..0191d414 100644 --- a/pkg/config/conf.gen.go +++ b/pkg/config/conf.gen.go @@ -8,6 +8,7 @@ type Postgresql struct { Schemas []string `mapstructure:"schemas"` IncludeColumns bool `mapstructure:"include-columns"` IncludeLargeObjects bool `mapstructure:"include-large-objects"` + SyncAllDatabases bool `mapstructure:"sync-all-databases"` } func (c* Postgresql) findFieldByTag(tagValue string) (any, bool) { diff --git a/pkg/config/config.go b/pkg/config/config.go index bb4b164e..4e01ced5 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -9,11 +9,12 @@ var ( schemas = field.StringSliceField("schemas", field.WithDefaultValue([]string{"public"}), field.WithDescription("The schemas to include in the sync")) includeColumns = field.BoolField("include-columns", field.WithDescription("Include column privileges when syncing. This can result in large amounts of data")) includeLargeObjects = field.BoolField("include-large-objects", field.WithDescription("Include large objects when syncing. This can result in large amounts of data")) + syncAllDatabases = field.BoolField("sync-all-databases", field.WithDescription("Sync all databases. This can result in large amounts of data"), field.WithDefaultValue(false)) ) var relationships = []field.SchemaFieldRelationship{} //go:generate go run ./gen var Config = field.NewConfiguration([]field.SchemaField{ - dsn, schemas, includeColumns, includeLargeObjects, + dsn, schemas, includeColumns, includeLargeObjects, syncAllDatabases, }, relationships...) diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index 88f75de9..07663428 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -16,6 +16,7 @@ type Postgresql struct { schemas []string includeColumns bool includeLargeObjects bool + syncAllDatabases bool } func (o *Postgresql) ResourceSyncers(ctx context.Context) []connectorbuilder.ResourceSyncer { @@ -28,7 +29,7 @@ func (o *Postgresql) ResourceSyncers(ctx context.Context) []connectorbuilder.Res newFunctionSyncer(ctx, o.clientPool), newProcedureSyncer(ctx, o.clientPool), newLargeObjectSyncer(ctx, o.clientPool.Default(ctx), o.includeLargeObjects), - newDatabaseSyncer(ctx, o.clientPool), + newDatabaseSyncer(ctx, o.clientPool, o.syncAllDatabases), newSequenceSyncer(ctx, o.clientPool), } } @@ -60,7 +61,7 @@ func (c *Postgresql) Asset(ctx context.Context, asset *v2.AssetRef) (string, io. return "", nil, fmt.Errorf("not implemented") } -func New(ctx context.Context, dsn string, schemas []string, includeColumns bool, includeLargeObjects bool) (*Postgresql, error) { +func New(ctx context.Context, dsn string, schemas []string, includeColumns bool, includeLargeObjects bool, syncAllDatabases bool) (*Postgresql, error) { clientPool, err := postgres.NewClientDatabasesPool(ctx, dsn, postgres.WithSchemaFilter(schemas)) if err != nil { return nil, fmt.Errorf("failed to create postgres client pool: %w", err) @@ -71,5 +72,6 @@ func New(ctx context.Context, dsn string, schemas []string, includeColumns bool, schemas: schemas, includeColumns: includeColumns, includeLargeObjects: includeLargeObjects, + syncAllDatabases: syncAllDatabases, }, nil } diff --git a/pkg/connector/database.go b/pkg/connector/database.go index 46f47c0b..d027b57a 100644 --- a/pkg/connector/database.go +++ b/pkg/connector/database.go @@ -24,9 +24,10 @@ var databaseResourceType = &v2.ResourceType{ } type databaseSyncer struct { - resourceType *v2.ResourceType - clientPool *postgres.ClientDatabasesPool - client *postgres.Client + resourceType *v2.ResourceType + clientPool *postgres.ClientDatabasesPool + client *postgres.Client + syncAllDatabases bool } func (r *databaseSyncer) ResourceType(ctx context.Context) *v2.ResourceType { @@ -64,11 +65,11 @@ func (r *databaseSyncer) List(ctx context.Context, parentResourceID *v2.Resource return nil, "", nil, err } - defaultDatabase := r.clientPool.DefaultDatabase(ctx) + defaultDbClient := r.clientPool.Default(ctx) var ret []*v2.Resource for _, o := range databases { - if defaultDatabase != "" && o.Name != defaultDatabase { + if !r.syncAllDatabases && o.Name != defaultDbClient.DatabaseName() { continue } @@ -329,10 +330,11 @@ func (r *databaseSyncer) Revoke(ctx context.Context, grant *v2.Grant) (annotatio return nil, err } -func newDatabaseSyncer(ctx context.Context, c *postgres.ClientDatabasesPool) *databaseSyncer { +func newDatabaseSyncer(ctx context.Context, c *postgres.ClientDatabasesPool, syncAllDatabases bool) *databaseSyncer { return &databaseSyncer{ - resourceType: databaseResourceType, - clientPool: c, - client: c.Default(ctx), + resourceType: databaseResourceType, + clientPool: c, + client: c.Default(ctx), + syncAllDatabases: syncAllDatabases, } } diff --git a/pkg/postgres/client.go b/pkg/postgres/client.go index d4bbfc21..5f2bb670 100644 --- a/pkg/postgres/client.go +++ b/pkg/postgres/client.go @@ -2,7 +2,6 @@ package postgres import ( "context" - "errors" "sync" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" @@ -11,33 +10,17 @@ import ( ) type ClientDatabasesPool struct { - databases map[string]*Client - opts []ClientOpt - mutex *sync.Mutex - logger *Logger - dsn string - // Only used if dsn have any database name - defaultClientWithDatabase *Client - defaultClientDsn *Client + databases map[string]*Client + opts []ClientOpt + mutex *sync.Mutex + logger *Logger + dsn string + defaultClientDsn *Client } func NewClientDatabasesPool(ctx context.Context, dsn string, opts ...ClientOpt) (*ClientDatabasesPool, error) { l := ctxzap.Extract(ctx) - config, err := pgxpool.ParseConfig(dsn) - if err != nil { - return nil, err - } - - var defaultClientWithDatabase *Client - if config.ConnConfig.Database != "" { - l.Info("using default database database config", zap.String("database", config.ConnConfig.Database)) - defaultClientWithDatabase, err = New(ctx, dsn, opts...) - if err != nil { - return nil, err - } - } - defaultClientDsn, err := New(ctx, dsn, opts...) if err != nil { l.Error("failed to create default database client", zap.Error(err)) @@ -45,24 +28,15 @@ func NewClientDatabasesPool(ctx context.Context, dsn string, opts ...ClientOpt) } return &ClientDatabasesPool{ - dsn: dsn, - databases: make(map[string]*Client), - opts: opts, - mutex: &sync.Mutex{}, - logger: &Logger{}, - defaultClientWithDatabase: defaultClientWithDatabase, - defaultClientDsn: defaultClientDsn, + dsn: dsn, + databases: make(map[string]*Client), + opts: opts, + mutex: &sync.Mutex{}, + logger: &Logger{}, + defaultClientDsn: defaultClientDsn, }, nil } -func (c *ClientDatabasesPool) DefaultDatabase(ctx context.Context) string { - if c.defaultClientWithDatabase == nil { - return "" - } - - return c.defaultClientWithDatabase.cfg.ConnConfig.Database -} - func (p *ClientDatabasesPool) Default(ctx context.Context) *Client { return p.defaultClientDsn } @@ -70,15 +44,6 @@ func (p *ClientDatabasesPool) Default(ctx context.Context) *Client { func (p *ClientDatabasesPool) Get(ctx context.Context, database string) (*Client, string, error) { l := ctxzap.Extract(ctx) - if database == "" { - if p.defaultClientWithDatabase != nil { - dbName := p.defaultClientWithDatabase.db.Config().ConnConfig.Database - return p.defaultClientWithDatabase, dbName, nil - } - - return nil, "", errors.New("database name is required") - } - dbModel, err := p.defaultClientDsn.GetDatabaseById(ctx, database) if err != nil { return nil, "", err @@ -179,3 +144,7 @@ func New(ctx context.Context, dsn string, opts ...ClientOpt) (*Client, error) { return c, nil } + +func (c *Client) DatabaseName() string { + return c.cfg.ConnConfig.Database +}