refactor: deprecate non-uri connection

docs: update

ci: update

fix: remove default options

chore: resolve issues

docs: update mvt provider readme too

Author: iwpnd

.github/workflows/on_pr_push.yml

@@ -79,12 +79,8 @@ jobs:
 
           # PostGIS tests
           RUN_POSTGIS_TESTS: yes
-          PGHOST: localhost
-          PGPORT: 5432
-          PGDATABASE: tegola
-          PGUSER: postgres
-          PGPASSWORD: postgres
-          PGUSER_NO_ACCESS: tegola_no_access
+          PGURI: postgres://postgres:postgres@localhost:5432/tegola
+          PGURI_NO_ACCESS: postgres://tegola_no_access:postgres@localhost:5432/tegola
           PGSSLMODE: disable
           PGSSLKEY: ""
           PGSSLCERT: ""

mvtprovider/postgis/README.md

@@ -1,32 +1,48 @@
 # PostGIS MVT Provider
 
-The PostGIS MVT provider manages querying for tile requests against a Postgres database (version 12+) with the [PostGIS](http://postgis.net/)(version 3.0+) extension installed and leverages [ST_AsMVT](https://postgis.net/docs/ST_AsMVT.html) to handle the MVT encoding at the database. 
+The PostGIS MVT provider manages querying for tile requests against a Postgres database (version 12+) with the [PostGIS](http://postgis.net/)(version 3.0+) extension installed and leverages [ST_AsMVT](https://postgis.net/docs/ST_AsMVT.html) to handle the MVT encoding at the database.
 
 The connection between tegola and PostGIS is configured in a `tegola.toml` file. An example minimum connection config:
 
-
 ```toml
 [[providers]]
 name = "test_postgis"       # provider name is referenced from map layers (required)
 type = "mvt_postgis"        # the type of data provider must be "mvt_postgis" for this data provider (required)
-host = "localhost"          # PostGIS database host (required)
-port = 5432                 # PostGIS database port (required)
-database = "tegola"         # PostGIS database name (required)
-user = "tegola"             # PostGIS database user (required)
-password = ""               # PostGIS database password (required)
+uri = "postgres://postgres:postgres@localhost:5432/tegola"          # PostGIS database uri (required)
+```
+
+## Connection Properties
+
+Establishing a connection via connection string (`uri`) will become the default
+connection method as of v0.16.0. Connecting via host/port/database is deprecated.
+
+-   `name` (string): [Required] provider name is referenced from map layers.
+-   `type` (string): [Required] the type of data provider. must be "postgis" to use this data provider
+-   `uri` (string): [Required] PostGIS database uri (postgres://user:password@host:port/database)
+-   `srid` (int): [Optional] The default SRID for the provider. Defaults to WebMercator (3857) but also supports WGS84 (4326)
+
+### Connection string properties
+
+#### Example
+
 ```
+# {protocol}://{user}:{password}@{host}:{port}/{database}?{options}=
+postgres://tegola:supersecret@localhost:5432/tegola?sslmode=prefer&pool_max_conns=10
+```
+
+#### Options
 
-### Connection Properties
+Tegola uses [pgx](https://github.com/jackc/pgx/blob/master/pgxpool/pool.go#L111) to manage
+PostgresSQL connections that allows the following configurations to be passed
+as parameters.
 
-- `name` (string): [Required] provider name is referenced from map layers.
-- `type` (string): [Required] the type of data provider. must be "postgis" to use this data provider
-- `host` (string): [Required] PostGIS database host
-- `port` (int): [Required] PostGIS database port (required)
-- `database` (string): [Required] PostGIS database name
-- `user` (string): [Required] PostGIS database user
-- `password` (string): [Required] PostGIS database password
-- `max_connections` (int): [Optional] The max connections to maintain in the connection pool. Defaults to 100. 0 means no max.
-- `srid` (int): [Optional] The default SRID for the provider. Defaults to WebMercator (3857) but also supports WGS84 (4326)
+-   `sslmode`: [Optional] PostGIS SSL mode. Default: "prefer"
+-   `pool_min_conns`: [Optional] The min connections to maintain in the connection pool. Defaults to 100. 0 means no max.
+-   `pool_max_conns`: [Optional] The max connections to maintain in the connection pool. Defaults to 100. 0 means no max.
+-   `pool_max_conn_idle_time`: [Optional] The maximum time an idle connection is kept alive. Defaults to "30m".
+-   `pool_max_connection_lifetime` [Optional] The maximum time a connection lives before it is terminated and recreated. Defaults to "1h".
+-   `pool_max_conn_lifetime_jitter` [Optional] Duration after `max_conn_lifetime` to randomly decide to close a connection.
+-   `pool_health_check_period` [Optional] Is the duration between checks of the health of idle connections. Defaults to 1m
 
 ## Provider Layers
 
@@ -37,41 +53,37 @@ In addition to the connection configuration above, Provider Layers need to be co
 name = "landuse"
 # MVT data provider must use SQL statements
 # this table uses "geom" for the geometry_fieldname and "gid" for the id_fieldname so they don't need to be configured
-# Wrapping the geom with ST_AsMVTGeom is required. 
+# Wrapping the geom with ST_AsMVTGeom is required.
 sql = "SELECT ST_AsMVTGeom(geom,!BBOX!) AS geom, gid FROM gis.landuse WHERE geom && !BBOX!"
 ```
 
 ### Provider Layers Properties
 
-- `name` (string): [Required] the name of the layer. This is used to reference this layer from map layers.
-- `geometry_fieldname` (string): [Optional] the name of the filed which contains the geometry for the feature. defaults to `geom`.
-- `id_fieldname` (string): [Optional] the name of the feature id field. defaults to `gid`.
-- `geometry_type` (string): [Optional] the layer geometry type. If not set, the table will be inspected at startup to try and infer the gemetry type. Valid values are: `Point`, `LineString`, `Polygon`, `MultiPoint`, `MultiLineString`, `MultiPolygon`, `GeometryCollection`.
-- `srid` (int): [Optional] the SRID of the layer. Supports `3857` (WebMercator) or `4326` (WGS84).
-- `sql` (string): [Required] custom SQL to use use. Supports the following tokens:
-  - `!BBOX!` - [Required] will be replaced with the bounding box of the tile before the query is sent to the database. `!bbox!` and`!BOX!` are supported as well for compatibilitiy with queries from Mapnik and MapServer styles.
-  - `!X!` - [Optional] will replaced with the "X" value of the requested tile.
-  - `!Y!` - [Optional] will replaced with the "Y" value of the requested tile.
-  - `!Z!` - [Optional] will replaced with the "Z" value of the requested tile.
-  - `!ZOOM!` - [Optional] will be replaced with the "Z" (zoom) value of the requested tile.
-  - `!SCALE_DENOMINATOR!` - [Optional] scale denominator, assuming 90.7 DPI (i.e. 0.28mm pixel size)
-  - `!PIXEL_WIDTH!` - [Optional] the pixel width in meters, assuming 256x256 tiles
-  - `!PIXEL_HEIGHT!` - [Optional] the pixel height in meters, assuming 256x256 tiles
-  - `!ID_FIELD!` - [Optional] the id field name
-  - `!GEOM_FIELD!` - [Optional] the geom field name
-  - `!GEOM_TYPE!` - [Optional] the geom type if defined otherwise ""
+-   `name` (string): [Required] the name of the layer. This is used to reference this layer from map layers.
+-   `geometry_fieldname` (string): [Optional] the name of the filed which contains the geometry for the feature. defaults to `geom`.
+-   `id_fieldname` (string): [Optional] the name of the feature id field. defaults to `gid`.
+-   `geometry_type` (string): [Optional] the layer geometry type. If not set, the table will be inspected at startup to try and infer the gemetry type. Valid values are: `Point`, `LineString`, `Polygon`, `MultiPoint`, `MultiLineString`, `MultiPolygon`, `GeometryCollection`.
+-   `srid` (int): [Optional] the SRID of the layer. Supports `3857` (WebMercator) or `4326` (WGS84).
+-   `sql` (string): [Required] custom SQL to use use. Supports the following tokens:
+    -   `!BBOX!` - [Required] will be replaced with the bounding box of the tile before the query is sent to the database. `!bbox!` and`!BOX!` are supported as well for compatibilitiy with queries from Mapnik and MapServer styles.
+    -   `!X!` - [Optional] will replaced with the "X" value of the requested tile.
+    -   `!Y!` - [Optional] will replaced with the "Y" value of the requested tile.
+    -   `!Z!` - [Optional] will replaced with the "Z" value of the requested tile.
+    -   `!ZOOM!` - [Optional] will be replaced with the "Z" (zoom) value of the requested tile.
+    -   `!SCALE_DENOMINATOR!` - [Optional] scale denominator, assuming 90.7 DPI (i.e. 0.28mm pixel size)
+    -   `!PIXEL_WIDTH!` - [Optional] the pixel width in meters, assuming 256x256 tiles
+    -   `!PIXEL_HEIGHT!` - [Optional] the pixel height in meters, assuming 256x256 tiles
+    -   `!ID_FIELD!` - [Optional] the id field name
+    -   `!GEOM_FIELD!` - [Optional] the geom field name
+    -   `!GEOM_TYPE!` - [Optional] the geom type if defined otherwise ""
 
 ## Example mvt_postgis and map config
 
 ```toml
 [[providers]]
 name = "test_postgis"
 type = "mvt_postgis"
-host = "localhost"
-port = 5432
-database = "tegola"
-user = "tegola"
-password = ""
+uri = "postgres://postgres:postgres@localhost:5432/tegola"
 
   [[providers.layers]]
   name = "landuse"
@@ -96,11 +108,7 @@ When using a 4326 projection with ST_AsMVT the SQL statement needs to be modifie
 [[providers]]
 name = "test_postgis"
 type = "mvt_postgis"
-host = "localhost"
-port = 5432
-database = "tegola"
-user = "tegola"
-password = ""
+uri = "postgres://postgres:postgres@localhost:5432/tegola"
 srid = 4326 # setting the srid on the provider to 4326 will cause the !BBOX! value to use the 4326 projection.
 
   [[providers.layers]]
@@ -127,11 +135,8 @@ Testing is designed to work against a live PostGIS database. To see how to set u
 
 ```bash
 $ export RUN_POSTGIS_TESTS=yes
-$ export PGHOST="localhost"
-$ export PGPORT=5432
-$ export PGDATABASE="tegola"
-$ export PGUSER="postgres"
-$ export PGUSER_NO_ACCESS="tegola_no_access" # used for testing errors when user does not have read permissions on a table
+$ export PGURI="postgres://postgres:postgres@localhost:5432/tegola"
+$ export PGURI_NO_ACCESS="postgres://tegola_no_access:postgres@localhost:5432/tegola" # used for testing errors when user does not have read permissions on a table
 $ export PGPASSWORD=""
 $ export PGSSLMODE="disable"
 ```

provider/postgis/README.md

@@ -28,22 +28,12 @@ application_name = "tegola"
 # The default is OFF (read/write).
 # (optional)
 default_transaction_read_only = "off"
-
-host = "localhost"                # PostGIS database host (deprecated)
-port = 5432                       # PostGIS database port (deprecated)
-database = "tegola"               # PostGIS database name (deprecated)
-user = "tegola"                   # PostGIS database user (deprecated)
-password = "supersecret"          # PostGIS database password (deprecated)
-max_connections = 10              # PostGIS max connections (deprecated)
-max_connection_idle_time = "30m"  # PostGIS max connection idle time (deprecated)
-max_connection_lifetime = "1h"    # PostGIS max connection life time (deprecated)
 ```
 
 ## Connection Properties
 
 Establishing a connection via connection string (`uri`) will become the default
-connection method as of v0.16.0. Connecting via host/port/database is flagged
-for deprecation as of v0.15.0 but will be possible until v0.16.0 still.
+connection method as of v0.16.0. Connecting via host/port/database is deprecated.
 
 -   `uri` (string): [Required] PostGIS connection string
 -   `name` (string): [Required] provider name is referenced from map layers
@@ -61,26 +51,17 @@ postgres://tegola:supersecret@localhost:5432/tegola?sslmode=prefer&pool_max_conn
 
 #### Options
 
+Tegola uses [pgx](https://github.com/jackc/pgx/blob/master/pgxpool/pool.go#L111) to manage
+PostgresSQL connections that allows the following configurations to be passed
+as parameters.
+
 -   `sslmode`: [Optional] PostGIS SSL mode. Default: "prefer"
+-   `pool_min_conns`: [Optional] The min connections to maintain in the connection pool. Defaults to 100. 0 means no max.
 -   `pool_max_conns`: [Optional] The max connections to maintain in the connection pool. Defaults to 100. 0 means no max.
 -   `pool_max_conn_idle_time`: [Optional] The maximum time an idle connection is kept alive. Defaults to "30m".
--   `max_connection_lifetime` [Optional] The maximum time a connection lives before it is terminated and recreated. Defaults to "1h".
-
-### [DEPRECATED] Connection Properties
-
--   `uri` (string): [Required] PostGIS connection string
--   `name` (string): [Required] provider name is referenced from map layers
--   `type` (string): [Required] the type of data provider. must be "postgis" to use this data provider
--   `host` (string): [deprecated] PostGIS database host
--   `port` (int): [deprecated] PostGIS database port (required)
--   `database` (string): [deprecated] PostGIS database name
--   `user` (string): [deprecated] PostGIS database user
--   `password` (string): [deprecated] PostGIS database password
--   `srid` (int): [Optional] The default SRID for the provider. Defaults to WebMercator (3857) but also supports WGS84 (4326)
--   `ssl_mode`: (string): [Optional]. PostGIS SSL mode. Default is "prefer".
--   `max_connections` (int): [deprecated] The max connections to maintain in the connection pool. Defaults to 100. 0 means no max.
--   `max_connection_idle_time` (duration string): [deprecated] The maximum time an idle connection is kept alive.
--   `max_connection_lifetime` (duration string): [deprecated] The maximum time a connection lives before it is terminated and recreated.
+-   `pool_max_connection_lifetime` [Optional] The maximum time a connection lives before it is terminated and recreated. Defaults to "1h".
+-   `pool_max_conn_lifetime_jitter` [Optional] Duration after `max_conn_lifetime` to randomly decide to close a connection.
+-   `pool_health_check_period` [Optional] Is the duration between checks of the health of idle connections. Defaults to 1m
 
 ## Provider Layers
 
@@ -151,11 +132,8 @@ To run the PostGIS tests, the following environment variables need to be set:
 
 ```bash
 $ export RUN_POSTGIS_TESTS=yes
-$ export PGHOST="localhost"
-$ export PGPORT=5432
-$ export PGDATABASE="tegola"
-$ export PGUSER="postgres"
-$ export PGUSER_NO_ACCESS="tegola_no_access" # used for testing errors when user does not have read permissions on a table
+$ export PGURI="postgres://postgres:postgres@localhost:5432/tegola"
+$ export PGURI_NO_ACCESS="postgres://tegola_no_access:@localhost:5432/tegola" # used for testing errors when user does not have read permissions on a table
 $ export PGPASSWORD=""
 $ export PGSSLMODE="disable"
 ```

provider/postgis/postgis.go

@@ -175,12 +175,7 @@ const (
 )
 
 const (
-	DefaultURI                        = ""
-	DefaultPort                       = 5432
 	DefaultSRID                       = tegola.WebMercator
-	DefaultMaxConn                    = 100
-	DefaultMaxConnIdleTime            = "30m"
-	DefaultMaxConnLifetime            = "1h"
 	DefaultSSLMode                    = "prefer"
 	DefaultSSLKey                     = ""
 	DefaultSSLCert                    = ""
@@ -191,18 +186,10 @@ const (
 const (
 	ConfigKeyName                       = "name"
 	ConfigKeyURI                        = "uri"
-	ConfigKeyHost                       = "host"
-	ConfigKeyPort                       = "port"
-	ConfigKeyDB                         = "database"
-	ConfigKeyUser                       = "user"
-	ConfigKeyPassword                   = "password"
 	ConfigKeySSLMode                    = "ssl_mode"
 	ConfigKeySSLKey                     = "ssl_key"
 	ConfigKeySSLCert                    = "ssl_cert"
 	ConfigKeySSLRootCert                = "ssl_root_cert"
-	ConfigKeyMaxConn                    = "max_connections"
-	ConfigKeyMaxConnIdleTime            = "max_connection_idle_time"
-	ConfigKeyMaxConnLifetime            = "max_connection_lifetime"
 	ConfigKeySRID                       = "srid"
 	ConfigKeyLayers                     = "layers"
 	ConfigKeyLayerName                  = "name"
@@ -262,7 +249,6 @@ func validateURI(u string) error {
 	return nil
 }
 
-// TODO: (iwpnd) to be removed/refactored in v0.17.0
 // BuildURI creates a database URI from config
 func BuildURI(config dict.Dicter) (*url.URL, *url.Values, error) {
 
@@ -272,97 +258,33 @@ func BuildURI(config dict.Dicter) (*url.URL, *url.Values, error) {
 		return nil, nil, err
 	}
 
-	uri := DefaultURI
-	uri, err = config.String(ConfigKeyURI, &uri)
+	uri, err := config.String(ConfigKeyURI, nil)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	// if uri is set in the config, we add sslmode and return early
-	if uri != "" {
-		if err := validateURI(uri); err != nil {
-			return nil, nil, err
-		}
-
-		parsedUri, err := url.Parse(uri)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		// parse query to make sure sslmode is attached
-		parsedQuery, err := url.ParseQuery(parsedUri.RawQuery)
-		if err != nil {
-			return &url.URL{}, nil, err
-		}
-
-		if ok := parsedQuery.Get("sslmode"); ok == "" {
-			parsedQuery.Add("sslmode", sslmode)
-		}
-
-		parsedUri.RawQuery = parsedQuery.Encode()
-
-		return parsedUri, &parsedQuery, nil
-	}
-
-	host, err := config.String(ConfigKeyHost, nil)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	port := DefaultPort
-	if port, err = config.Int(ConfigKeyPort, &port); err != nil {
+	if err := validateURI(uri); err != nil {
 		return nil, nil, err
 	}
 
-	db, err := config.String(ConfigKeyDB, nil)
+	parsedUri, err := url.Parse(uri)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	user, err := config.String(ConfigKeyUser, nil)
+	// parse query to make sure sslmode is attached
+	parsedQuery, err := url.ParseQuery(parsedUri.RawQuery)
 	if err != nil {
-		return nil, nil, err
-	}
-
-	password, err := config.String(ConfigKeyPassword, nil)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	maxcon := DefaultMaxConn
-	if maxcon, err = config.Int(ConfigKeyMaxConn, &maxcon); err != nil {
-		return nil, nil, err
+		return &url.URL{}, nil, err
 	}
 
-	idletime := DefaultMaxConnIdleTime
-	if idletime, err = config.String(ConfigKeyMaxConnIdleTime, &idletime); err != nil {
-		return nil, nil, err
-	}
-
-	lifetime := DefaultMaxConnLifetime
-	if lifetime, err = config.String(ConfigKeyMaxConnLifetime, &lifetime); err != nil {
-		return nil, nil, err
+	if ok := parsedQuery.Get("sslmode"); ok == "" {
+		parsedQuery.Add("sslmode", sslmode)
 	}
 
-	params := &url.Values{}
-	params.Add("sslmode", sslmode)
-	params.Add("pool_max_conns", fmt.Sprintf("%v", maxcon))
-	params.Add("pool_max_conn_lifetime", lifetime)
-	params.Add("pool_max_conn_idle_time", idletime)
-
-	if uri == "" {
-		log.Warn("Connecting to PostGIS with connection parameters is deprecated. Use 'uri' instead.")
-	}
-
-	u := &url.URL{
-		Scheme:   "postgres",
-		Host:     fmt.Sprintf("%v:%v", host, port),
-		User:     url.UserPassword(user, password),
-		Path:     db,
-		RawQuery: params.Encode(),
-	}
+	parsedUri.RawQuery = parsedQuery.Encode()
 
-	return u, params, nil
+	return parsedUri, &parsedQuery, nil
 }
 
 type DBConfigOptions struct {