Merge branch 'master' into param-builder-optional

Author: rekby

.github/workflows/lint.yml

@@ -68,7 +68,7 @@ jobs:
           go-version: "1.23.x"
       - name: Install utilities
         run: |
-          go install mvdan.cc/gofumpt@v0.3.1
+          go install mvdan.cc/gofumpt@v0.7.0
           go install github.com/daixiang0/[email protected]
       - name: format all files with auto-formatter
         run: bash ./.github/scripts/format-all-go-code.sh "$PWD"

.github/workflows/slo.yml

@@ -73,12 +73,6 @@ jobs:
           workload_build_context4: ../..
           workload_build_options4: -f Dockerfile --build-arg SRC_PATH=native/query --build-arg JOB_NAME=workload-native-query
 
-          language_id5: 'native-query-with-pool'
-          workload_path5: 'tests/slo'
-          language5: 'Native ydb-go-sdk/v3 over query-service with session pool'
-          workload_build_context5: ../..
-          workload_build_options5: -f Dockerfile --build-arg SRC_PATH=native/query/with/pool --build-arg JOB_NAME=workload-native-query-with-pool
-
       - uses: actions/upload-artifact@v4
         if: always()
         with:

.github/workflows/tests.yml

@@ -66,6 +66,7 @@ jobs:
           YDB_LOCAL_SURVIVE_RESTART: true
           YDB_USE_IN_MEMORY_PDISKS: true
           YDB_TABLE_ENABLE_PREPARED_DDL: true
+          YDB_ENABLE_COLUMN_TABLES: true
         options: '-h localhost'
     env:
       OS: ubuntu-latest
@@ -119,6 +120,7 @@ jobs:
           YDB_USE_IN_MEMORY_PDISKS: true
           YDB_TABLE_ENABLE_PREPARED_DDL: true
           YDB_FEATURE_FLAGS: enable_topic_service_tx
+          YDB_ENABLE_COLUMN_TABLES: true
         options: '-h localhost'
     env:
       OS: ubuntu-latest

CHANGELOG.md

@@ -1,3 +1,46 @@
+## v3.80.8
+* Added `ydb.WithLazyTx(bool)` option for create lazy transactions on `query.Session.Begin` call
+* Added initial experimental topic and cdc-helpers, see examples in [tests/integration/topic_helpers_test.go](https://github.com/ydb-platform/ydb-go-sdk/blob/master/tests/integration/topic_helpers_test.go)
+* Added experimental `sugar.UnmarshalRows` for user unmarshaller structs in own code in go 1.23, change example for use the iterator.
+* Added `ydb_go_sdk_ydb_query_pool_size_index` metrics
+
+## v3.80.7
+* Fixed bug with doesn't rollback the transaction on the operation error in table service
+
+## v3.80.6
+* Fixed concurrent map writes in metrics
+* Renamed method at experimental API `reader.PopBatchTx` to `reader.PopMessagesBatchTx`
+
+## v3.80.5
+* Fixed connections pool leak on failed `ydb.Open` call
+
+## v3.80.4
+* Fixed panic on usage metrics package from prometheus adapter on `trace.Driver.OnNewStream` callback
+
+## v3.80.3
+* Added option `ydb.WithSessionPoolSessionIdleTimeToLive` for restrict idle time of query sessions
+* Fixed bug with leak of query transactions
+* Changed `ydb_go_sdk_ydb_driver_conn_requests` metrics splitted to `ydb_go_sdk_ydb_driver_conn_request_statuses` and `ydb_go_sdk_ydb_driver_conn_request_methods`
+* Fixed metadata for operation service connection
+* Fixed composing query traces in call `db.Query.Do[Tx]` using option `query.WithTrace`
+
+## v3.80.2
+* Added `balancers.PreferNearestDC[WithFallback]` balancers
+* Marked as deprecated `balancers.PreferLocalDC[WithFallback]` balancers because `local` word is ambiguous for balancer idea
+
+## v3.80.1
+* Added `lastErr` from previous attempt in `retry.RetryWithResult`
+
+## v3.80.0
+* Replaced internal table client pool entities to `internal/pool`
+
+## v3.79.2
+* Enabled by default usage of `internal/pool` in `internal/query.Client`
+
+## v3.79.1
+* Changed `trace.Table` and `trace.Query` traces
+* Implemented `internal/pool` the same as table client pool from `internal/table.Client`
+
 ## v3.79.0
 * Added commit messages for topic listener
 * EOF error in RecvMsg is no longer logged

README.md

@@ -42,37 +42,7 @@ if err != nil {
     log.Fatal(err)
 }
 ```
-* execute `SELECT` query over `Table` service client
- ```go
-// Do retry operation on errors with best effort
-err := db.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) {
-    _, res, err := s.Execute(ctx, table.DefaultTxControl(), 
-		`SELECT 42 as id, "myStr" as myStr;`, 
-		nil, // empty parameters
-	)
-    if err != nil {
-        return err
-    }
-    defer res.Close()
-    if err = res.NextResultSetErr(ctx); err != nil {
-        return err
-    }
-    for res.NextRow() {
-        var id    int32
-        var myStr string
-        err = res.ScanNamed(named.Required("id", &id),named.OptionalWithDefault("myStr", &myStr))
-        if err != nil {
-            log.Fatal(err)
-        }
-        log.Printf("id=%v, myStr='%s'\n", id, myStr)
-    }
-    return res.Err() // for driver retry if not nil
-})
-if err != nil {
-    log.Fatal(err)
-}
-```
-* execute `SELECT` query over `Query` service client
+* execute `SELECT` query with `Query` service client
  ```go
 // Do retry operation on errors with best effort
 err := db.Query().Do( // Do retry operation on errors with best effort
@@ -149,6 +119,7 @@ Driver implements several ways for making credentials for `YDB`:
 - `ydb.WithAnonymousCredentials()` (enabled by default unless otherwise specified)
 - `ydb.WithAccessTokenCredentials("token")`
 - `ydb.WithStaticCredentials("user", "password")`, 
+- `ydb.WithOauth2TokenExchangeCredentials()` and `ydb,WithOauth2TokenExchangeCredentialsFile(configFilePath)`
 - as part of connection string, like as `grpcs://user:password@endpoint/database`
 
 Another variants of `credentials.Credentials` object provides with external packages:

balancers/balancers.go

@@ -37,21 +37,41 @@ func (filterLocalDC) String() string {
 	return "LocalDC"
 }
 
-// PreferLocalDC creates balancer which use endpoints only in location such as initial endpoint location
-// Balancer "balancer" defines balancing algorithm between endpoints selected with filter by location
-// PreferLocalDC balancer try to autodetect local DC from client side.
+// Deprecated: use PreferNearestDC instead
+// Will be removed after March 2025.
+// Read about versioning policy: https://github.com/ydb-platform/ydb-go-sdk/blob/master/VERSIONING.md#deprecated
 func PreferLocalDC(balancer *balancerConfig.Config) *balancerConfig.Config {
 	balancer.Filter = filterLocalDC{}
-	balancer.DetectLocalDC = true
+	balancer.DetectNearestDC = true
 
 	return balancer
 }
 
-// PreferLocalDCWithFallBack creates balancer which use endpoints only in location such as initial endpoint location
+// PreferNearestDC creates balancer which use endpoints only in location such as initial endpoint location
 // Balancer "balancer" defines balancing algorithm between endpoints selected with filter by location
-// If filter returned zero endpoints from all discovery endpoints list - used all endpoint instead
+// PreferNearestDC balancer try to autodetect local DC from client side.
+func PreferNearestDC(balancer *balancerConfig.Config) *balancerConfig.Config {
+	balancer.Filter = filterLocalDC{}
+	balancer.DetectNearestDC = true
+
+	return balancer
+}
+
+// Deprecated: use PreferNearestDCWithFallBack instead
+// Will be removed after March 2025.
+// Read about versioning policy: https://github.com/ydb-platform/ydb-go-sdk/blob/master/VERSIONING.md#deprecated
 func PreferLocalDCWithFallBack(balancer *balancerConfig.Config) *balancerConfig.Config {
-	balancer = PreferLocalDC(balancer)
+	balancer = PreferNearestDC(balancer)
+	balancer.AllowFallback = true
+
+	return balancer
+}
+
+// PreferNearestDCWithFallBack creates balancer which use endpoints only in location such as initial endpoint location
+// Balancer "balancer" defines balancing algorithm between endpoints selected with filter by location
+// If filter returned zero endpoints from all discovery endpoints list - used all endpoint instead
+func PreferNearestDCWithFallBack(balancer *balancerConfig.Config) *balancerConfig.Config {
+	balancer = PreferNearestDC(balancer)
 	balancer.AllowFallback = true
 
 	return balancer

balancers/balancers_test.go

@@ -17,7 +17,7 @@ func TestPreferLocalDC(t *testing.T) {
 		&mock.Conn{AddrField: "2", State: conn.Online, LocationField: "2"},
 		&mock.Conn{AddrField: "3", State: conn.Online, LocationField: "2"},
 	}
-	rr := PreferLocalDC(RandomChoice())
+	rr := PreferNearestDC(RandomChoice())
 	require.False(t, rr.AllowFallback)
 	require.Equal(t, []conn.Conn{conns[1], conns[2]}, applyPreferFilter(balancerConfig.Info{SelfLocation: "2"}, rr, conns))
 }
@@ -28,7 +28,7 @@ func TestPreferLocalDCWithFallBack(t *testing.T) {
 		&mock.Conn{AddrField: "2", State: conn.Online, LocationField: "2"},
 		&mock.Conn{AddrField: "3", State: conn.Online, LocationField: "2"},
 	}
-	rr := PreferLocalDCWithFallBack(RandomChoice())
+	rr := PreferNearestDCWithFallBack(RandomChoice())
 	require.True(t, rr.AllowFallback)
 	require.Equal(t, []conn.Conn{conns[1], conns[2]}, applyPreferFilter(balancerConfig.Info{SelfLocation: "2"}, rr, conns))
 }

balancers/config.go

@@ -20,8 +20,13 @@ const (
 type preferType string
 
 const (
-	preferTypeLocalDC   = preferType("local_dc")
+	preferTypeNearestDC = preferType("nearest_dc")
 	preferTypeLocations = preferType("locations")
+
+	// Deprecated
+	// Will be removed after March 2025.
+	// Read about versioning policy: https://github.com/ydb-platform/ydb-go-sdk/blob/master/VERSIONING.md#deprecated
+	preferTypeLocalDC = preferType("local_dc")
 )
 
 type balancersConfig struct {
@@ -90,10 +95,16 @@ func CreateFromConfig(s string) (*balancerConfig.Config, error) {
 	switch c.Prefer {
 	case preferTypeLocalDC:
 		if c.Fallback {
-			return PreferLocalDCWithFallBack(b), nil
+			return PreferNearestDCWithFallBack(b), nil
+		}
+
+		return PreferNearestDC(b), nil
+	case preferTypeNearestDC:
+		if c.Fallback {
+			return PreferNearestDCWithFallBack(b), nil
 		}
 
-		return PreferLocalDC(b), nil
+		return PreferNearestDC(b), nil
 	case preferTypeLocations:
 		if len(c.Locations) == 0 {
 			return nil, xerrors.WithStackTrace(fmt.Errorf("empty locations list in balancer '%s' config", c.Type))

balancers/config_test.go

@@ -70,7 +70,21 @@ func TestFromConfig(t *testing.T) {
 				"prefer": "local_dc"
 			}`,
 			res: balancerConfig.Config{
-				DetectLocalDC: true,
+				DetectNearestDC: true,
+				Filter: filterFunc(func(info balancerConfig.Info, e endpoint.Info) bool {
+					// some non nil func
+					return false
+				}),
+			},
+		},
+		{
+			name: "prefer_nearest_dc",
+			config: `{
+				"type": "random_choice",
+				"prefer": "nearest_dc"
+			}`,
+			res: balancerConfig.Config{
+				DetectNearestDC: true,
 				Filter: filterFunc(func(info balancerConfig.Info, e endpoint.Info) bool {
 					// some non nil func
 					return false
@@ -93,8 +107,24 @@ func TestFromConfig(t *testing.T) {
 				"fallback": true
 			}`,
 			res: balancerConfig.Config{
-				AllowFallback: true,
-				DetectLocalDC: true,
+				AllowFallback:   true,
+				DetectNearestDC: true,
+				Filter: filterFunc(func(info balancerConfig.Info, e endpoint.Info) bool {
+					// some non nil func
+					return false
+				}),
+			},
+		},
+		{
+			name: "prefer_nearest_dc_with_fallback",
+			config: `{
+				"type": "random_choice",
+				"prefer": "nearest_dc",
+				"fallback": true
+			}`,
+			res: balancerConfig.Config{
+				AllowFallback:   true,
+				DetectNearestDC: true,
 				Filter: filterFunc(func(info balancerConfig.Info, e endpoint.Info) bool {
 					// some non nil func
 					return false

driver.go

@@ -274,6 +274,8 @@ func Open(ctx context.Context, dsn string, opts ...Option) (_ *Driver, _ error)
 	}()
 
 	if err = d.connect(ctx); err != nil {
+		_ = d.pool.Release(ctx)
+
 		return nil, xerrors.WithStackTrace(err)
 	}
 
@@ -519,7 +521,7 @@ func (d *Driver) connect(ctx context.Context) (err error) {
 
 	d.operation = xsync.OnceValue(func() (*operation.Client, error) {
 		return operation.New(xcontext.ValueOnly(ctx),
-			d.pool.Get(endpoint.New(d.config.Endpoint())),
+			d.balancer,
 		), nil
 	})