[#29115] YSQL: Add PG backend type for auto analyze

yifanguan · yifanguan · commit 25f4d62fd514 · 2025-11-05T18:29:50.000Z
Summary: Auto Analyze service periodically starts a local PG connection and runs ANALYZEs in the background. Concurrent CREATE INDEX can time out waiting for the postgres backend used by auto analyze to catch up the latest catalog version because of long-running ANALYZE command. On the happy path, concurrent DDL can abort auto-ANALYZE. However, it was discovered that a ANALYZE request with a large size of ybctids can exceed the read buffer memory limit causing ANALYZE execution to hang. Since ANALYZE doesn't write to user tables and only writes to system catalogs (pg_class, pg_statistic, etc), it is safe to let CREATE INDEX ignore the backend used by auto-ANALYZE. To achieve this goal, this diff adds one PG backend type `YB_AUTO_ANALYZE_BACKEND` specific for auto analyze to use. A libpq option: `yb_auto_analyze` is added accordingly. This option can only be used by tserver to postgres libpq connection with yb-tserver-key authentication. Jira: DB-18884 Test Plan: ./yb_build.sh --cxx-test pgwrapper_pg_auto_analyze-test --gtest_filter PgConcurrentCreateIndexTest.ConcurrentCreateIndex Reviewers: sanketh, jason, smishra, pjain Reviewed By: jason Subscribers: ybase, svc_phabricator, jason, yql Differential Revision: https://phorge.dev.yugabyte.com/D47906
diff --git a/src/postgres/contrib/postgres_fdw/expected/yb.port.postgres_fdw.out b/src/postgres/contrib/postgres_fdw/expected/yb.port.postgres_fdw.out
@@ -9906,7 +9906,7 @@ DO $d$
     END;
 $d$;
 ERROR:  invalid option "password"
-HINT:  Valid options in this context are: service, passfile, channel_binding, connect_timeout, dbname, host, hostaddr, port, options, application_name, keepalives, keepalives_idle, keepalives_interval, keepalives_count, tcp_user_timeout, sslmode, sslcompression, sslcert, sslkey, sslrootcert, sslcrl, sslcrldir, sslsni, requirepeer, ssl_min_protocol_version, ssl_max_protocol_version, gssencmode, krbsrvname, gsslib, target_session_attrs, use_remote_estimate, fdw_startup_cost, fdw_tuple_cost, extensions, updatable, truncatable, fetch_size, batch_size, async_capable, parallel_commit, keep_connections, server_type
+HINT:  Valid options in this context are: service, passfile, channel_binding, connect_timeout, dbname, host, hostaddr, port, options, application_name, keepalives, keepalives_idle, keepalives_interval, keepalives_count, tcp_user_timeout, sslmode, sslcompression, sslcert, sslkey, sslrootcert, sslcrl, sslcrldir, sslsni, requirepeer, ssl_min_protocol_version, ssl_max_protocol_version, gssencmode, krbsrvname, gsslib, target_session_attrs, yb_auto_analyze, use_remote_estimate, fdw_startup_cost, fdw_tuple_cost, extensions, updatable, truncatable, fetch_size, batch_size, async_capable, parallel_commit, keep_connections, server_type
 CONTEXT:  SQL statement "ALTER SERVER loopback_nopw OPTIONS (ADD password 'dummypw')"
 PL/pgSQL function inline_code_block line 3 at EXECUTE
 -- If we add a password for our user mapping instead, we should get a different
diff --git a/src/postgres/src/backend/commands/indexcmds.c b/src/postgres/src/backend/commands/indexcmds.c
@@ -5151,6 +5151,7 @@ YbWaitForBackendsCatalogVersion()
 								 " pg_stat_activity WHERE"
 								 " backend_type != 'walsender' AND"
 								 " backend_type != 'yb-conn-mgr walsender' AND"
+								 " backend_type != 'yb auto analyze backend' AND"
 								 " catalog_version < %" PRIu64
 								 " AND datid = %u;",
 								 catalog_version,
diff --git a/src/postgres/src/backend/libpq/auth.c b/src/postgres/src/backend/libpq/auth.c
@@ -460,6 +460,18 @@ ClientAuthentication(Port *port)
 
 	CHECK_FOR_INTERRUPTS();
 
+	/*
+	 * Only tserver-owned backends using yb-tserver-key authentication are
+	 * allowed to run as yb_auto_analyze.
+	 */
+	if (IsYugaByteEnabled() && MyBackendType == YB_AUTO_ANALYZE_BACKEND &&
+		port->hba->auth_method != uaYbTserverKey &&
+		!YBCGetGFlags()->TEST_ysql_bypass_auto_analyze_auth_check)
+		ereport(FATAL,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("yb_auto_analyze can only be set if the authentication method "
+						"is yb-tserver-key")));
+
 	/*
 	 * This is the first point where we have access to the hba record for the
 	 * current connection, so perform any verifications based on the hba
diff --git a/src/postgres/src/backend/postmaster/postmaster.c b/src/postgres/src/backend/postmaster/postmaster.c
@@ -2129,6 +2129,7 @@ ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done)
 	char	   *yb_auth_backend_remote_host = NULL;
 	char		yb_logical_conn_type = 'U'; /* Unencrypted */
 	bool		yb_logical_conn_type_provided = false;
+	bool		yb_auto_analyze_backend = false;
 
 	pq_startmsgread();
 
@@ -2426,6 +2427,17 @@ ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done)
 				yb_logical_conn_type = *pstrdup(valptr);
 				yb_logical_conn_type_provided = true;
 			}
+			else if (YBIsEnabledInPostgresEnvVar()
+					 && strcmp(nameptr, "yb_auto_analyze") == 0)
+			{
+				if (!parse_bool(valptr, &yb_auto_analyze_backend))
+					ereport(FATAL,
+							(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+							 errmsg("invalid value for parameter \"%s\": \"%s\"",
+									"yb_auto_analyze",
+									valptr),
+							 errhint("Valid values are: \"false\", 0, \"true\", 1.")));
+			}
 			else if (strncmp(nameptr, "_pq_.", 5) == 0)
 			{
 				/*
@@ -2559,6 +2571,8 @@ ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done)
 
 	if (am_walsender)
 		MyBackendType = B_WAL_SENDER;
+	else if (yb_auto_analyze_backend)
+		MyBackendType = YB_AUTO_ANALYZE_BACKEND;
 	else
 		MyBackendType = B_BACKEND;
 
diff --git a/src/postgres/src/backend/utils/activity/backend_status.c b/src/postgres/src/backend/utils/activity/backend_status.c
@@ -377,6 +377,7 @@ pgstat_bestart(void)
 	/* YB: Increment the total connections counter */
 	if (lbeentry.st_procpid > 0 &&
 		(lbeentry.st_backendType == B_BACKEND ||
+		 lbeentry.st_backendType == YB_AUTO_ANALYZE_BACKEND ||
 		 lbeentry.st_backendType == YB_YSQL_CONN_MGR))
 		(*yb_new_conn)++;
 
@@ -399,6 +400,7 @@ pgstat_bestart(void)
 	if (lbeentry.st_backendType == B_BACKEND
 		|| lbeentry.st_backendType == B_WAL_SENDER
 		|| lbeentry.st_backendType == B_BG_WORKER
+		|| lbeentry.st_backendType == YB_AUTO_ANALYZE_BACKEND
 		|| lbeentry.st_backendType == YB_YSQL_CONN_MGR
 		|| lbeentry.st_backendType == YB_YSQL_CONN_MGR_WAL_SENDER)
 		lbeentry.st_userid = GetSessionUserId();
diff --git a/src/postgres/src/backend/utils/adt/pgstatfuncs.c b/src/postgres/src/backend/utils/adt/pgstatfuncs.c
@@ -805,6 +805,7 @@ pg_stat_get_activity(PG_FUNCTION_ARGS)
 			proc = BackendPidGetProc(beentry->st_procpid);
 
 			if (proc == NULL && (beentry->st_backendType != B_BACKEND &&
+								 beentry->st_backendType != YB_AUTO_ANALYZE_BACKEND &&
 								 beentry->st_backendType != YB_YSQL_CONN_MGR))
 			{
 				/*
diff --git a/src/postgres/src/backend/utils/init/miscinit.c b/src/postgres/src/backend/utils/init/miscinit.c
@@ -301,6 +301,9 @@ GetBackendTypeDesc(BackendType backendType)
 			break;
 		case YB_YSQL_CONN_MGR_WAL_SENDER:
 			backendDesc = "yb-conn-mgr walsender";
+			break;
+		case YB_AUTO_ANALYZE_BACKEND:
+			backendDesc = "yb auto analyze backend";
 	}
 
 	return backendDesc;
diff --git a/src/postgres/src/include/miscadmin.h b/src/postgres/src/include/miscadmin.h
@@ -361,6 +361,7 @@ typedef enum BackendType
 	B_LOGGER,
 	YB_YSQL_CONN_MGR,
 	YB_YSQL_CONN_MGR_WAL_SENDER,
+	YB_AUTO_ANALYZE_BACKEND,
 } BackendType;
 
 extern PGDLLIMPORT BackendType MyBackendType;
diff --git a/src/postgres/src/interfaces/libpq/fe-connect.c b/src/postgres/src/interfaces/libpq/fe-connect.c
@@ -345,6 +345,10 @@ static const internalPQconninfoOption PQconninfoOptions[] = {
 		"Target-Session-Attrs", "", 15, /* sizeof("prefer-standby") = 15 */
 	offsetof(struct pg_conn, target_session_attrs)},
 
+	{"yb_auto_analyze", NULL, NULL, NULL,
+		"YB-Auto-Analyze", "", 6, /* sizeof("false") = 6 */
+	offsetof(struct pg_conn, yb_auto_analyze)},
+
 	/* Terminating entry --- MUST BE LAST */
 	{NULL, NULL, NULL, NULL,
 	NULL, NULL, 0}
@@ -4181,6 +4185,8 @@ freePGconn(PGconn *conn)
 		free(conn->rowBuf);
 	if (conn->target_session_attrs)
 		free(conn->target_session_attrs);
+	if (conn->yb_auto_analyze)
+		free(conn->yb_auto_analyze);
 	termPQExpBuffer(&conn->errorMessage);
 	termPQExpBuffer(&conn->workBuffer);
 
diff --git a/src/postgres/src/interfaces/libpq/fe-protocol3.c b/src/postgres/src/interfaces/libpq/fe-protocol3.c
@@ -2236,6 +2236,8 @@ build_startup_packet(const PGconn *conn, char *packet,
 		ADD_STARTUP_OPTION("replication", conn->replication);
 	if (conn->pgoptions && conn->pgoptions[0])
 		ADD_STARTUP_OPTION("options", conn->pgoptions);
+	if (conn->yb_auto_analyze && conn->yb_auto_analyze[0])
+		ADD_STARTUP_OPTION("yb_auto_analyze", conn->yb_auto_analyze);
 	if (conn->send_appname)
 	{
 		/* Use appname if present, otherwise use fallback */
diff --git a/src/postgres/src/interfaces/libpq/libpq-int.h b/src/postgres/src/interfaces/libpq/libpq-int.h
@@ -396,6 +396,8 @@ struct pg_conn
 	char	   *ssl_max_protocol_version;	/* maximum TLS protocol version */
 	char	   *target_session_attrs;	/* desired session properties */
 
+	char	   *yb_auto_analyze; /* use the connection for auto analyze? */
+
 	/* Optional file to write trace info to */
 	FILE	   *Pfdebug;
 	int			traceFlags;
diff --git a/src/postgres/yb-extensions/yb_pg_metrics/yb_pg_metrics.c b/src/postgres/yb-extensions/yb_pg_metrics/yb_pg_metrics.c
@@ -589,7 +589,8 @@ pullRpczEntries(void)
 			rpcz[i].leader_pid = -1;
 			PGPROC	   *proc = NULL;
 
-			if (beentry->st_backendType == B_BACKEND)
+			if (beentry->st_backendType == B_BACKEND ||
+				beentry->st_backendType == YB_AUTO_ANALYZE_BACKEND)
 				proc = BackendPidGetProc(rpcz[i].proc_id);
 			else if (beentry->st_backendType != YB_YSQL_CONN_MGR)
 			{
diff --git a/src/yb/tserver/tablet_server.cc b/src/yb/tserver/tablet_server.cc
@@ -711,7 +711,7 @@ Status TabletServer::RegisterServices() {
                                 const CoarseTimePoint& deadline) {
       return pgwrapper::CreateInternalPGConnBuilder(pgsql_proxy_bind_address(), database_name,
                                                     GetSharedMemoryPostgresAuthKey(),
-                                                    deadline).Connect();
+                                                    deadline, true).Connect();
     };
     auto pg_auto_analyze_service =
         std::make_shared<stateful_service::PgAutoAnalyzeService>(metric_entity(), client_future(),
diff --git a/src/yb/tserver/tablet_service.cc b/src/yb/tserver/tablet_service.cc
@@ -2491,6 +2491,7 @@ void TabletServiceAdminImpl::WaitForYsqlBackendsCatalogVersion(
   const std::string num_lagging_backends_query = Format(
       "SELECT count(*) FROM pg_stat_activity WHERE"
       " backend_type != 'walsender' AND backend_type != 'yb-conn-mgr walsender'"
+      " AND backend_type != 'yb auto analyze backend'"
       " AND catalog_version < $0 AND datid = $1$2",
       catalog_version, database_oid,
       (req->has_requestor_pg_backend_pid() ?
diff --git a/src/yb/yql/pggate/ybc_gflags.cc b/src/yb/yql/pggate/ybc_gflags.cc
@@ -122,6 +122,10 @@ DEFINE_NON_RUNTIME_bool(ysql_enable_relcache_init_optimization, true,
     "disconnected. Instead an internal super user connection is made to perform the "
     "relcache init file rebuild.");
 
+DEFINE_test_flag(bool, ysql_bypass_auto_analyze_auth_check, false,
+    "Bypass the yb-tserver-key authentication method check when connecting using "
+    "yb_auto_analyze backend type.");
+
 DECLARE_bool(ysql_enable_colocated_tables_with_tablespaces);
 DECLARE_bool(TEST_ysql_enable_db_logical_client_version_mode);
 DECLARE_bool(TEST_ysql_yb_enable_ddl_savepoint_support);
@@ -236,7 +240,8 @@ const YbcPgGFlagsAccessor* YBCGetGFlags() {
       .ysql_enable_relcache_init_optimization = &FLAGS_ysql_enable_relcache_init_optimization,
       .placement_cloud = FLAGS_placement_cloud.c_str(),
       .placement_region = FLAGS_placement_region.c_str(),
-      .placement_zone = FLAGS_placement_zone.c_str()
+      .placement_zone = FLAGS_placement_zone.c_str(),
+      .TEST_ysql_bypass_auto_analyze_auth_check = &FLAGS_TEST_ysql_bypass_auto_analyze_auth_check,
   };
   // clang-format on
   return &accessor;
diff --git a/src/yb/yql/pggate/ybc_gflags.h b/src/yb/yql/pggate/ybc_gflags.h
@@ -75,6 +75,7 @@ typedef struct {
   const char *    placement_cloud;
   const char *    placement_region;
   const char *    placement_zone;
+  const bool*     TEST_ysql_bypass_auto_analyze_auth_check;
 } YbcPgGFlagsAccessor;
 
 const YbcPgGFlagsAccessor* YBCGetGFlags();
diff --git a/src/yb/yql/pgwrapper/libpq_utils.cc b/src/yb/yql/pgwrapper/libpq_utils.cc
@@ -140,6 +140,9 @@ std::string BuildConnectionString(const PGConnSettings& settings, bool mask_pass
   if (!settings.replication.empty()) {
     result += Format(" replication=$0", PqEscapeLiteral(settings.replication));
   }
+  if (settings.yb_auto_analyze) {
+    result += Format(" yb_auto_analyze=true");
+  }
   return result;
 }
 
@@ -863,7 +866,8 @@ PGConnPerf::~PGConnPerf() {
 
 PGConnBuilder CreateInternalPGConnBuilder(
     const HostPort& pgsql_proxy_bind_address, const std::string& database_name,
-    uint64_t postgres_auth_key, const std::optional<CoarseTimePoint>& deadline) {
+    uint64_t postgres_auth_key, const std::optional<CoarseTimePoint>& deadline,
+    bool yb_auto_analyze) {
   size_t connect_timeout = 0;
   if (deadline && *deadline != CoarseTimePoint::max()) {
     // By default, connect_timeout is 0, meaning infinite. 1 is automatically converted to 2, so set
@@ -880,7 +884,8 @@ PGConnBuilder CreateInternalPGConnBuilder(
        .dbname = database_name,
        .user = "postgres",
        .password = UInt64ToString(postgres_auth_key),
-       .connect_timeout = connect_timeout});
+       .connect_timeout = connect_timeout,
+       .yb_auto_analyze = yb_auto_analyze});
 }
 
 namespace libpq_utils::internal {
diff --git a/src/yb/yql/pgwrapper/libpq_utils.h b/src/yb/yql/pgwrapper/libpq_utils.h
@@ -375,6 +375,7 @@ struct PGConnSettings {
   std::string password = {};
   std::string replication = {};
   size_t connect_timeout = 0;
+  bool yb_auto_analyze = false;
 };
 
 class PGConnBuilder {
@@ -408,7 +409,8 @@ class PGConnPerf {
 // authentication).
 PGConnBuilder CreateInternalPGConnBuilder(
     const HostPort& pgsql_proxy_bind_address, const std::string& database_name,
-    uint64_t postgres_auth_key, const std::optional<CoarseTimePoint>& deadline);
+    uint64_t postgres_auth_key, const std::optional<CoarseTimePoint>& deadline,
+    bool yb_auto_analyze = false);
 
 Result<std::string> ResultAsString(
     PGresult* res, const std::string& column_sep = DefaultColumnSeparator(),
diff --git a/src/yb/yql/pgwrapper/pg_auto_analyze-test.cc b/src/yb/yql/pgwrapper/pg_auto_analyze-test.cc
@@ -36,6 +36,7 @@
 
 #include "yb/util/backoff_waiter.h"
 #include "yb/util/logging_test_util.h"
+#include "yb/util/pg_util.h"
 #include "yb/util/string_case.h"
 #include "yb/util/test_macros.h"
 #include "yb/util/test_thread_holder.h"
@@ -1405,6 +1406,61 @@ TEST_F(PgConcurrentDDLAnalyzeTest, ConcurrentDDLWithinTxnAnalyze) {
     thread_holder.JoinAll();
 }
 
+class PgConcurrentCreateIndexTest : public PgConcurrentDDLAnalyzeTest {
+ protected:
+  void UpdateMiniClusterOptions(ExternalMiniClusterOptions* options) override {
+    options->extra_tserver_flags.emplace_back(
+            "--ysql_yb_wait_for_backends_catalog_version_timeout=5000");
+    options->extra_tserver_flags.emplace_back(
+            "--wait_for_ysql_backends_catalog_version_client_master_rpc_timeout_ms=1000");
+    options->extra_tserver_flags.emplace_back(
+            "--TEST_ysql_bypass_auto_analyze_auth_check=true");
+    PgConcurrentDDLAnalyzeTest::UpdateMiniClusterOptions(options);
+  }
+};
+
+TEST_F(PgConcurrentCreateIndexTest, ConcurrentCreateIndex) {
+    auto* ts1 = cluster_->tserver_daemons()[0];
+    auto* ts2 = cluster_->tserver_daemons()[1];
+    // Set the read buffer memory limit so that we can successfully establish a connection, but
+    // fail ANALYZEing a table with large primary keys.
+    ts2->AddExtraFlag("read_buffer_memory_limit", "4000000");
+    // Shutdown() is needed before calling Restart().
+    ts2->Shutdown();
+    ASSERT_OK(ts2->Restart());
+    auto conn1 = ASSERT_RESULT(PGConnBuilder({
+      .host = ts1->bind_host(),
+      .port = ts1->ysql_port(),
+    }).Connect());
+    auto conn2 = ASSERT_RESULT(PGConnBuilder({
+      .host = PgDeriveSocketDir(HostPort(ts2->bind_host(), ts2->ysql_port())),
+      .port = ts2->ysql_port(),
+      .user = "yugabyte",
+      .yb_auto_analyze = true,
+    }).Connect());
+
+    ASSERT_OK(conn1.Execute("CREATE TABLE test (k TEXT PRIMARY KEY)"));
+    ASSERT_OK(conn1.Execute("INSERT INTO test SELECT repeat('abcdefg', 100) || '-' || s::TEXT "
+                            "FROM generate_series(1, 10000) AS s"));
+    TestThreadHolder thread_holder;
+    CountDownLatch latch{1};
+    thread_holder.AddThreadFunctor([&conn2, &latch]() -> void {
+        const auto backend_type =
+            ASSERT_RESULT(conn2.FetchRow<std::string>("SELECT backend_type FROM pg_stat_activity "
+                                                       "WHERE pid = pg_backend_pid()"));
+        ASSERT_EQ(backend_type, "yb auto analyze backend");
+        ASSERT_OK(conn2.Execute("SET yb_use_internal_auto_analyze_service_conn=true"));
+        latch.CountDown();
+        auto status = conn2.Execute("ANALYZE test");
+    });
+    // Wait for ANALYZE starts and hangs in conn2.
+    latch.Wait();
+    std::this_thread::sleep_for(3s);
+    ASSERT_OK(conn1.Execute("CREATE INDEX idx ON test(k)"));
+    // TODO(#28915): Force shutdown of ts2 because ANALYZE hangs.
+    ts2->Shutdown(SafeShutdown::kFalse);
+}
+
 class PgConcurrentDDLAnalyzeTestTxnDDL : public PgConcurrentDDLAnalyzeTest {
  protected:
   void UpdateMiniClusterOptions(ExternalMiniClusterOptions* options) override {

Original file line number	Diff line number	Diff line change
`@@ -805,6 +805,7 @@ pg_stat_get_activity(PG_FUNCTION_ARGS)`
`805`	`805`	`proc = BackendPidGetProc(beentry->st_procpid);`
`806`	`806`
`807`	`807`	`if (proc == NULL && (beentry->st_backendType != B_BACKEND &&`
	`808`	`+ beentry->st_backendType != YB_AUTO_ANALYZE_BACKEND &&`
`808`	`809`	`beentry->st_backendType != YB_YSQL_CONN_MGR))`
`809`	`810`	`{`
`810`	`811`	`/*`
Original file line number	Diff line number	Diff line change
`@@ -301,6 +301,9 @@ GetBackendTypeDesc(BackendType backendType)`
`301`	`301`	`break;`
`302`	`302`	`case YB_YSQL_CONN_MGR_WAL_SENDER:`
`303`	`303`	`backendDesc = "yb-conn-mgr walsender";`
	`304`	`+ break;`
	`305`	`+ case YB_AUTO_ANALYZE_BACKEND:`
	`306`	`+ backendDesc = "yb auto analyze backend";`
`304`	`307`	`}`
`305`	`308`
`306`	`309`	`return backendDesc;`
Original file line number	Diff line number	Diff line change
`@@ -2236,6 +2236,8 @@ build_startup_packet(const PGconn conn, char packet,`
`2236`	`2236`	`ADD_STARTUP_OPTION("replication", conn->replication);`
`2237`	`2237`	`if (conn->pgoptions && conn->pgoptions[0])`
`2238`	`2238`	`ADD_STARTUP_OPTION("options", conn->pgoptions);`
	`2239`	`+ if (conn->yb_auto_analyze && conn->yb_auto_analyze[0])`
	`2240`	`+ ADD_STARTUP_OPTION("yb_auto_analyze", conn->yb_auto_analyze);`
`2239`	`2241`	`if (conn->send_appname)`
`2240`	`2242`	`{`
`2241`	`2243`	`/* Use appname if present, otherwise use fallback */`