YQ-3561 FQrun supported remote databases (ydb-platform#15016)

Author: GrigoriyPA

ydb/core/fq/libs/actors/pending_fetcher.cpp

@@ -105,6 +105,13 @@ struct TEvPrivate {
 
 constexpr auto CLEANUP_PERIOD = TDuration::Seconds(60);
 
+TDuration GetPendingFetchPeriod(const NFq::NConfig::TPendingFetcherConfig& config) {
+    if (const auto periodMs = config.GetPendingFetchPeriodMs()) {
+        return TDuration::MilliSeconds(periodMs);
+    }
+    return TDuration::Seconds(1);
+}
+
 } // namespace
 
 class TPendingFetcher : public NActors::TActorBootstrapped<TPendingFetcher> {
@@ -170,6 +177,7 @@ class TPendingFetcher : public NActors::TActorBootstrapped<TPendingFetcher> {
         , ServiceCounters(serviceCounters, "pending_fetcher")
         , GetTaskCounters("GetTask", ServiceCounters.Counters)
         , FailedStatusCodeCounters(MakeIntrusive<TStatusCodeByScopeCounters>("IntermediateFailedStatusCode", ServiceCounters.RootCounters->GetSubgroup("component", "QueryDiagnostic")))
+        , PendingFetchPeriod(GetPendingFetchPeriod(config.GetPendingFetcher()))
         , CredentialsFactory(credentialsFactory)
         , S3Gateway(s3Gateway)
         , ConnectorClient(connectorClient)
@@ -518,7 +526,7 @@ class TPendingFetcher : public NActors::TActorBootstrapped<TPendingFetcher> {
     IModuleResolver::TPtr ModuleResolver;
 
     bool HasRunningRequest = false;
-    const TDuration PendingFetchPeriod = TDuration::Seconds(1);
+    const TDuration PendingFetchPeriod;
 
     TActorId DatabaseResolver;
 

ydb/core/fq/libs/config/protos/pending_fetcher.proto

@@ -8,4 +8,5 @@ option java_package = "ru.yandex.kikimr.proto";
 
 message TPendingFetcherConfig {
     bool Enabled = 1;
+    uint64 PendingFetchPeriodMs = 2;  // 1s by default
 }

ydb/core/fq/libs/row_dispatcher/leader_election.cpp

@@ -180,7 +180,7 @@ TLeaderElection::TLeaderElection(
     : Config(config)
     , CredentialsProviderFactory(credentialsProviderFactory)
     , YqSharedResources(yqSharedResources)
-    , YdbConnection(NewYdbConnection(config.GetDatabase(), credentialsProviderFactory, yqSharedResources->UserSpaceYdbDriver))
+    , YdbConnection(config.GetLocalMode() ? nullptr : NewYdbConnection(config.GetDatabase(), credentialsProviderFactory, yqSharedResources->UserSpaceYdbDriver))
     , TablePathPrefix(JoinPath(config.GetDatabase().GetDatabase(), config.GetCoordinationNodePath()))
     , CoordinationNodePath(JoinPath(TablePathPrefix, tenant))
     , ParentId(parentId)

ydb/tests/tools/fqrun/README.md

@@ -8,7 +8,7 @@ For profiling memory allocations build fqrun with ya make flags `-D PROFILE_MEMO
 
 * `flame_graph.sh` - script for collecting flame graphs in svg format, usage:
     ```(bash)
-    ./flame_graph.sh [graph collection time in seconds]
+    ./flame_graph.sh [graph collection time in seconds] [use sudo]
     ```
 
 ## Examples

ydb/tests/tools/fqrun/configuration/fq_config.conf

@@ -3,7 +3,6 @@ EnableDynamicNameservice: true
 EnableTaskCounters: true
 
 CheckpointCoordinator {
-  Enabled: true
   CheckpointingPeriodMillis: 30000
   MaxInflight: 1
 
@@ -24,7 +23,7 @@ Common {
   MdbGateway: "https://mdb.api.cloud.yandex.net:443"
   MdbTransformHost: false
   ObjectStorageEndpoint: "https://storage.yandexcloud.net"
-  IdsPrefix: "kr"
+  IdsPrefix: "fr"
   QueryArtifactsCompressionMethod: "zstd_6"
   MonitoringEndpoint: "monitoring.api.cloud.yandex.net"
   KeepInternalErrors: true
@@ -42,10 +41,12 @@ Common {
 
 ControlPlaneProxy {
   Enabled: true
+  RequestTimeout: "1m"
 }
 
 ControlPlaneStorage {
   Enabled: true
+  UseInMemory: true
   StatsMode: STATS_MODE_PROFILE
   DumpRawStatistics: true
   TasksBatchSize: 100
@@ -84,6 +85,9 @@ ControlPlaneStorage {
     OperationTimeoutSec: 60
     CancelAfterSec: 60
   }
+
+  RetryPolicyMapping {
+  }
 }
 
 DbPool {
@@ -201,10 +205,6 @@ Gateways {
   }
 }
 
-Health {
-  Enabled: true
-}
-
 NodesManager {
   Enabled: true
 }
@@ -223,7 +223,6 @@ PrivateProxy {
 }
 
 QuotasManager {
-  Enabled: true
   QuotaDescriptions {
     SubjectType: "cloud"
     MetricName: "yq.cpuPercent.count"
@@ -244,10 +243,6 @@ QuotasManager {
 }
 
 RateLimiter {
-  Enabled: true
-  ControlPlaneEnabled: true
-  DataPlaneEnabled: true
-
   Database {
     TablePrefix: "yq/rate_limiter"
     ClientTimeoutSec: 70

ydb/tests/tools/fqrun/flame_graph.sh

@@ -2,24 +2,11 @@
 
 set -eux
 
-function cleanup {
-    sudo rm ./profdata
-    rm ./profdata.txt
-}
-trap cleanup EXIT
-
-if [ $# -gt 1 ]; then
+if [ $# -gt 2 ]; then
     echo "Too many arguments"
     exit -1
 fi
 
-fqrun_pid=$(pgrep -u $USER fqrun)
-
-sudo perf record -F 50 --call-graph dwarf -g --proc-map-timeout=10000 --pid $fqrun_pid -v -o profdata -- sleep ${1:-'30'}
-sudo perf script -i profdata > profdata.txt
-
 SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
 
-flame_graph_tool="$SCRIPT_DIR/../../../../contrib/tools/flame-graph/"
-
-${flame_graph_tool}/stackcollapse-perf.pl profdata.txt | ${flame_graph_tool}/flamegraph.pl > profdata.svg
+$SCRIPT_DIR/../kqprun/flame_graph.sh ${1:-'30'} ${2:-''} fqrun