YDB FQ: support Redis as an external data source (#16957)

Author: Glebbs

ydb/core/external_sources/external_source_factory.cpp

@@ -140,6 +140,10 @@ IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector<TStri
         {
             ToString(NYql::EDatabaseType::Iceberg),
             BuildIcebergSource(hostnamePatternsRegEx)
+        },
+        {
+            ToString(NYql::EDatabaseType::Redis),
+            CreateExternalDataSource(TString{NYql::GenericProviderName}, {"BASIC"}, {"database_name", "use_tls"}, hostnamePatternsRegEx)
         }
     },
     availableExternalDataSources); 

ydb/library/yql/providers/common/db_id_async_resolver/database_type.cpp

@@ -18,7 +18,8 @@ std::set<TString> GetAllExternalDataSourceTypes() {
         ToString(NYql::EDatabaseType::Oracle),
         ToString(NYql::EDatabaseType::Logging),
         ToString(NYql::EDatabaseType::Solomon),
-        ToString(NYql::EDatabaseType::Iceberg)
+        ToString(NYql::EDatabaseType::Iceberg),
+        ToString(NYql::EDatabaseType::Redis)
     };
     return allTypes;
 }
@@ -43,6 +44,8 @@ EDatabaseType DatabaseTypeFromDataSourceKind(NYql::EGenericDataSourceKind dataSo
             return EDatabaseType::Logging;
         case NYql::EGenericDataSourceKind::ICEBERG:
             return EDatabaseType::Iceberg;
+        case NYql::EGenericDataSourceKind::REDIS:
+            return EDatabaseType::Redis;
         default:
             ythrow yexception() << "Unknown data source kind: " << NYql::EGenericDataSourceKind_Name(dataSourceKind);
     }
@@ -68,6 +71,8 @@ NYql::EGenericDataSourceKind DatabaseTypeToDataSourceKind(EDatabaseType database
             return NYql::EGenericDataSourceKind::LOGGING;
         case EDatabaseType::Iceberg:
             return NYql::EGenericDataSourceKind::ICEBERG;
+        case EDatabaseType::Redis:
+            return NYql::EGenericDataSourceKind::REDIS;
         default:
             ythrow yexception() << "Unknown database type: " << ToString(databaseType);
     }

ydb/library/yql/providers/common/db_id_async_resolver/database_type.h

@@ -19,7 +19,8 @@ enum class EDatabaseType {
     Oracle,
     Logging,
     Solomon,
-    Iceberg
+    Iceberg,
+    Redis
 };
 
 std::set<TString> GetAllExternalDataSourceTypes();

ydb/library/yql/providers/generic/actors/yql_generic_provider_factories.cpp

@@ -52,7 +52,8 @@ namespace NYql::NDq {
                  "MsSQLServerGeneric",
                  "OracleGeneric",
                  "LoggingGeneric",
-                 "IcebergGeneric"}) {
+                 "IcebergGeneric",
+                 "RedisGeneric"}) {
             factory.RegisterSource<NGeneric::TSource>(name, readActorFactory);
             factory.RegisterLookupSource<NGeneric::TLookupSource>(name, lookupActorFactory);
         }

ydb/library/yql/providers/generic/connector/tests/utils/run/kqprun.py

@@ -197,6 +197,7 @@ class AppConfigRenderer:
   AvailableExternalDataSources: "Oracle"
   AvailableExternalDataSources: "Logging"
   AvailableExternalDataSources: "Solomon"
+  AvailableExternalDataSources: "Redis"
   Generic {
     Connector {
         Endpoint {

ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.cpp

@@ -214,7 +214,8 @@ namespace NYql {
                 EGenericDataSourceKind::MYSQL,
                 EGenericDataSourceKind::MS_SQL_SERVER,
                 EGenericDataSourceKind::ORACLE,
-                EGenericDataSourceKind::ICEBERG
+                EGenericDataSourceKind::ICEBERG,
+                EGenericDataSourceKind::REDIS
                 }, 
                clusterConfig.GetKind()
             )) {
@@ -358,7 +359,6 @@ namespace NYql {
         ParseServiceAccountIdSignature(properties, clusterConfig);
         ParseToken(properties, clusterConfig);
         ParseFolderId(properties, clusterConfig);
-
         ParseIcebergFields(properties, clusterConfig);
 
         return clusterConfig;

ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp

@@ -40,6 +40,8 @@ namespace NYql {
                     return "LoggingGeneric";
                 case NYql::EGenericDataSourceKind::ICEBERG:
                     return "IcebergGeneric";
+                case NYql::EGenericDataSourceKind::REDIS:
+                    return "RedisGeneric";
                 default:
                     throw yexception() << "Data source kind is unknown or not specified";
             }
@@ -282,6 +284,9 @@ namespace NYql {
                         case NYql::EGenericDataSourceKind::ICEBERG:
                             properties["SourceType"] = "Iceberg";
                             break;
+                        case NYql::EGenericDataSourceKind::REDIS:
+                            properties["SourceType"] = "Redis";
+                            break;
                         case NYql::EGenericDataSourceKind::DATA_SOURCE_KIND_UNSPECIFIED:
                             break;
                         default:

ydb/library/yql/providers/generic/provider/yql_generic_load_meta.cpp

@@ -540,6 +540,8 @@ namespace NYql {
                     auto* options = request.mutable_data_source_instance()->mutable_iceberg_options();
                     SetIcebergOptions(*options, clusterConfig);
                 } break;
+                case NYql::EGenericDataSourceKind::REDIS:
+                    break;
                 default:
                     throw yexception() << "Unexpected data source kind: '"
                                        << NYql::EGenericDataSourceKind_Name(dataSourceKind) << "'";

ydb/tests/tools/kqprun/configuration/app_config.conf

@@ -98,6 +98,7 @@ QueryServiceConfig {
   AvailableExternalDataSources: "Logging"
   AvailableExternalDataSources: "Solomon"
   AvailableExternalDataSources: "Iceberg"
+  AvailableExternalDataSources: "Redis"
 
   FileStorage {
     MaxFiles: 1000