feat(tables): add yc_quotamanager_* tables

commit_hash:71c1e194b01afe0957bbacae413836ef928b9da3

Author: kabachook

.mapping.json

@@ -96,6 +96,8 @@
   "docs/tables/yc_organizationmanager_organizations.md":"security/cloudquery/cq-source-yc/docs/tables/yc_organizationmanager_organizations.md",
   "docs/tables/yc_organizationmanager_oslogin_settings.md":"security/cloudquery/cq-source-yc/docs/tables/yc_organizationmanager_oslogin_settings.md",
   "docs/tables/yc_organizationmanager_users.md":"security/cloudquery/cq-source-yc/docs/tables/yc_organizationmanager_users.md",
+  "docs/tables/yc_quotamanager_quota_limits.md":"security/cloudquery/cq-source-yc/docs/tables/yc_quotamanager_quota_limits.md",
+  "docs/tables/yc_quotamanager_quota_services.md":"security/cloudquery/cq-source-yc/docs/tables/yc_quotamanager_quota_services.md",
   "docs/tables/yc_resourcemanager_clouds.md":"security/cloudquery/cq-source-yc/docs/tables/yc_resourcemanager_clouds.md",
   "docs/tables/yc_resourcemanager_folders.md":"security/cloudquery/cq-source-yc/docs/tables/yc_resourcemanager_folders.md",
   "docs/tables/yc_serverless_apigateway_gateways.md":"security/cloudquery/cq-source-yc/docs/tables/yc_serverless_apigateway_gateways.md",
@@ -193,6 +195,8 @@
   "resources/organizationmanager/organizations.go":"security/cloudquery/cq-source-yc/resources/organizationmanager/organizations.go",
   "resources/organizationmanager/oslogin_settings.go":"security/cloudquery/cq-source-yc/resources/organizationmanager/oslogin_settings.go",
   "resources/organizationmanager/users.go":"security/cloudquery/cq-source-yc/resources/organizationmanager/users.go",
+  "resources/quotamanager/limits.go":"security/cloudquery/cq-source-yc/resources/quotamanager/limits.go",
+  "resources/quotamanager/services.go":"security/cloudquery/cq-source-yc/resources/quotamanager/services.go",
   "resources/resourcemanager/clouds.go":"security/cloudquery/cq-source-yc/resources/resourcemanager/clouds.go",
   "resources/resourcemanager/folders.go":"security/cloudquery/cq-source-yc/resources/resourcemanager/folders.go",
   "resources/serverless/api_gateways.go":"security/cloudquery/cq-source-yc/resources/serverless/api_gateways.go",

client/client.go

@@ -25,10 +25,11 @@ const (
 type Client struct {
 	hierarchy *ResourceHierarchy
 
-	OrganizationId        string
-	CloudId               string
-	FolderId              string
-	MultiplexedResourceId string
+	OrganizationId          string
+	CloudId                 string
+	FolderId                string
+	MultiplexedResourceId   string
+	MultiplexedResourceType ResourceType
 
 	Backend state.Client
 	Logger  zerolog.Logger
@@ -59,20 +60,23 @@ func (c *Client) WithOrganization(id string) *Client {
 	newC := *c
 	newC.Logger = c.Logger.With().Str("organization", id).Logger()
 	newC.OrganizationId = id
+	newC.MultiplexedResourceType = ResourceTypeOrganization
 	return &newC
 }
 
 func (c *Client) WithCloud(id string) *Client {
 	newC := *c
 	newC.Logger = c.Logger.With().Str("cloud", id).Logger()
 	newC.CloudId = id
+	newC.MultiplexedResourceType = ResourceTypeCloud
 	return &newC
 }
 
 func (c *Client) WithFolder(id string) *Client {
 	newC := *c
 	newC.Logger = c.Logger.With().Str("folder", id).Logger()
 	newC.FolderId = id
+	newC.MultiplexedResourceType = ResourceTypeFolder
 	return &newC
 }
 
@@ -96,8 +100,7 @@ func New(ctx context.Context, logger zerolog.Logger, spec *Spec) (*Client, error
 	streamInterceptors := []grpc.StreamClientInterceptor{}
 
 	var dialOpts = []grpc.DialOption{
-		grpc.WithChainUnaryInterceptor(unaryInterceptors...),
-		grpc.WithChainStreamInterceptor(streamInterceptors...),
+		grpc.WithUserAgent(DefaultUserAgent),
 	}
 
 	if spec.MaxRetries > 0 {
@@ -117,14 +120,14 @@ func New(ctx context.Context, logger zerolog.Logger, spec *Spec) (*Client, error
 		streamInterceptors = append(streamInterceptors, logging.StreamClientInterceptor(grpczerolog.InterceptorLogger(logger)))
 	}
 
+	dialOpts = append(dialOpts, grpc.WithChainUnaryInterceptor(unaryInterceptors...), grpc.WithChainStreamInterceptor(streamInterceptors...))
+
 	sdk, err := ycsdk.Build(ctx,
 		ycsdk.Config{
 			Credentials: credentials,
 			Endpoint:    spec.Endpoint,
 		},
-		grpc.WithUserAgent(DefaultUserAgent),
-		grpc.WithChainUnaryInterceptor(unaryInterceptors...),
-		grpc.WithChainStreamInterceptor(streamInterceptors...),
+		dialOpts...,
 	)
 	if err != nil {
 		return nil, fmt.Errorf("initialize Yandex Cloud SDK: %w", err)

client/multiplexer.go

@@ -43,3 +43,13 @@ func PrependEmptyMultiplex(multiplexer schema.Multiplexer) schema.Multiplexer {
 		return append([]schema.ClientMeta{client}, multiplexer(meta)...)
 	}
 }
+
+func CombineMultiplex(multiplexers ...schema.Multiplexer) schema.Multiplexer {
+	return func(meta schema.ClientMeta) []schema.ClientMeta {
+		clients := []schema.ClientMeta{}
+		for _, m := range multiplexers {
+			clients = append(clients, m(meta)...)
+		}
+		return clients
+	}
+}

client/resolvers.go

@@ -10,6 +10,7 @@ import (
 	"google.golang.org/protobuf/reflect/protoreflect"
 	"google.golang.org/protobuf/runtime/protoimpl"
 	"google.golang.org/protobuf/types/known/timestamppb"
+	"google.golang.org/protobuf/types/known/wrapperspb"
 )
 
 func ResolveProtoEnum(path string) schema.ColumnResolver {
@@ -40,6 +41,20 @@ func ResolveProtoTimestamp(path string) schema.ColumnResolver {
 	}
 }
 
+func ResolveDouble(path string) schema.ColumnResolver {
+	return func(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error {
+		data := funk.Get(resource.Item, path)
+		if data == nil {
+			return nil
+		}
+		double, ok := data.(*wrapperspb.DoubleValue)
+		if !ok {
+			return fmt.Errorf("unexpected type, wanted \"*wrapperspb.DoubleValue\", have \"%T\"", data)
+		}
+		return resource.Set(c.Name, double.GetValue())
+	}
+}
+
 var ParentIdColumn schema.Column = schema.Column{
 	Name:       "id",
 	Type:       arrow.BinaryTypes.String,

client/resourcehierarchy.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/rs/zerolog"
 	"github.com/yandex-cloud/cq-source-yc/internal/util"
+	"github.com/yandex-cloud/go-genproto/yandex/cloud/billing/v1"
 	"github.com/yandex-cloud/go-genproto/yandex/cloud/endpoint"
 	"github.com/yandex-cloud/go-genproto/yandex/cloud/organizationmanager/v1"
 	"github.com/yandex-cloud/go-genproto/yandex/cloud/resourcemanager/v1"
@@ -18,7 +19,8 @@ import (
 )
 
 // Self-made name for struct holding info about resource model hierarchy.
-// Each field could be nil.
+// Each field could be empty.
+// https://yandex.cloud/en/docs/resource-manager/concepts/resources-hierarchy
 type ResourceHierarchyItem struct {
 	Organization string
 	Cloud        string
@@ -84,11 +86,36 @@ func (h *ResourceHierarchy) FolderRows() []ResourceHierarchyItem {
 	})
 }
 
+type Service string
+
+const (
+	ServiceResourceManager     Service = "resource-manager"
+	ServiceOrganizationManager Service = "organization-manager"
+)
+
+type ResourceType string
+
 const (
-	serviceResourceManager     = "resource-manager"
-	serviceOrganizationManager = "organization-manager"
+	ResourceTypeOrganization   ResourceType = "organization-manager.organization"
+	ResourceTypeCloud          ResourceType = "resource-manager.cloud"
+	ResourceTypeFolder         ResourceType = "resource-manager.folder"
+	ResourceTypeBillingAccount ResourceType = "billing.account"
 )
 
+// TODO: codegen
+func ResourceTypeFromProto(p any) (ResourceType, bool) {
+	switch p.(type) {
+	case organizationmanager.Organization:
+		return ResourceTypeOrganization, true
+	case resourcemanager.Cloud:
+		return ResourceTypeCloud, true
+	case billing.BillingAccount:
+		return ResourceTypeBillingAccount, true
+	default:
+		return ResourceType(""), false
+	}
+}
+
 // discover the hierarchy using Breadth-first search
 func bfs(ctx context.Context, sdk *ycsdk.SDK, logger zerolog.Logger, init []ResourceHierarchyItem, organizations []string, clouds []string, folders []string, opts ...grpc.CallOption) ([]ResourceHierarchyItem, error) {
 	// call ApiEndpoint.List to fill internal endpoint list in ycsdk.SDK
@@ -123,7 +150,7 @@ func bfs(ctx context.Context, sdk *ycsdk.SDK, logger zerolog.Logger, init []Reso
 		if item.Folder != "" {
 			// just append to items
 			continue
-		} else if item.Cloud != "" && services[serviceResourceManager] {
+		} else if item.Cloud != "" && services[string(ServiceResourceManager)] {
 			// discover folders
 			it := sdk.ResourceManager().Folder().FolderIterator(ctx, &resourcemanager.ListFoldersRequest{CloudId: item.Cloud}, opts...)
 			for it.Next() {
@@ -144,7 +171,7 @@ func bfs(ctx context.Context, sdk *ycsdk.SDK, logger zerolog.Logger, init []Reso
 				}
 				return nil, err
 			}
-		} else if item.Organization != "" && services[serviceResourceManager] {
+		} else if item.Organization != "" && services[string(ServiceResourceManager)] {
 			// discover clouds
 			it := sdk.ResourceManager().Cloud().CloudIterator(ctx, &resourcemanager.ListCloudsRequest{OrganizationId: item.Organization}, opts...)
 			for it.Next() {
@@ -165,7 +192,7 @@ func bfs(ctx context.Context, sdk *ycsdk.SDK, logger zerolog.Logger, init []Reso
 				}
 				return nil, err
 			}
-		} else if services[serviceOrganizationManager] {
+		} else if services[string(ServiceOrganizationManager)] {
 			// discover organizations
 			it := sdk.OrganizationManager().Organization().OrganizationIterator(ctx, &organizationmanager.ListOrganizationsRequest{}, opts...)
 			for it.Next() {

client/transformers.go

@@ -9,13 +9,17 @@ import (
 	cqtypes "github.com/cloudquery/plugin-sdk/v4/types"
 	"google.golang.org/protobuf/reflect/protoreflect"
 	"google.golang.org/protobuf/types/known/timestamppb"
+	"google.golang.org/protobuf/types/known/wrapperspb"
 )
 
 func typeTransformer(field reflect.StructField) (arrow.DataType, error) {
 	switch reflect.New(field.Type).Elem().Interface().(type) {
 	case *timestamppb.Timestamp,
 		timestamppb.Timestamp:
 		return arrow.FixedWidthTypes.Timestamp_us, nil
+	case *wrapperspb.DoubleValue,
+		wrapperspb.DoubleValue:
+		return arrow.PrimitiveTypes.Float64, nil
 	case protoreflect.Enum:
 		return arrow.BinaryTypes.String, nil
 	case nil:
@@ -30,6 +34,9 @@ func resolverTransformer(field reflect.StructField, path string) schema.ColumnRe
 	case *timestamppb.Timestamp,
 		timestamppb.Timestamp:
 		return ResolveProtoTimestamp(path)
+	case *wrapperspb.DoubleValue,
+		wrapperspb.DoubleValue:
+		return ResolveDouble(path)
 	case protoreflect.Enum:
 		return ResolveProtoEnum(path)
 	default:

docs/tables/README.md

@@ -71,6 +71,8 @@
 - [yc_organizationmanager_organizations](yc_organizationmanager_organizations.md)
 - [yc_organizationmanager_oslogin_settings](yc_organizationmanager_oslogin_settings.md)
 - [yc_organizationmanager_users](yc_organizationmanager_users.md)
+- [yc_quotamanager_quota_services](yc_quotamanager_quota_services.md)
+  - [yc_quotamanager_quota_limits](yc_quotamanager_quota_limits.md)
 - [yc_resourcemanager_clouds](yc_resourcemanager_clouds.md)
 - [yc_resourcemanager_folders](yc_resourcemanager_folders.md)
 - [yc_serverless_apigateway_gateways](yc_serverless_apigateway_gateways.md)
@@ -88,4 +90,4 @@
 - [yc_vpc_security_groups](yc_vpc_security_groups.md)
 - [yc_vpc_subnets](yc_vpc_subnets.md)
 - [yc_ydb_databases](yc_ydb_databases.md)
-  - [yc_access_bindings_ydb_databases](yc_access_bindings_ydb_databases.md)
+  - [yc_access_bindings_ydb_databases](yc_access_bindings_ydb_databases.md)

docs/tables/yc_cdn_resources.md

@@ -25,4 +25,6 @@ The primary key for this table is **id**.
 |origin_group_name|`utf8`|
 |origin_protocol|`utf8`|
 |ssl_certificate|`json`|
-|labels|`json`|
+|labels|`json`|
+|provider_type|`utf8`|
+|provider_cname|`utf8`|

docs/tables/yc_quotamanager_quota_limits.md

@@ -0,0 +1,23 @@
+# Table: yc_quotamanager_quota_limits
+
+This table shows data for YC Quota Manager Quota Limits.
+
+https://yandex.cloud/ru/docs/quota-manager/api-ref/grpc/QuotaLimit/list#yandex.cloud.quotamanager.v1.QuotaLimit
+
+The composite primary key for this table is (**resource_id**, **quota_id**).
+
+## Relations
+
+This table depends on [yc_quotamanager_quota_services](yc_quotamanager_quota_services.md).
+
+## Columns
+
+| Name          | Type          |
+| ------------- | ------------- |
+|_cq_id|`uuid`|
+|_cq_parent_id|`uuid`|
+|resource_id (PK)|`utf8`|
+|resource_type|`utf8`|
+|quota_id (PK)|`utf8`|
+|limit|`json`|
+|usage|`json`|

docs/tables/yc_quotamanager_quota_services.md

@@ -0,0 +1,22 @@
+# Table: yc_quotamanager_quota_services
+
+This table shows data for YC Quota Manager Quota Services.
+
+https://yandex.cloud/ru/docs/quota-manager/api-ref/grpc/QuotaLimit/listServices#yandex.cloud.quotamanager.v1.Service
+
+The composite primary key for this table is (**id**, **resource_type**).
+
+## Relations
+
+The following tables depend on yc_quotamanager_quota_services:
+  - [yc_quotamanager_quota_limits](yc_quotamanager_quota_limits.md)
+
+## Columns
+
+| Name          | Type          |
+| ------------- | ------------- |
+|_cq_id|`uuid`|
+|_cq_parent_id|`uuid`|
+|id (PK)|`utf8`|
+|name|`utf8`|
+|resource_type (PK)|`utf8`|