diff --git a/VERSION b/VERSION index 758a46e..d941c12 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.15 \ No newline at end of file +1.0.16 \ No newline at end of file diff --git a/docs/data-sources/instance.md b/docs/data-sources/instance.md index e3e515d..5c71752 100644 --- a/docs/data-sources/instance.md +++ b/docs/data-sources/instance.md @@ -22,6 +22,7 @@ The instance data source. ### Read-Only - `data_sources` (Set of Object) (see [below for nested schema](#nestedatt--data_sources)) +- `databases` (Set of String) The databases full name in the resource. - `engine` (String) The instance engine. Support MYSQL, POSTGRES, TIDB, SNOWFLAKE, CLICKHOUSE, MONGODB, SQLITE, REDIS, ORACLE, SPANNER, MSSQL, REDSHIFT, MARIADB, OCEANBASE. - `engine_version` (String) The engine version. - `environment` (String) The environment name for your instance in "environments/{resource id}" format. diff --git a/docs/data-sources/instance_list.md b/docs/data-sources/instance_list.md index 859e9d1..22b4d2b 100644 --- a/docs/data-sources/instance_list.md +++ b/docs/data-sources/instance_list.md @@ -30,6 +30,7 @@ The instance data source list. Read-Only: - `data_sources` (Set of Object) (see [below for nested schema](#nestedobjatt--instances--data_sources)) +- `databases` (Set of String) - `engine` (String) - `engine_version` (String) - `environment` (String) diff --git a/docs/data-sources/project.md b/docs/data-sources/project.md index f7e28ae..f41a088 100644 --- a/docs/data-sources/project.md +++ b/docs/data-sources/project.md @@ -24,6 +24,7 @@ The project data source. - `allow_modify_statement` (Boolean) Allow modifying statement after issue is created. - `auto_enable_backup` (Boolean) Whether to automatically enable backup. - `auto_resolve_issue` (Boolean) Enable auto resolve issue. +- `databases` (Set of String) The databases full name in the resource. - `enforce_issue_title` (Boolean) Enforce issue title created by user instead of generated by Bytebase. - `id` (String) The ID of this resource. - `key` (String) The project key. diff --git a/docs/data-sources/project_list.md b/docs/data-sources/project_list.md index ac762ac..221682d 100644 --- a/docs/data-sources/project_list.md +++ b/docs/data-sources/project_list.md @@ -32,6 +32,7 @@ Read-Only: - `allow_modify_statement` (Boolean) - `auto_enable_backup` (Boolean) - `auto_resolve_issue` (Boolean) +- `databases` (Set of String) - `enforce_issue_title` (Boolean) - `key` (String) - `members` (Set of Object) (see [below for nested schema](#nestedobjatt--projects--members)) diff --git a/docs/resources/instance.md b/docs/resources/instance.md index c192bdd..435e187 100644 --- a/docs/resources/instance.md +++ b/docs/resources/instance.md @@ -31,6 +31,7 @@ The instance resource. ### Read-Only +- `databases` (Set of String) The databases full name in the resource. - `engine_version` (String) The engine version. - `id` (String) The ID of this resource. - `name` (String) The instance full name in instances/{resource id} format. diff --git a/docs/resources/project.md b/docs/resources/project.md index 83ec524..455940b 100644 --- a/docs/resources/project.md +++ b/docs/resources/project.md @@ -26,6 +26,7 @@ The project resource. - `allow_modify_statement` (Boolean) Allow modifying statement after issue is created. - `auto_enable_backup` (Boolean) Whether to automatically enable backup. - `auto_resolve_issue` (Boolean) Enable auto resolve issue. +- `databases` (Set of String) The databases full name in the resource. - `enforce_issue_title` (Boolean) Enforce issue title created by user instead of generated by Bytebase. - `members` (Block Set) The members in the project. (see [below for nested schema](#nestedblock--members)) - `postgres_database_tenant_mode` (Boolean) Whether to enable the database tenant mode for PostgreSQL. If enabled, the issue will be created with the pre-appended "set role " statement. diff --git a/examples/environments/main.tf b/examples/environments/main.tf index 84b68ad..ef99921 100644 --- a/examples/environments/main.tf +++ b/examples/environments/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { bytebase = { - version = "1.0.15" + version = "1.0.16" # For local development, please use "terraform.local/bytebase/bytebase" instead source = "registry.terraform.io/bytebase/bytebase" } diff --git a/examples/groups/main.tf b/examples/groups/main.tf index 4fec0e7..b537339 100644 --- a/examples/groups/main.tf +++ b/examples/groups/main.tf @@ -1,7 +1,7 @@ terraform { required_providers { bytebase = { - version = "1.0.15" + version = "1.0.16" # For local development, please use "terraform.local/bytebase/bytebase" instead source = "registry.terraform.io/bytebase/bytebase" } diff --git a/examples/instances/main.tf b/examples/instances/main.tf index 503fb02..140e2e7 100644 --- a/examples/instances/main.tf +++ b/examples/instances/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { bytebase = { - version = "1.0.15" + version = "1.0.16" # For local development, please use "terraform.local/bytebase/bytebase" instead source = "registry.terraform.io/bytebase/bytebase" } diff --git a/examples/policies/main.tf b/examples/policies/main.tf index daea781..ef1474d 100644 --- a/examples/policies/main.tf +++ b/examples/policies/main.tf @@ -1,7 +1,7 @@ terraform { required_providers { bytebase = { - version = "1.0.15" + version = "1.0.16" # For local development, please use "terraform.local/bytebase/bytebase" instead source = "registry.terraform.io/bytebase/bytebase" } diff --git a/examples/projects/main.tf b/examples/projects/main.tf index cd77c15..989dd20 100644 --- a/examples/projects/main.tf +++ b/examples/projects/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { bytebase = { - version = "1.0.15" + version = "1.0.16" # For local development, please use "terraform.local/bytebase/bytebase" instead source = "registry.terraform.io/bytebase/bytebase" } diff --git a/examples/settings/main.tf b/examples/settings/main.tf index 979e1af..f9e0bbe 100644 --- a/examples/settings/main.tf +++ b/examples/settings/main.tf @@ -1,7 +1,7 @@ terraform { required_providers { bytebase = { - version = "1.0.15" + version = "1.0.16" # For local development, please use "terraform.local/bytebase/bytebase" instead source = "registry.terraform.io/bytebase/bytebase" } diff --git a/examples/setup/main.tf b/examples/setup/main.tf index 9b02b18..bdb27e5 100644 --- a/examples/setup/main.tf +++ b/examples/setup/main.tf @@ -1,7 +1,7 @@ terraform { required_providers { bytebase = { - version = "1.0.15" + version = "1.0.16" # For local development, please use "terraform.local/bytebase/bytebase" instead source = "registry.terraform.io/bytebase/bytebase" } diff --git a/examples/setup/project.tf b/examples/setup/project.tf index 8d438f3..118f39a 100644 --- a/examples/setup/project.tf +++ b/examples/setup/project.tf @@ -31,4 +31,6 @@ resource "bytebase_project" "sample_project" { expire_timestamp = "2027-03-09T16:17:49Z" } } + + databases = bytebase_instance.prod.databases } diff --git a/examples/users/main.tf b/examples/users/main.tf index 80cdb6a..ca916b4 100644 --- a/examples/users/main.tf +++ b/examples/users/main.tf @@ -1,7 +1,7 @@ terraform { required_providers { bytebase = { - version = "1.0.15" + version = "1.0.16" # For local development, please use "terraform.local/bytebase/bytebase" instead source = "registry.terraform.io/bytebase/bytebase" } diff --git a/examples/vcs/main.tf b/examples/vcs/main.tf index f776b2e..307c6c4 100644 --- a/examples/vcs/main.tf +++ b/examples/vcs/main.tf @@ -1,7 +1,7 @@ terraform { required_providers { bytebase = { - version = "1.0.15" + version = "1.0.16" # For local development, please use "terraform.local/bytebase/bytebase" instead source = "registry.terraform.io/bytebase/bytebase" } diff --git a/provider/data_source_instance.go b/provider/data_source_instance.go index 7b651c4..7993f9d 100644 --- a/provider/data_source_instance.go +++ b/provider/data_source_instance.go @@ -125,6 +125,7 @@ func dataSourceInstance() *schema.Resource { }, Set: dataSourceHash, }, + "databases": getDatabasesSchema(true), }, } } @@ -140,5 +141,5 @@ func dataSourceInstanceRead(ctx context.Context, d *schema.ResourceData, m inter d.SetId(ins.Name) - return setInstanceMessage(ctx, d, ins) + return setInstanceMessage(ctx, c, d, ins) } diff --git a/provider/data_source_instance_list.go b/provider/data_source_instance_list.go index 70ceb26..342916d 100644 --- a/provider/data_source_instance_list.go +++ b/provider/data_source_instance_list.go @@ -136,6 +136,7 @@ func dataSourceInstanceList() *schema.Resource { }, Set: dataSourceHash, }, + "databases": getDatabasesSchema(true), }, }, }, @@ -181,6 +182,12 @@ func dataSourceInstanceListRead(ctx context.Context, d *schema.ResourceData, m i } ins["data_sources"] = schema.NewSet(dataSourceHash, dataSources) + databases, err := c.ListDatabase(ctx, instance.Name, "") + if err != nil { + return diag.FromErr(err) + } + ins["databases"] = flattenDatabaseList(databases) + instances = append(instances, ins) } diff --git a/provider/data_source_project.go b/provider/data_source_project.go index b3915b7..5d82538 100644 --- a/provider/data_source_project.go +++ b/provider/data_source_project.go @@ -80,7 +80,20 @@ func dataSourceProject() *schema.Resource { Computed: true, Description: "Whether to enable the database tenant mode for PostgreSQL. If enabled, the issue will be created with the pre-appended \"set role \" statement.", }, - "members": getProjectMembersSchema(true), + "members": getProjectMembersSchema(true), + "databases": getDatabasesSchema(true), + }, + } +} + +func getDatabasesSchema(computed bool) *schema.Schema { + return &schema.Schema{ + Type: schema.TypeSet, + Computed: computed, + Optional: !computed, + Description: "The databases full name in the resource.", + Elem: &schema.Schema{ + Type: schema.TypeString, }, } } @@ -231,6 +244,14 @@ func flattenMemberList(iamPolicy *v1pb.IamPolicy) ([]interface{}, error) { return memberList, nil } +func flattenDatabaseList(databases []*v1pb.Database) []interface{} { + dbList := []interface{}{} + for _, database := range databases { + dbList = append(dbList, database.Name) + } + return dbList +} + func setProject( ctx context.Context, client api.Client, @@ -241,6 +262,11 @@ func setProject( "project": project.Name, }) + databases, err := client.ListDatabase(ctx, project.Name, "") + if err != nil { + return diag.FromErr(err) + } + iamPolicy, err := client.GetProjectIAMPolicy(ctx, project.Name) if err != nil { return diag.Errorf("failed to get project iam with error: %v", err) @@ -290,6 +316,17 @@ func setProject( } startTime := time.Now() + databaseList := flattenDatabaseList(databases) + if err := d.Set("databases", databaseList); err != nil { + return diag.Errorf("cannot set databases for project: %s", err.Error()) + } + tflog.Debug(ctx, "[read project] set project databases", map[string]interface{}{ + "project": project.Name, + "databases": len(databases), + "ms": time.Since(startTime).Milliseconds(), + }) + + startTime = time.Now() memberList, err := flattenMemberList(iamPolicy) if err != nil { return diag.FromErr(err) diff --git a/provider/data_source_project_list.go b/provider/data_source_project_list.go index 71607e1..4b0f697 100644 --- a/provider/data_source_project_list.go +++ b/provider/data_source_project_list.go @@ -83,7 +83,8 @@ func dataSourceProjectList() *schema.Resource { Computed: true, Description: "Whether to enable the database tenant mode for PostgreSQL. If enabled, the issue will be created with the pre-appended \"set role \" statement.", }, - "members": getProjectMembersSchema(true), + "members": getProjectMembersSchema(true), + "databases": getDatabasesSchema(true), }, }, }, @@ -122,6 +123,14 @@ func dataSourceProjectListRead(ctx context.Context, d *schema.ResourceData, m in proj["skip_backup_errors"] = project.AllowModifyStatement proj["postgres_database_tenant_mode"] = project.PostgresDatabaseTenantMode + databases, err := c.ListDatabase(ctx, project.Name, "") + if err != nil { + return diag.FromErr(err) + } + + databaseList := flattenDatabaseList(databases) + proj["databases"] = databaseList + iamPolicy, err := c.GetProjectIAMPolicy(ctx, project.Name) if err != nil { return diag.Errorf("failed to get project iam with error: %v", err) diff --git a/provider/resource_instance.go b/provider/resource_instance.go index 7ec872e..8e33947 100644 --- a/provider/resource_instance.go +++ b/provider/resource_instance.go @@ -181,6 +181,7 @@ func resourceInstance() *schema.Resource { }, Set: dataSourceHash, }, + "databases": getDatabasesSchema(true), }, } } @@ -338,7 +339,7 @@ func resourceInstanceRead(ctx context.Context, d *schema.ResourceData, m interfa return diag.FromErr(err) } - resp := setInstanceMessage(ctx, d, instance) + resp := setInstanceMessage(ctx, c, d, instance) tflog.Debug(ctx, "[read instance] read instance finished", map[string]interface{}{ "instance": instance.Name, }) @@ -454,6 +455,7 @@ func resourceInstanceDelete(ctx context.Context, d *schema.ResourceData, m inter func setInstanceMessage( ctx context.Context, + client api.Client, d *schema.ResourceData, instance *v1pb.Instance, ) diag.Diagnostics { @@ -503,6 +505,19 @@ func setInstanceMessage( return diag.Errorf("cannot set data_sources for instance: %s", err.Error()) } + tflog.Debug(ctx, "[read instance] start set instance databases", map[string]interface{}{ + "instance": instance.Name, + }) + + databases, err := client.ListDatabase(ctx, instance.Name, "") + if err != nil { + return diag.FromErr(err) + } + databaseList := flattenDatabaseList(databases) + if err := d.Set("databases", databaseList); err != nil { + return diag.Errorf("cannot set databases for instance: %s", err.Error()) + } + return nil } diff --git a/provider/resource_project.go b/provider/resource_project.go index 4bce1f9..6147779 100644 --- a/provider/resource_project.go +++ b/provider/resource_project.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" "github.com/pkg/errors" "google.golang.org/genproto/googleapis/type/expr" + "google.golang.org/protobuf/types/known/fieldmaskpb" "github.com/bytebase/terraform-provider-bytebase/api" "github.com/bytebase/terraform-provider-bytebase/provider/internal" @@ -97,7 +98,8 @@ func resourceProjct() *schema.Resource { Computed: true, Description: "Whether to enable the database tenant mode for PostgreSQL. If enabled, the issue will be created with the pre-appended \"set role \" statement.", }, - "members": getProjectMembersSchema(false), + "members": getProjectMembersSchema(false), + "databases": getDatabasesSchema(false), }, } } @@ -214,6 +216,11 @@ func resourceProjectCreate(ctx context.Context, d *schema.ResourceData, m interf d.SetId(projectName) + if diag := updateDatabasesInProject(ctx, d, c, d.Id()); diag != nil { + diags = append(diags, diag...) + return diags + } + if diag := updateMembersInProject(ctx, d, c, d.Id()); diag != nil { diags = append(diags, diag...) return diags @@ -317,6 +324,13 @@ func resourceProjectUpdate(ctx context.Context, d *schema.ResourceData, m interf } } + if d.HasChange("databases") { + if diag := updateDatabasesInProject(ctx, d, c, d.Id()); diag != nil { + diags = append(diags, diag...) + return diags + } + } + if d.HasChange("members") { if diag := updateMembersInProject(ctx, d, c, d.Id()); diag != nil { diags = append(diags, diag...) @@ -446,3 +460,105 @@ func updateMembersInProject(ctx context.Context, d *schema.ResourceData, client } return nil } + +const batchSize = 100 + +func updateDatabasesInProject(ctx context.Context, d *schema.ResourceData, client api.Client, projectName string) diag.Diagnostics { + databases, err := client.ListDatabase(ctx, projectName, "") + if err != nil { + return diag.Errorf("failed to list database with error: %v", err.Error()) + } + existedDBMap := map[string]*v1pb.Database{} + for _, db := range databases { + existedDBMap[db.Name] = db + } + + rawSet, ok := d.Get("databases").(*schema.Set) + if !ok { + return nil + } + updatedDBMap := map[string]*v1pb.Database{} + batchTransferDatabases := []*v1pb.UpdateDatabaseRequest{} + for _, raw := range rawSet.List() { + dbName := raw.(string) + if _, _, err := internal.GetInstanceDatabaseID(dbName); err != nil { + return diag.Errorf("invalid database full name: %v", err.Error()) + } + + updatedDBMap[dbName] = &v1pb.Database{ + Name: dbName, + Project: projectName, + } + _, ok := existedDBMap[dbName] + if !ok { + // new assigned database + batchTransferDatabases = append(batchTransferDatabases, &v1pb.UpdateDatabaseRequest{ + Database: updatedDBMap[dbName], + UpdateMask: &fieldmaskpb.FieldMask{ + Paths: []string{"project"}, + }, + }) + } else { + delete(existedDBMap, dbName) + } + } + + tflog.Debug(ctx, "[transfer databases] batch transfer databases to project", map[string]interface{}{ + "project": projectName, + "databases": len(batchTransferDatabases), + }) + + for i := 0; i < len(batchTransferDatabases); i += batchSize { + end := i + batchSize + if end > len(batchTransferDatabases) { + end = len(batchTransferDatabases) + } + batch := batchTransferDatabases[i:end] + startTime := time.Now() + + if _, err := client.BatchUpdateDatabases(ctx, &v1pb.BatchUpdateDatabasesRequest{ + Requests: batch, + Parent: "instances/-", + }); err != nil { + return diag.Errorf("failed to assign databases to project %s with error: %v", projectName, err.Error()) + } + + tflog.Debug(ctx, "[transfer databases]", map[string]interface{}{ + "count": end + 1 - i, + "project": projectName, + "ms": time.Since(startTime).Milliseconds(), + }) + } + + if len(existedDBMap) > 0 { + tflog.Debug(ctx, "[transfer databases] batch unassign databases", map[string]interface{}{ + "project": projectName, + "databases": len(existedDBMap), + }) + + startTime := time.Now() + unassignDatabases := []*v1pb.UpdateDatabaseRequest{} + for _, db := range existedDBMap { + // move db to default project + db.Project = defaultProj + unassignDatabases = append(unassignDatabases, &v1pb.UpdateDatabaseRequest{ + Database: db, + UpdateMask: &fieldmaskpb.FieldMask{ + Paths: []string{"project"}, + }, + }) + } + if _, err := client.BatchUpdateDatabases(ctx, &v1pb.BatchUpdateDatabasesRequest{ + Requests: unassignDatabases, + Parent: "instances/-", + }); err != nil { + return diag.Errorf("failed to move databases to default project with error: %v", err.Error()) + } + tflog.Debug(ctx, "[unassign databases]", map[string]interface{}{ + "count": len(unassignDatabases), + "ms": time.Since(startTime).Milliseconds(), + }) + } + + return nil +}