diff --git a/examples/v2/cloudsql_import/README.md b/examples/v2/cloudsql_import/README.md index f838219c5..aed8ae858 100644 --- a/examples/v2/cloudsql_import/README.md +++ b/examples/v2/cloudsql_import/README.md @@ -2,87 +2,110 @@ ## Overview -This is a [Google Cloud Deployment -Manager](https://cloud.google.com/deployment-manager/overview) template that -deploys a Cloud SQL Master instance and create a list of databases and users. I also use a DM `action` to -import structure to the instance from Google Storage bucket. +This is a [Google Cloud Deployment Manager](https://cloud.google.com/deployment-manager/overview) template that +deploys a Cloud SQL Master instance and creates a list of databases and users. -![Google DM Screenshot](./image/dm-screenshot.png) +I use a DM `action` to: -> Note, CloudSQL does not allow parallel updates to the User table +* Add an object Access Controls to the SQL dump file with the created cloudsql service account. +* Import structure to the instance from Google Storage bucket. + + +![Google DM Screenshot](./img/cloudsql.png) + +> Note, CloudSQL does not allow parallel updates to the User, Databases table. ## Prerequsites -You need to Share publicly the sql file in your bucket. +You need to grant the following roles to the service account that DM uses (```projectNumber@cloudservices.gserviceaccount.com```) -You need to grant +- [roles/cloudsql.admin](https://cloud.google.com/iam/docs/understanding-roles#sql_name_short_roles) -- [roles/cloudsql.admin](https://cloud.google.com/iam/docs/understanding-roles#sql_name_short_roles) role -to the service account DM uses (```projectNumber@cloudservices.gserviceaccount.com```) +- [roles/storage.legacyObjectOwner](https://cloud.google.com/storage/docs/access-control/iam-roles#legacy-roles) + +The template will take care of adding the needed `READER` permission to the sql file used for the import. ## Deploy the template -Use `cloudsql_example.yaml` to deploy this example template. When ready, deploy -with the following command: +Use `cloudsql.yaml` to deploy this example template. When ready, deploy with the following command: ``` -gcloud deployment-manager deployments create my-database --config cloudsql_example.yaml +gcloud deployment-manager deployments update test-dm-1 --config cloudsql_example.yaml ``` **`cloudsql_example.yaml`** ``` +# +# title: CloudSQL +# author: osm.hammami@gmail.com +# description: | +# Creates a CloudSQL deployment +# version: 0.2 + imports: - path: cloudsql.jinja resources: - - name: rebtel-mycloudsql-16 + - name: test-deployment type: cloudsql.jinja properties: - region: us-central1 - zone: us-central1-a - dataDiskSizeGb: 15 - sqlimportstructure: gs://mydeploymentmanager-repository-example/sqldump-export-v0.1.sql - rootpass: myrootsupersafepas - machinetype: db-f1-micro + cloudsql: + databaseVersion: MYSQL_5_7 + region: europe-west3 + tier: db-f1-micro + dataDiskSizeGb: 10 + dataDiskType: PD_SSD + zone: europe-west3-a + backupStartTime: 09:00 + privateNetwork: default + authorizedNetworks: + - name: home-network + value: 192.168.1.1/32 + sqlimport: + bucket: bucket-repository + file: sql-dump.sql + failover: false databases: - - name: dbname1 - - name: dbname2 - charset: latin2 - users: - - name: username1 - password: mysupersafepass1 - host: "44.33.22.11" - - name: username2 - password: mysupersafepass2 - authorization: - - name: ip-test-01 - network: 11.22.33.44/32 - - name: ip-test-02 - network: 55.66.77.88/24 + - name: db1 + charset: utf8 + - name: db2 + charset: utf8 + dbusers: + - user: root + host: '%' + password: mySuperSafePassword + - user: user1 + host: '%' + password: mySuperSafePassword + - user: user2 + host: '%' + password: mySuperSafePassword + databaseFlags: + - name: log_bin_trust_function_creators + value: 'On' + - name: default_time_zone + value: '+00:00' ``` ### Deployment ``` -# gcloud deployment-manager deployments create mydmsqlcloud16 --config cloudsql.yaml -The fingerprint of the deployment is iBtGCTEDiLeGZogkxehXkA== -Waiting for create [operation-1531767208303-5712258545998-5d41e537-f5b748d9]...done. -Create operation operation-1531767208303-5712258545998-5d41e537-f5b748d9 completed successfully. -NAME TYPE STATE ERRORS INTENT -rebtel-mycloudsql-16-db-dbname1 sqladmin.v1beta4.database COMPLETED [] -rebtel-mycloudsql-16-db-dbname2 sqladmin.v1beta4.database COMPLETED [] -rebtel-mycloudsql-16-import-structure gcp-types/sqladmin-v1beta4:sql.instances.import COMPLETED [] -rebtel-mycloudsql-16-master sqladmin.v1beta4.instance COMPLETED [] -rebtel-mycloudsql-16-user-username1 sqladmin.v1beta4.user COMPLETED [] -rebtel-mycloudsql-16-user-username2 sqladmin.v1beta4.user COMPLETED [] -rebtel-mycloudsql-16-user-root sqladmin.v1beta4.user COMPLETED [] +# gcloud deployment-manager deployments update test-dm-1 --config cloudsql_example.yaml + +NAME TYPE STATE ERRORS INTENT +test-deployment-cloudsql sqladmin.v1beta4.instance COMPLETED [] +test-deployment-cloudsql-acl-deploymentmanager-repository-rebtelsqldbinfra-v0.1.sql gcp-types/storage-v1:storage.objectAccessControls.insert COMPLETED [] +test-deployment-cloudsql-db-db1 sqladmin.v1beta4.database COMPLETED [] +test-deployment-cloudsql-db-db2 sqladmin.v1beta4.database COMPLETED [] +test-deployment-cloudsql-dbuser-root sqladmin.v1beta4.user COMPLETED [] +test-deployment-cloudsql-dbuser-user1 sqladmin.v1beta4.user COMPLETED [] +test-deployment-cloudsql-dbuser-user2 sqladmin.v1beta4.user COMPLETED [] +test-deployment-cloudsql-import-structure gcp-types/sqladmin-v1beta4:sql.instances.import COMPLETED [] ``` -## References +### References -* [Services > Cloud SQL Administration API v1beta4 > sql.instances.import](https://developers.google.com/apis-explorer/#p/sqladmin/v1beta4/sql.instances.import) -* [Cloud SQL Example](https://github.com/GoogleCloudPlatform/deploymentmanager-samples/tree/master/examples/v2/sqladmin) -* [Supported Resource Types](https://cloud.google.com/deployment-manager/docs/configuration/supported-resource-types) -* [Access control for Deployment Manager](https://cloud.google.com/deployment-manager/docs/access-control#access_control_for_deployment_manager) -* [Cloud storage objects Action to upload ](https://github.com/GoogleCloudPlatform/deploymentmanager-samples/issues/40) \ No newline at end of file +* [API reference Cloud SQL v1beta4](https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/) +* [API explorer Cloud SQL v1beta4](https://developers.google.com/apis-explorer/#p/sqladmin/v1beta4/) +* [CloudSQL Testing Framework](https://github.com/GoogleCloudPlatform/deploymentmanager-samples/tree/master/examples/v2/cloudsql) diff --git a/examples/v2/cloudsql_import/cloudsql.jinja b/examples/v2/cloudsql_import/cloudsql.jinja index 657a43fe6..cf7d34746 100644 --- a/examples/v2/cloudsql_import/cloudsql.jinja +++ b/examples/v2/cloudsql_import/cloudsql.jinja @@ -1,37 +1,75 @@ {# - 20180716 - ohammami - version 0.1 + title: CloudSQL + author: osm.hammami@gmail.com + description: | + Creates a CloudSQL deployment + version: 0.2 #} {% set ID = env['name'] %} +{% set MASTER = ID %} +{% set FAILOVER = ID + '-failover' %} + + +{%- macro dbDependencyList(failover='', databases='', dbusers='') -%} + {% if failover %} + - {{ FAILOVER }} + {% endif %} + {% for i in range(databases| length) %} + - {{ ID }}-db-{{ properties['databases'][i]['name'] }} + {% endfor %} + {% for i in range(dbusers| length) %} + - {{ ID }}-dbuser-{{ properties['dbusers'][i]['user'] }} + {% endfor %} +{%- endmacro %} + +{% set usersDeps = dbDependencyList(properties['failover'], properties['databases']) %} +{% set importDeps = dbDependencyList(properties['failover'], properties['databases'], properties['dbusers']) %} resources: -- name: {{ ID }}-master +- name: {{ MASTER }} type: sqladmin.v1beta4.instance properties: backendType: SECOND_GEN instanceType: CLOUD_SQL_INSTANCE - databaseVersion: MYSQL_5_7 + databaseVersion: {{ properties['cloudsql']['databaseVersion'] }} region: {{ properties['region'] }} settings: - tier: {{ properties['machinetype'] }} - dataDiskSizeGb: {{ properties['dataDiskSizeGb'] }} - dataDiskType: PD_SSD + tier: {{ properties['cloudsql']['tier'] }} + dataDiskSizeGb: {{ properties['cloudsql']['dataDiskSizeGb'] }} + dataDiskType: {{ properties['cloudsql']['dataDiskType'] }} storageAutoResize: true + replicationType: SYNCHRONOUS locationPreference: zone: {{ properties['zone'] }} + {% if properties['databaseFlags'] %} + databaseFlags: {{ properties['databaseFlags'] }} + {% endif %} activationPolicy: ALWAYS backupConfiguration: enabled: true binaryLogEnabled: true - startTime: 00:00 -{% if properties['authorization'] %} + startTime: {{ properties['cloudsql']['backupStartTime'] }} + replicationLogArchivingEnabled: false ipConfiguration: - authorizedNetworks: - {% for i in range(properties['authorization']| length) %} - - name: {{ properties['authorization'][i]['name'] }} - value: {{ properties['authorization'][i]['network'] }} - {% endfor %} + privateNetwork: projects/{{ env['project'] }}/global/networks/{{ properties['cloudsql']['privateNetwork'] }} + ipv4Enabled: true + requireSsl: false + authorizedNetworks: {{ properties['cloudsql']['authorizedNetworks'] }} + +{% if properties['failover'] %} +- name: {{ FAILOVER }} + type: sqladmin.v1beta4.instance + properties: + backendType: SECOND_GEN + instanceType: READ_REPLICA_INSTANCE + databaseVersion: {{ properties['cloudsql']['databaseVersion'] }} + region: {{ properties['region'] }} + masterInstanceName: $(ref.{{ MASTER }}.name) + replicaConfiguration: + failoverTarget: true + settings: + tier: {{ properties['cloudsql']['tier'] }} {% endif %} {% if properties['databases'] %} @@ -40,79 +78,73 @@ resources: type: sqladmin.v1beta4.database properties: name: {{ properties['databases'][i]['name'] }} - instance: $(ref.{{ ID }}-master.name) - {% if properties['databases'][i]['charset'] %} + instance: $(ref.{{ MASTER }}.name) charset: {{ properties['databases'][i]['charset'] }} - {% else %} - charset: utf8 - {% endif %} - {% if (i - 1) >= 0 %} metadata: - dependsOn: - - {{ ID }}-db-{{ properties['databases'][i - 1]['name'] }} + dependsOn: + - {{ MASTER }} + {% if properties['failover'] %} + - {{ FAILOVER }} + {% endif %} + {% if i %} + - {{ ID }}-db-{{ properties['databases'][i - 1]['name'] }} {% endif %} {% endfor %} {% endif %} -- name: {{ ID }}-user-root +{% if properties['dbusers'] %} + {% for i in range(properties['dbusers']| length) %} +- name: {{ ID }}-dbuser-{{ properties['dbusers'][i]['user'] }} type: sqladmin.v1beta4.user properties: - name: root - host: "%" - instance: $(ref.{{ ID }}-master.name) - password: {{ properties['rootpass'] }} -{% if properties['databases'] %} + name: {{ properties['dbusers'][i]['user'] }} + host: "{{ properties['dbusers'][i]['host'] }}" + instance: $(ref.{{ MASTER }}.name) + password: {{ properties['dbusers'][i]['password'] }} metadata: dependsOn: - - {{ ID }}-db-{{ properties['databases'][(properties['databases']| length) - 1]['name'] }} + {{ usersDeps }} + {% if i %} + - {{ ID }}-dbuser-{{ properties['dbusers'][i - 1]['user'] }} + {% endif %} + {% endfor %} {% endif %} -{% if properties['users'] %} - {% for i in range(properties['users']| length) %} -- name: {{ ID }}-user-{{ properties['users'][i]['name'] }} - type: sqladmin.v1beta4.user +{% if properties['sqlimport'] %} +{# write permissions to the bucket; entity -> env[username] and read access to the file; entity -> ref[username] #} +- name: {{ ID }}-acl-{{ properties['sqlimport']['bucket'] + '-' + properties['sqlimport']['file'] }} + action: gcp-types/storage-v1:storage.objectAccessControls.insert properties: - name: {{ properties['users'][i]['name'] }} -{% if properties['users'][i]['host'] %} - host: "{{ properties['users'][i]['host'] }}" -{% else %} - host: "%" -{% endif %} - instance: $(ref.{{ ID }}-master.name) - password: {{ properties['users'][i]['password'] }} - metadata: - dependsOn: - {% if (i - 1) >= 0 %} - - {{ ID }}-user-{{ properties['users'][i - 1]['name'] }} - {% else %} - - {{ ID }}-user-root - {% endif %} - {% endfor %} -{% endif %} + bucket: {{ properties['sqlimport']['bucket'] }} + object: {{ properties['sqlimport']['file'] }} + entity: user-$(ref.{{ MASTER }}.serviceAccountEmailAddress) + role: READER -{% if properties['sqlimportstructure'] %} - name: {{ ID }}-import-structure action: gcp-types/sqladmin-v1beta4:sql.instances.import properties: - instance: {{ ID }}-master - project: {{ env['project'] }} + instance: {{ MASTER }} importContext: - kind: sql#importContext fileType: SQL - uri: {{ properties['sqlimportstructure'] }} + uri: gs://{{ properties['sqlimport']['bucket'] }}/{{ properties['sqlimport']['file'] }} metadata: dependsOn: - {% if properties['users'] %} - - {{ ID }}-user-{{ properties['users'][(properties['users']| length) - 1]['name'] }} - {% else %} - - {{ ID }}-user-root - {% endif %} + - {{ ID }}-acl-{{ properties['sqlimport']['bucket'] + '-' + properties['sqlimport']['file'] }} + {{ importDeps }} {% endif %} outputs: - - name: {{ ID }}-master-ip - value: $(ref.{{ ID }}-master.ipAddresses[0].ipAddress) - - name: {{ ID }}-master-connectionName - value: $(ref.{{ ID }}-master.connectionName) - - name: {{ ID }}-master-serviceAccount - value: $(ref.{{ ID }}-master.serviceAccountEmailAddress) + - name: {{ MASTER }}-private-ip + value: $(ref.{{ MASTER }}.ipAddresses[1].ipAddress) + - name: {{ MASTER }}-ip + value: $(ref.{{ MASTER }}.ipAddresses[0].ipAddress) + - name: {{ MASTER }}-connectionName + value: $(ref.{{ MASTER }}.connectionName) + - name: {{ MASTER }}-serviceAccount + value: $(ref.{{ MASTER }}.serviceAccountEmailAddress) + {% if properties['failover'] %} + - name: {{ FAILOVER }}-ip + value: $(ref.{{ FAILOVER }}.ipAddresses[0].ipAddress) + - name: {{ FAILOVER }}-connectionName + value: $(ref.{{ FAILOVER }}.connectionName) + {% endif %} diff --git a/examples/v2/cloudsql_import/cloudsql.jinja.schema b/examples/v2/cloudsql_import/cloudsql.jinja.schema index f1d1fdc0c..53f7e6ec2 100644 --- a/examples/v2/cloudsql_import/cloudsql.jinja.schema +++ b/examples/v2/cloudsql_import/cloudsql.jinja.schema @@ -1,16 +1,16 @@ info: - title: Cloud SQL Template - author: ohammami - description: Creates a new sql cloud instance - version: 1.0 - -imports: -- path: cloudsql.jinja - -#required: -#- rootpass + title: CloudSQL + author: osm.hammami@gmail.com + description: | + Creates a CloudSQL deployment + version: 0.2 properties: + failover: + type: boolean + description: enable failover replica + default: true + region: type: string description: i.e. europe-west3 @@ -18,39 +18,100 @@ properties: zone: type: string + description: i.e. europe-west3-a default: europe-west3-a - description: Zone to run - dataDiskSizeGb: - type: integer - minimum: 10 - maximum: 10000 - default: 20 + sqlimport: + type: object + description: An object of SQL dump file to import to Cloud SQL + required: + - bucket + - file + properties: + bucket: + type: string + file: + type: string - sqlimportstructure: - type: string - description: uri of the sqldump to import + cloudsql: + type: object + default: + properties: + properties: + databaseVersion: + type: string + description: MYSQL_5_7 or MYSQL_5_6 + default: MYSQL_5_7 + dataDiskSizeGb: + type: integer + minimum: 10 + maximum: 10000 + default: 20 + dataDiskType: + type: string + decription: PD_SSD or PD_HDD + default: PD_SSD + backupStartTime: + type: string + description: HH:MM in 24 hour format + default: 00:00 + privateNetwork: + type: string + description: The ressource link for the VPC network from which the Cloud SQL instance is accessible for private IP. + default: default + tier: + type: string + description: https://cloud.google.com/sql/pricing#2nd-gen-pricing + default: db-n1-standard-2 + authorizedNetworks: + type: array + description: An array of allowed CIDR blocks - rootpass: - type: string - description: root password - default: mysaferootpass - - machinetype: - type: string - description: machine type, for more info https://cloud.google.com/sql/pricing#2nd-gen-pricing - default: db-f1-micro - - databases: + databaseFlags: type: array - description: An array of mysql databases + description: An array of https://cloud.google.com/sql/docs/mysql/flags + items: + type: object + required: + - name + - value + properties: + name: + type: string + value: + type: + - integer + - string - users: + dbusers: type: array - description: An array of allowed userss - - authorization: + description: An array for the SQL users. + items: + type: object + required: + - password + properties: + name: + type: string + default: root + host: + type: string + default: '%' + password: + type: string + + databases: type: array - description: An array of allowed CIDR blocks + description: An array for the SQL databases. + items: + type: object + required: + - name + properties: + name: + type: string + charset: + type: string + description: https://dev.mysql.com/doc/refman/5.7/en/charset.html + default: utf8 - diff --git a/examples/v2/cloudsql_import/cloudsql_example.yaml b/examples/v2/cloudsql_import/cloudsql_example.yaml index 6c953bc6e..417e7f033 100644 --- a/examples/v2/cloudsql_import/cloudsql_example.yaml +++ b/examples/v2/cloudsql_import/cloudsql_example.yaml @@ -1,34 +1,49 @@ -# -# 20180716 - ohammami -# version 0.1 -# +# title: CloudSQL +# author: osm.hammami@gmail.com +# description: | +# Creates a CloudSQL deployment +# version: 0.2 imports: - path: cloudsql.jinja resources: - - name: rebtel-mycloudsql-16 + - name: test-deployment-cloudsql type: cloudsql.jinja properties: - region: us-central1 - zone: us-central1-a - dataDiskSizeGb: 15 - sqlimportstructure: gs://deploymentmanager-repository/rebtelsqldbinfra-v0.1.sql - rootpass: myrootsupersafepas - machinetype: db-f1-micro + region: europe-west3 + zone: europe-west3-a + cloudsql: + databaseVersion: MYSQL_5_7 + tier: db-f1-micro + dataDiskSizeGb: 10 + dataDiskType: PD_SSD + backupStartTime: 09:00 + privateNetwork: default + authorizedNetworks: + - name: home-network + value: 213.89.132.110/32 + sqlimport: + bucket: deploymentmanager-repository + file: rebtelsqldbinfra-v0.1.sql + failover: false databases: - - name: kamailio - - name: asterisk - charset: latin2 - users: - - name: kamailio - password: mysupersafepass1 - host: "44.33.22.11" - - name: asterisk - password: mysupersafepass2 - authorization: - - name: ip-test-01 - network: 79.142.255.106/32 - - name: ip-test-02 - network: 79.142.255.206/32 - + - name: db1 + charset: utf8 + - name: db2 + charset: utf8 + dbusers: + - user: root + host: '%' + password: mySuperSafePassword + - user: user1 + host: '%' + password: mySuperSafePassword + - user: user2 + host: '%' + password: mySuperSafePassword + databaseFlags: + - name: log_bin_trust_function_creators + value: 'On' + - name: default_time_zone + value: '+00:00' diff --git a/examples/v2/cloudsql_import/img/cloudsql.png b/examples/v2/cloudsql_import/img/cloudsql.png new file mode 100644 index 000000000..3e999e176 Binary files /dev/null and b/examples/v2/cloudsql_import/img/cloudsql.png differ