Merge pull request #1094 from atykhyy/assume_container_exists

Azure storage: allow skipping container creation

Author: Slach

ReadMe.md

@@ -205,19 +205,20 @@ clickhouse:
   max_connections: 0 # CLICKHOUSE_MAX_CONNECTIONS, how many parallel connections could be opened during operations
 azblob:
   endpoint_suffix: "core.windows.net" # AZBLOB_ENDPOINT_SUFFIX
-  account_name: ""             # AZBLOB_ACCOUNT_NAME
-  account_key: ""              # AZBLOB_ACCOUNT_KEY
-  sas: ""                      # AZBLOB_SAS
-  use_managed_identity: false  # AZBLOB_USE_MANAGED_IDENTITY
-  container: ""                # AZBLOB_CONTAINER
-  path: ""                     # AZBLOB_PATH, `system.macros` values can be applied as {macro_name}
-  object_disk_path: ""         # AZBLOB_OBJECT_DISK_PATH, path for backup of part from clickhouse object disks, if object disks present in clickhouse, then shall not be zero and shall not be prefixed by `path`
-  compression_level: 1         # AZBLOB_COMPRESSION_LEVEL
-  compression_format: tar      # AZBLOB_COMPRESSION_FORMAT, allowed values tar, lz4, bzip2, gzip, sz, xz, brortli, zstd, `none` for upload data part folders as is
-  sse_key: ""                  # AZBLOB_SSE_KEY
-  max_parts_count: 256         # AZBLOB_MAX_PARTS_COUNT, number of parts for AZBLOB uploads, for properly calculate buffer size
-  max_buffers: 3               # AZBLOB_MAX_BUFFERS, similar with S3_CONCURRENCY
-  debug: false                 # AZBLOB_DEBUG
+  account_name: ""               # AZBLOB_ACCOUNT_NAME
+  account_key: ""                # AZBLOB_ACCOUNT_KEY
+  sas: ""                        # AZBLOB_SAS
+  use_managed_identity: false    # AZBLOB_USE_MANAGED_IDENTITY
+  container: ""                  # AZBLOB_CONTAINER
+  assume_container_exists: false # AZBLOB_ASSUME_CONTAINER_EXISTS, enables assignment of rights granting clickhouse-backup access only to blobs in the container
+  path: ""                       # AZBLOB_PATH, `system.macros` values can be applied as {macro_name}
+  object_disk_path: ""           # AZBLOB_OBJECT_DISK_PATH, path for backup of part from clickhouse object disks, if object disks present in clickhouse, then shall not be zero and shall not be prefixed by `path`
+  compression_level: 1           # AZBLOB_COMPRESSION_LEVEL
+  compression_format: tar        # AZBLOB_COMPRESSION_FORMAT, allowed values tar, lz4, bzip2, gzip, sz, xz, brortli, zstd, `none` for upload data part folders as is
+  sse_key: ""                    # AZBLOB_SSE_KEY
+  max_parts_count: 256           # AZBLOB_MAX_PARTS_COUNT, number of parts for AZBLOB uploads, for properly calculate buffer size
+  max_buffers: 3                 # AZBLOB_MAX_BUFFERS, similar with S3_CONCURRENCY
+  debug: false                   # AZBLOB_DEBUG
 s3:
   access_key: ""                   # S3_ACCESS_KEY
   secret_key: ""                   # S3_SECRET_KEY

pkg/config/config.go

@@ -105,6 +105,7 @@ type AzureBlobConfig struct {
 	SharedAccessSignature string `yaml:"sas" envconfig:"AZBLOB_SAS"`
 	UseManagedIdentity    bool   `yaml:"use_managed_identity" envconfig:"AZBLOB_USE_MANAGED_IDENTITY"`
 	Container             string `yaml:"container" envconfig:"AZBLOB_CONTAINER"`
+	AssumeContainerExists bool   `yaml:"assume_container_exists" envconfig:"AZBLOB_ASSUME_CONTAINER_EXISTS"`
 	Path                  string `yaml:"path" envconfig:"AZBLOB_PATH"`
 	ObjectDiskPath        string `yaml:"object_disk_path" envconfig:"AZBLOB_OBJECT_DISK_PATH"`
 	CompressionLevel      int    `yaml:"compression_level" envconfig:"AZBLOB_COMPRESSION_LEVEL"`

pkg/storage/azblob.go

@@ -151,9 +151,11 @@ func (a *AzureBlob) Connect(ctx context.Context) error {
 		}
 		a.Pipeline = azblob.NewPipeline(credential, options)
 		a.Container = azblob.NewServiceURL(*u, a.Pipeline).NewContainerURL(a.Config.Container)
-		_, err = a.Container.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone)
-		if err != nil && !isContainerAlreadyExists(err) {
-			return err
+		if !a.Config.AssumeContainerExists {
+			_, err = a.Container.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone)
+			if err != nil && !isContainerAlreadyExists(err) {
+				return err
+			}
 		}
 		if a.Config.SSEKey != "" {
 			key, err := base64.StdEncoding.DecodeString(a.Config.SSEKey)

test/testflows/clickhouse_backup/tests/snapshots/cli.py.cli.snapshot

@@ -1,4 +1,4 @@
-default_config = r"""'[\'general:\', \'    remote_storage: none\', \'    backups_to_keep_local: 0\', \'    backups_to_keep_remote: 0\', \'    log_level: info\', \'    allow_empty_backups: false\', \'    allow_object_disk_streaming: false\', \'    use_resumable_state: true\', \'    restore_schema_on_cluster: ""\', \'    upload_by_part: true\', \'    download_by_part: true\', \'    restore_database_mapping: {}\', \'    restore_table_mapping: {}\', \'    retries_on_failure: 3\', \'    retries_pause: 5s\', \'    watch_interval: 1h\', \'    full_interval: 24h\', \'    watch_backup_name_template: shard{shard}-{type}-{time:20060102150405}\', \'    sharded_operation_mode: ""\', \'    cpu_nice_priority: 15\', \'    io_nice_priority: idle\', \'    rbac_backup_always: true\', \'    rbac_conflict_resolution: recreate\', \'    retriesduration: 5s\', \'    watchduration: 1h0m0s\', \'    fullduration: 24h0m0s\', \'clickhouse:\', \'    username: default\', \'    password: ""\', \'    host: localhost\', \'    port: 9000\', \'    disk_mapping: {}\', \'    skip_tables:\', \'        - system.*\', \'        - INFORMATION_SCHEMA.*\', \'        - information_schema.*\', \'        - _temporary_and_external_tables.*\', \'    skip_table_engines: []\', \'    timeout: 30m\', \'    freeze_by_part: false\', \'    freeze_by_part_where: ""\', \'    use_embedded_backup_restore: false\', \'    embedded_backup_disk: ""\', \'    backup_mutations: true\', \'    restore_as_attach: false\', \'    check_parts_columns: true\', \'    secure: false\', \'    skip_verify: false\', \'    sync_replicated_tables: false\', \'    log_sql_queries: true\', \'    config_dir: /etc/clickhouse-server/\', \'    restart_command: exec:systemctl restart clickhouse-server\', \'    ignore_not_exists_error_during_freeze: true\', \'    check_replicas_before_attach: true\', \'    default_replica_path: /clickhouse/tables/{cluster}/{shard}/{database}/{table}\', "    default_replica_name: \'{replica}\'", \'    tls_key: ""\', \'    tls_cert: ""\', \'    tls_ca: ""\', \'    debug: false\', \'s3:\', \'    access_key: ""\', \'    secret_key: ""\', \'    bucket: ""\', \'    endpoint: ""\', \'    region: us-east-1\', \'    acl: private\', \'    assume_role_arn: ""\', \'    force_path_style: false\', \'    path: ""\', \'    object_disk_path: ""\', \'    disable_ssl: false\', \'    compression_level: 1\', \'    compression_format: tar\', \'    sse: ""\', \'    sse_kms_key_id: ""\', \'    sse_customer_algorithm: ""\', \'    sse_customer_key: ""\', \'    sse_customer_key_md5: ""\', \'    sse_kms_encryption_context: ""\', \'    disable_cert_verification: false\', \'    use_custom_storage_class: false\', \'    storage_class: STANDARD\', \'    custom_storage_class_map: {}\', \'    allow_multipart_download: false\', \'    object_labels: {}\', \'    request_payer: ""\', \'    check_sum_algorithm: ""\', \'    debug: false\', \'gcs:\', \'    credentials_file: ""\', \'    credentials_json: ""\', \'    credentials_json_encoded: ""\', \'    embedded_access_key: ""\', \'    embedded_secret_key: ""\', \'    skip_credentials: false\', \'    bucket: ""\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_level: 1\', \'    compression_format: tar\', \'    debug: false\', \'    force_http: false\', \'    endpoint: ""\', \'    storage_class: STANDARD\', \'    object_labels: {}\', \'    custom_storage_class_map: {}\', \'    chunk_size: 0\', \'cos:\', \'    url: ""\', \'    timeout: 2m\', \'    secret_id: ""\', \'    secret_key: ""\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_format: tar\', \'    compression_level: 1\', \'    debug: false\', \'api:\', \'    listen: localhost:7171\', \'    enable_metrics: true\', \'    enable_pprof: false\', \'    username: ""\', \'    password: ""\', \'    secure: false\', \'    certificate_file: ""\', \'    private_key_file: ""\', \'    ca_cert_file: ""\', \'    ca_key_file: ""\', \'    create_integration_tables: false\', \'    integration_tables_host: ""\', \'    allow_parallel: false\', \'    complete_resumable_after_restart: true\', \'    watch_is_main_process: false\', \'ftp:\', \'    address: ""\', \'    timeout: 2m\', \'    username: ""\', \'    password: ""\', \'    tls: false\', \'    skip_tls_verify: false\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_format: tar\', \'    compression_level: 1\', \'    debug: false\', \'sftp:\', \'    address: ""\', \'    port: 22\', \'    username: ""\', \'    password: ""\', \'    key: ""\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_format: tar\', \'    compression_level: 1\', \'    debug: false\', \'azblob:\', \'    endpoint_schema: https\', \'    endpoint_suffix: core.windows.net\', \'    account_name: ""\', \'    account_key: ""\', \'    sas: ""\', \'    use_managed_identity: false\', \'    container: ""\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_level: 1\', \'    compression_format: tar\', \'    sse_key: ""\', \'    buffer_count: 3\', \'    timeout: 4h\', \'    debug: false\', \'custom:\', \'    upload_command: ""\', \'    download_command: ""\', \'    list_command: ""\', \'    delete_command: ""\', \'    command_timeout: 4h\', \'    commandtimeoutduration: 4h0m0s\']'"""
+default_config = r"""'[\'general:\', \'    remote_storage: none\', \'    backups_to_keep_local: 0\', \'    backups_to_keep_remote: 0\', \'    log_level: info\', \'    allow_empty_backups: false\', \'    allow_object_disk_streaming: false\', \'    use_resumable_state: true\', \'    restore_schema_on_cluster: ""\', \'    upload_by_part: true\', \'    download_by_part: true\', \'    restore_database_mapping: {}\', \'    restore_table_mapping: {}\', \'    retries_on_failure: 3\', \'    retries_pause: 5s\', \'    watch_interval: 1h\', \'    full_interval: 24h\', \'    watch_backup_name_template: shard{shard}-{type}-{time:20060102150405}\', \'    sharded_operation_mode: ""\', \'    cpu_nice_priority: 15\', \'    io_nice_priority: idle\', \'    rbac_backup_always: true\', \'    rbac_conflict_resolution: recreate\', \'    retriesduration: 5s\', \'    watchduration: 1h0m0s\', \'    fullduration: 24h0m0s\', \'clickhouse:\', \'    username: default\', \'    password: ""\', \'    host: localhost\', \'    port: 9000\', \'    disk_mapping: {}\', \'    skip_tables:\', \'        - system.*\', \'        - INFORMATION_SCHEMA.*\', \'        - information_schema.*\', \'        - _temporary_and_external_tables.*\', \'    skip_table_engines: []\', \'    timeout: 30m\', \'    freeze_by_part: false\', \'    freeze_by_part_where: ""\', \'    use_embedded_backup_restore: false\', \'    embedded_backup_disk: ""\', \'    backup_mutations: true\', \'    restore_as_attach: false\', \'    check_parts_columns: true\', \'    secure: false\', \'    skip_verify: false\', \'    sync_replicated_tables: false\', \'    log_sql_queries: true\', \'    config_dir: /etc/clickhouse-server/\', \'    restart_command: exec:systemctl restart clickhouse-server\', \'    ignore_not_exists_error_during_freeze: true\', \'    check_replicas_before_attach: true\', \'    default_replica_path: /clickhouse/tables/{cluster}/{shard}/{database}/{table}\', "    default_replica_name: \'{replica}\'", \'    tls_key: ""\', \'    tls_cert: ""\', \'    tls_ca: ""\', \'    debug: false\', \'s3:\', \'    access_key: ""\', \'    secret_key: ""\', \'    bucket: ""\', \'    endpoint: ""\', \'    region: us-east-1\', \'    acl: private\', \'    assume_role_arn: ""\', \'    force_path_style: false\', \'    path: ""\', \'    object_disk_path: ""\', \'    disable_ssl: false\', \'    compression_level: 1\', \'    compression_format: tar\', \'    sse: ""\', \'    sse_kms_key_id: ""\', \'    sse_customer_algorithm: ""\', \'    sse_customer_key: ""\', \'    sse_customer_key_md5: ""\', \'    sse_kms_encryption_context: ""\', \'    disable_cert_verification: false\', \'    use_custom_storage_class: false\', \'    storage_class: STANDARD\', \'    custom_storage_class_map: {}\', \'    allow_multipart_download: false\', \'    object_labels: {}\', \'    request_payer: ""\', \'    check_sum_algorithm: ""\', \'    debug: false\', \'gcs:\', \'    credentials_file: ""\', \'    credentials_json: ""\', \'    credentials_json_encoded: ""\', \'    embedded_access_key: ""\', \'    embedded_secret_key: ""\', \'    skip_credentials: false\', \'    bucket: ""\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_level: 1\', \'    compression_format: tar\', \'    debug: false\', \'    force_http: false\', \'    endpoint: ""\', \'    storage_class: STANDARD\', \'    object_labels: {}\', \'    custom_storage_class_map: {}\', \'    chunk_size: 0\', \'cos:\', \'    url: ""\', \'    timeout: 2m\', \'    secret_id: ""\', \'    secret_key: ""\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_format: tar\', \'    compression_level: 1\', \'    debug: false\', \'api:\', \'    listen: localhost:7171\', \'    enable_metrics: true\', \'    enable_pprof: false\', \'    username: ""\', \'    password: ""\', \'    secure: false\', \'    certificate_file: ""\', \'    private_key_file: ""\', \'    ca_cert_file: ""\', \'    ca_key_file: ""\', \'    create_integration_tables: false\', \'    integration_tables_host: ""\', \'    allow_parallel: false\', \'    complete_resumable_after_restart: true\', \'    watch_is_main_process: false\', \'ftp:\', \'    address: ""\', \'    timeout: 2m\', \'    username: ""\', \'    password: ""\', \'    tls: false\', \'    skip_tls_verify: false\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_format: tar\', \'    compression_level: 1\', \'    debug: false\', \'sftp:\', \'    address: ""\', \'    port: 22\', \'    username: ""\', \'    password: ""\', \'    key: ""\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_format: tar\', \'    compression_level: 1\', \'    debug: false\', \'azblob:\', \'    endpoint_schema: https\', \'    endpoint_suffix: core.windows.net\', \'    account_name: ""\', \'    account_key: ""\', \'    sas: ""\', \'    use_managed_identity: false\', \'    container: ""\', \'    assume_container_exists: false\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_level: 1\', \'    compression_format: tar\', \'    sse_key: ""\', \'    buffer_count: 3\', \'    timeout: 4h\', \'    debug: false\', \'custom:\', \'    upload_command: ""\', \'    download_command: ""\', \'    list_command: ""\', \'    delete_command: ""\', \'    command_timeout: 4h\', \'    commandtimeoutduration: 4h0m0s\']'"""
 
 help_flag = r"""'NAME:\n   clickhouse-backup - Tool for easy backup of ClickHouse with cloud supportUSAGE:\n   clickhouse-backup <command> [-t, --tables=<db>.<table>] <backup_name>DESCRIPTION:\n   Run as \'root\' or \'clickhouse\' userCOMMANDS:\n   tables               List of tables, exclude skip_tables\n   create               Create new backup\n   create_remote        Create and upload new backup\n   upload               Upload backup to remote storage\n   list                 List of backups\n   download             Download backup from remote storage\n   restore              Create schema and restore data from backup\n   restore_remote       Download and restore\n   delete               Delete specific backup\n   default-config       Print default config\n   print-config         Print current config merged with environment variables\n   clean                Remove data in \'shadow\' folder from all \'path\' folders available from \'system.disks\'\n   clean_remote_broken  Remove all broken remote backups\n   watch                Run infinite loop which create full + incremental backup sequence to allow efficient backup sequences\n   server               Run API server\n   help, h              Shows a list of commands or help for one commandGLOBAL OPTIONS:\n   --config value, -c value                   Config \'FILE\' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG]\n   --environment-override value, --env value  override any environment variable via CLI parameter\n   --help, -h                                 show help\n   --version, -v                              print the version'"""