add support for listing system containers when listing containers (#1362)

From the documentation:
> -system: Version 2020-10-02 and later. Specifies if system containers are to be included in the response. Including this option will list system containers, such as $logs and $changefeed. Note that the specific system containers returned will vary, based on which service features are enabled on the storage account.

ref: https://learn.microsoft.com/en-us/rest/api/storageservices/list-containers2?tabs=azure-ad

Author: demoray

eng/scripts/emulator_tests.sh

@@ -5,7 +5,7 @@ cd $(dirname ${BASH_SOURCE[0]})/../../
 
 BUILD=${1:-stable}
 
-npm install azurite@3.13.1
+npm install azurite@3.26.0
 npx azurite &
 
 rustup update --no-self-update ${BUILD}

sdk/storage/src/clients.rs

@@ -22,7 +22,7 @@ pub const EMULATOR_ACCOUNT: &str = "devstoreaccount1";
 pub const EMULATOR_ACCOUNT_KEY: &str =
     "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
 
-const AZURE_VERSION: HeaderValue = HeaderValue::from_static("2019-12-12");
+const AZURE_VERSION: HeaderValue = HeaderValue::from_static("2020-10-02");
 
 #[derive(Debug, Clone, Copy)]
 pub enum ServiceType {

sdk/storage_blobs/src/service/operations/list_containers.rs

@@ -15,6 +15,7 @@ pub struct ListContainersBuilder {
     prefix: Option<Prefix>,
     include_metadata: bool,
     include_deleted: bool,
+    include_system: bool,
     max_results: Option<MaxResults>,
     context: Context,
 }
@@ -26,6 +27,7 @@ impl ListContainersBuilder {
             prefix: None,
             include_metadata: false,
             include_deleted: false,
+            include_system: false,
             max_results: None,
             context: Context::new(),
         }
@@ -35,6 +37,7 @@ impl ListContainersBuilder {
         prefix: Prefix => Some(prefix),
         include_metadata: bool => include_metadata,
         include_deleted: bool => include_deleted,
+        include_system: bool => include_system,
         max_results: MaxResults => Some(max_results),
         context: Context => context,
     }
@@ -54,13 +57,20 @@ impl ListContainersBuilder {
                     next_marker.append_to_url_query(&mut url);
                 }
 
-                if let Some(include) = match (this.include_metadata, this.include_deleted) {
-                    (true, true) => Some("metadata,deleted"),
-                    (true, false) => Some("metadata"),
-                    (false, true) => Some("deleted"),
-                    (false, false) => None,
-                } {
-                    url.query_pairs_mut().append_pair("include", include);
+                let mut to_include = vec![];
+
+                if this.include_metadata {
+                    to_include.push("metadata");
+                }
+                if this.include_deleted {
+                    to_include.push("deleted");
+                }
+                if this.include_system {
+                    to_include.push("system");
+                }
+                if !to_include.is_empty() {
+                    url.query_pairs_mut()
+                        .append_pair("include", &to_include.join(","));
                 }
                 this.max_results.append_to_url_query(&mut url);
 

test/transactions/datalake_file_create_delete/0_request.json

@@ -4,9 +4,9 @@
   "headers": {
     "authorization": "<<STRIPPED>>",
     "content-length": "0",
-    "user-agent": "azsdk-rust-storage_datalake/0.1.0 (1.58.0; linux; x86_64)",
-    "x-ms-date": "Tue, 08 Feb 2022 19:19:33 GMT",
-    "x-ms-version": "2019-12-12"
+    "user-agent": "azsdk-rust-storage/0.14.0 (1.72.0; linux; x86_64)",
+    "x-ms-date": "Thu, 07 Sep 2023 19:20:13 GMT",
+    "x-ms-version": "2020-10-02"
   },
   "body": ""
 }

test/transactions/datalake_file_create_delete/0_response.json

@@ -2,13 +2,13 @@
   "status": 201,
   "headers": {
     "content-length": "0",
-    "date": "Tue, 08 Feb 2022 19:19:34 GMT",
-    "etag": "\"0x8D9EB37F12127AD\"",
-    "last-modified": "Tue, 08 Feb 2022 19:19:35 GMT",
+    "date": "Thu, 07 Sep 2023 19:20:13 GMT",
+    "etag": "\"0x8DBAFD776AA95E2\"",
+    "last-modified": "Thu, 07 Sep 2023 19:20:14 GMT",
     "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0",
     "x-ms-namespace-enabled": "true",
-    "x-ms-request-id": "c47a931d-801f-0017-7920-1d865a000000",
-    "x-ms-version": "2019-12-12"
+    "x-ms-request-id": "e1ea9c5d-501f-001c-0ac0-e1106d000000",
+    "x-ms-version": "2020-10-02"
   },
   "body": ""
 }

test/transactions/datalake_file_create_delete/1_request.json

@@ -4,9 +4,9 @@
   "headers": {
     "authorization": "<<STRIPPED>>",
     "content-length": "0",
-    "user-agent": "azsdk-rust-storage_datalake/0.1.0 (1.58.0; linux; x86_64)",
-    "x-ms-date": "Tue, 08 Feb 2022 19:19:33 GMT",
-    "x-ms-version": "2019-12-12"
+    "user-agent": "azsdk-rust-storage/0.14.0 (1.72.0; linux; x86_64)",
+    "x-ms-date": "Thu, 07 Sep 2023 19:20:13 GMT",
+    "x-ms-version": "2020-10-02"
   },
   "body": ""
 }

test/transactions/datalake_file_create_delete/1_response.json

@@ -2,13 +2,13 @@
   "status": 201,
   "headers": {
     "content-length": "0",
-    "date": "Tue, 08 Feb 2022 19:19:34 GMT",
-    "etag": "\"0x8D9EB37F16507E0\"",
-    "last-modified": "Tue, 08 Feb 2022 19:19:35 GMT",
+    "date": "Thu, 07 Sep 2023 19:20:13 GMT",
+    "etag": "\"0x8DBAFD776B883D6\"",
+    "last-modified": "Thu, 07 Sep 2023 19:20:14 GMT",
     "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0",
-    "x-ms-request-id": "c47a931e-801f-0017-7a20-1d865a000000",
+    "x-ms-request-id": "e1ea9c60-501f-001c-0dc0-e1106d000000",
     "x-ms-request-server-encrypted": "true",
-    "x-ms-version": "2019-12-12"
+    "x-ms-version": "2020-10-02"
   },
   "body": ""
 }

test/transactions/datalake_file_create_delete/2_request.json

@@ -5,9 +5,9 @@
     "authorization": "<<STRIPPED>>",
     "content-length": "0",
     "if-none-match": "*",
-    "user-agent": "azsdk-rust-storage_datalake/0.1.0 (1.58.0; linux; x86_64)",
-    "x-ms-date": "Tue, 08 Feb 2022 19:19:34 GMT",
-    "x-ms-version": "2019-12-12"
+    "user-agent": "azsdk-rust-storage/0.14.0 (1.72.0; linux; x86_64)",
+    "x-ms-date": "Thu, 07 Sep 2023 19:20:13 GMT",
+    "x-ms-version": "2020-10-02"
   },
   "body": ""
 }

test/transactions/datalake_file_create_delete/2_response.json

@@ -3,11 +3,11 @@
   "headers": {
     "content-length": "168",
     "content-type": "application/json;charset=utf-8",
-    "date": "Tue, 08 Feb 2022 19:19:34 GMT",
+    "date": "Thu, 07 Sep 2023 19:20:13 GMT",
     "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0",
     "x-ms-error-code": "PathAlreadyExists",
-    "x-ms-request-id": "c47a931f-801f-0017-7b20-1d865a000000",
-    "x-ms-version": "2019-12-12"
+    "x-ms-request-id": "e1ea9c65-501f-001c-12c0-e1106d000000",
+    "x-ms-version": "2020-10-02"
   },
-  "body": "eyJlcnJvciI6eyJjb2RlIjoiUGF0aEFscmVhZHlFeGlzdHMiLCJtZXNzYWdlIjoiVGhlIHNwZWNpZmllZCBwYXRoIGFscmVhZHkgZXhpc3RzLlxuUmVxdWVzdElkOmM0N2E5MzFmLTgwMWYtMDAxNy03YjIwLTFkODY1YTAwMDAwMFxuVGltZToyMDIyLTAyLTA4VDE5OjE5OjM1Ljc4MzQxMDJaIn19"
+  "body": "eyJlcnJvciI6eyJjb2RlIjoiUGF0aEFscmVhZHlFeGlzdHMiLCJtZXNzYWdlIjoiVGhlIHNwZWNpZmllZCBwYXRoIGFscmVhZHkgZXhpc3RzLlxuUmVxdWVzdElkOmUxZWE5YzY1LTUwMWYtMDAxYy0xMmMwLWUxMTA2ZDAwMDAwMFxuVGltZToyMDIzLTA5LTA3VDE5OjIwOjE0Ljc5OTY4OTdaIn19"
 }

test/transactions/datalake_file_create_delete/3_request.json

@@ -4,9 +4,9 @@
   "headers": {
     "authorization": "<<STRIPPED>>",
     "content-length": "0",
-    "user-agent": "azsdk-rust-storage_datalake/0.1.0 (1.58.0; linux; x86_64)",
-    "x-ms-date": "Tue, 08 Feb 2022 19:19:34 GMT",
-    "x-ms-version": "2019-12-12"
+    "user-agent": "azsdk-rust-storage/0.14.0 (1.72.0; linux; x86_64)",
+    "x-ms-date": "Thu, 07 Sep 2023 19:20:13 GMT",
+    "x-ms-version": "2020-10-02"
   },
   "body": ""
 }