Resolve conflicts with main

Author: jaidisido

.bumpversion.cfg

@@ -33,7 +33,7 @@ first_value = 1
 
 [bumpversion:file:docs/source/layers.rst]
 
-[bumpversion:file:docs/source/what.rst]
+[bumpversion:file:docs/source/about.rst]
 
 [bumpversion:file:awswrangler/__metadata__.py]
 

.github/workflows/bandit.yml

@@ -17,9 +17,9 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install

.github/workflows/cfn-nag.yml

@@ -16,17 +16,21 @@ on:
 permissions:
   contents: read
 
+env:
+  CDK_DEFAULT_ACCOUNT: 111111111111
+  CDK_DEFAULT_REGION: us-east-1
+
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Use Node.js
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
         with:
-          node-version: '14.x'
+          node-version: 16
       - name: Cache Node.js modules
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.npm
           key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -37,16 +41,28 @@ jobs:
         run: |
           npm install -g aws-cdk
           cdk --version
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install Requirements
         run: |
-          cd test_infra && rm -rf poetry.lock
+          cd test_infra
+          cat <<EOT >> cdk.context.json
+          {
+            "availability-zones:account=111111111111:region=us-east-1": [
+              "us-east-1a",
+              "us-east-1b",
+              "us-east-1c",
+              "us-east-1d",
+              "us-east-1e",
+              "us-east-1f"
+            ]
+          }
+          EOT
           python -m pip install --upgrade pip
-          python -m pip install poetry
+          python -m pip install poetry==1.1.15 # 1.2.0 breaking resolution of packages
           poetry config virtualenvs.create false --local
           poetry install -vvv
       - name: CDK Synth

.github/workflows/minimal-tests.yml

@@ -24,9 +24,9 @@ jobs:
         python-version: [3.8]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install Requirements

.github/workflows/static-checking.yml

@@ -23,9 +23,9 @@ jobs:
         python-version: [3.8]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install Requirements

CONTRIBUTING_COMMON_ERRORS.md

@@ -111,22 +111,3 @@ brew install unixodbc
 ```
 
 -----
-
-## CloudFormation Deployment
-
-### Error Message
-
-During the deployment of `aws-sdk-pandas-databases`, the creation of the resource `CodeBuildTestRoleLFPermissions` fails with
-
-```
-Resource does not exist or requester is not authorized to access requested permissions. (Service: AWSLakeFormation; Status Code: 400; Error Code: AccessDeniedException; Request ID: 14a26718-ee4e-49f2-a7ca-d308e49485f8; Proxy: null)
-```
-
-### Solution
-
-The IAM role used to deploy the CloudForation stack does not have permissions to assign permissions in AWS Lake Formation. The quickest solution is to find the IAM role and set it as an admin in Lake Formation.
-
-In order to find the role:
-1. Navigate to the CloudFormation console in your account
-1. Select the `aws-sdk-pandas-databases` stack which failed to deploy
-1. Under the "Stack info" tab, find the value for "IAM role". The name of the role should be in the following format: `arn:aws:iam::{ACCOUNT_ID}:role/cdk-{UUID}-cfn-exec-role-{ACCOUNT_ID}-{REGION}`

awswrangler/_data_types.py

@@ -360,6 +360,8 @@ def athena2pandas(dtype: str) -> str:  # pylint: disable=too-many-branches,too-m
         return "decimal"
     if dtype in ("binary", "varbinary"):
         return "bytes"
+    if dtype in ("array", "row", "map"):
+        return "object"
     raise exceptions.UnsupportedType(f"Unsupported Athena type: {dtype}")
 
 

awswrangler/athena/_utils.py

@@ -211,16 +211,6 @@ def _get_query_metadata(  # pylint: disable=too-many-statements
     col_name: str
     col_type: str
     for col_name, col_type in cols_types.items():
-        if col_type == "array":
-            raise exceptions.UnsupportedType(
-                "List data type is not supported with regular (non-CTAS and non-UNLOAD) queries. "
-                "Please use ctas_approach=True or unload_approach=True for List columns."
-            )
-        if col_type == "row":
-            raise exceptions.UnsupportedType(
-                "Struct data type is not supported with regular (non-CTAS and non-UNLOAD) queries. "
-                "Please use ctas_approach=True or unload_approach=True for Struct columns."
-            )
         pandas_type: str = _data_types.athena2pandas(dtype=col_type)
         if (categories is not None) and (col_name in categories):
             dtype[col_name] = "category"

awswrangler/data_api/redshift.py

@@ -13,12 +13,19 @@
 class RedshiftDataApi(connector.DataApiConnector):
     """Provides access to a Redshift cluster via the Data API.
 
+    Note
+    ----
+    When connecting to a standard Redshift cluster, `cluster_id` is used.
+    When connecting to Redshift Serverless, `workgroup_name` is used. These two arguments are mutually exclusive.
+
     Parameters
     ----------
     cluster_id: str
-        Id for the target Redshift cluster.
+        Id for the target Redshift cluster - only required if `workgroup_name` not provided.
     database: str
         Target database name.
+    workgroup_name: str
+        Name for the target serverless Redshift workgroup - only required if `cluster_id` not provided.
     secret_arn: str
         The ARN for the secret to be used for authentication - only required if `db_user` not provided.
     db_user: str
@@ -35,8 +42,9 @@ class RedshiftDataApi(connector.DataApiConnector):
 
     def __init__(
         self,
-        cluster_id: str,
-        database: str,
+        cluster_id: str = "",
+        database: str = "",
+        workgroup_name: str = "",
         secret_arn: str = "",
         db_user: str = "",
         sleep: float = 0.25,
@@ -46,29 +54,44 @@ def __init__(
     ) -> None:
         self.cluster_id = cluster_id
         self.database = database
+        self.workgroup_name = workgroup_name
         self.secret_arn = secret_arn
         self.db_user = db_user
         self.client: boto3.client = _utils.client(service_name="redshift-data", session=boto3_session)
         self.waiter = RedshiftDataApiWaiter(self.client, sleep, backoff, retries)
         logger: logging.Logger = logging.getLogger(__name__)
         super().__init__(self.client, logger)
 
+    def _validate_redshift_target(self) -> None:
+        if self.database == "":
+            raise ValueError("`database` must be set for connection")
+        if self.cluster_id == "" and self.workgroup_name == "":
+            raise ValueError("Either `cluster_id` or `workgroup_name`(Redshift Serverless) must be set for connection")
+
     def _validate_auth_method(self) -> None:
-        if self.secret_arn == "" and self.db_user == "":
+        if self.workgroup_name == "" and self.secret_arn == "" and self.db_user == "":
             raise ValueError("Either `secret_arn` or `db_user` must be set for authentication")
 
     def _execute_statement(self, sql: str, database: Optional[str] = None) -> str:
+        self._validate_redshift_target()
         self._validate_auth_method()
-        credentials = {"SecretArn": self.secret_arn}
-        if self.db_user:
+        credentials = {}
+        if self.secret_arn:
+            credentials = {"SecretArn": self.secret_arn}
+        elif self.db_user:
             credentials = {"DbUser": self.db_user}
 
         if database is None:
             database = self.database
 
+        if self.cluster_id:
+            redshift_target = {"ClusterIdentifier": self.cluster_id}
+        elif self.workgroup_name:
+            redshift_target = {"WorkgroupName": self.workgroup_name}
+
         self.logger.debug("Executing %s", sql)
         response: Dict[str, Any] = self.client.execute_statement(
-            ClusterIdentifier=self.cluster_id,
+            **redshift_target,
             Database=database,
             Sql=sql,
             **credentials,
@@ -167,21 +190,29 @@ class RedshiftDataApiTimeoutException(Exception):
 
 
 def connect(
-    cluster_id: str,
-    database: str,
+    cluster_id: str = "",
+    database: str = "",
+    workgroup_name: str = "",
     secret_arn: str = "",
     db_user: str = "",
     boto3_session: Optional[boto3.Session] = None,
     **kwargs: Any,
 ) -> RedshiftDataApi:
     """Create a Redshift Data API connection.
 
+    Note
+    ----
+    When connecting to a standard Redshift cluster, `cluster_id` is used.
+    When connecting to Redshift Serverless, `workgroup_name` is used. These two arguments are mutually exclusive.
+
     Parameters
     ----------
     cluster_id: str
-        Id for the target Redshift cluster.
+        Id for the target Redshift cluster - only required if `workgroup_name` not provided.
     database: str
         Target database name.
+    workgroup_name: str
+        Name for the target serverless Redshift workgroup - only required if `cluster_id` not provided.
     secret_arn: str
         The ARN for the secret to be used for authentication - only required if `db_user` not provided.
     db_user: str
@@ -196,7 +227,13 @@ def connect(
     A RedshiftDataApi connection instance that can be used with `wr.redshift.data_api.read_sql_query`.
     """
     return RedshiftDataApi(
-        cluster_id, database, secret_arn=secret_arn, db_user=db_user, boto3_session=boto3_session, **kwargs
+        cluster_id=cluster_id,
+        database=database,
+        workgroup_name=workgroup_name,
+        secret_arn=secret_arn,
+        db_user=db_user,
+        boto3_session=boto3_session,
+        **kwargs,
     )
 
 

awswrangler/opensearch/_utils.py

@@ -33,6 +33,10 @@ def _strip_endpoint(endpoint: str) -> str:
     return uri_schema.sub("", endpoint).strip().strip("/")
 
 
+def _is_https(port: int) -> bool:
+    return port == 443
+
+
 def connect(
     host: str,
     port: Optional[int] = 443,
@@ -95,8 +99,8 @@ def connect(
             host=_strip_endpoint(host),
             port=port,
             http_auth=http_auth,
-            use_ssl=True,
-            verify_certs=True,
+            use_ssl=_is_https(port),
+            verify_certs=_is_https(port),
             connection_class=RequestsHttpConnection,
             timeout=30,
             max_retries=10,