lots of fixes

Author: daimor

dbt/adapters/iris/column.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass
 from typing import TypeVar, Optional, ClassVar, Dict, Any
-
+from dbt_common.exceptions import DbtRuntimeError
 from dbt.adapters.base.column import Column
 
 Self = TypeVar("Self", bound="IRISColumn")
@@ -9,10 +9,8 @@
 @dataclass
 class IRISColumn(Column):
     TYPE_LABELS: ClassVar[Dict[str, str]] = {
-        "STRING": "VARCHAR('')",
-        "TIMESTAMP": "TIMESTAMP",
-        "FLOAT": "FLOAT",
-        "INTEGER": "INTEGER",
+        "STRING": "VARCHAR(65535)",
+        "FLOAT": "DOUBLE",
         "BOOLEAN": "BIT",
     }
 
@@ -30,3 +28,17 @@ def quoted(self) -> str:
 
     def __repr__(self) -> str:
         return "<IRISColumn {} ({})>".format(self.name, self.data_type)
+
+    # def string_size(self) -> int:
+    #     if not self.is_string():
+    #         raise DbtRuntimeError("Called string_size() on non-string field!")
+
+    #     if self.dtype == "text" or not self.char_size:
+    #         return 65535
+    #     else:
+    #         return int(self.char_size)
+
+    # @property
+    # def data_type(self):
+    #     print("!!!! data_type", self.dtype, self.char_size)
+    #     return super().data_type

dbt/adapters/iris/connections.py

@@ -1,5 +1,5 @@
 import time
-from typing import Optional, Tuple, Any, Type
+from typing import Optional, Tuple, Any, Type, Union
 from contextlib import contextmanager
 from dataclasses import dataclass
 from dbt.exceptions import DbtRuntimeError
@@ -11,6 +11,7 @@
 from dbt_common.events.functions import fire_event
 from dbt.adapters.events.types import ConnectionUsed, SQLQuery, SQLQueryStatus
 import intersystems_iris.dbapi._DBAPI as dbapi
+from intersystems_iris.dbapi._SQLType import SQLType
 from dbt_common.helper_types import Port
 from mashumaro.jsonschema.annotations import Maximum, Minimum
 from typing_extensions import Annotated
@@ -183,3 +184,9 @@ def add_query(
             )
 
             return connection, cursor
+
+    @classmethod
+    def data_type_code_to_name(cls, type_code: Union[int, str]) -> str:
+        if isinstance(type_code, int):
+            return SQLType(type_code).name
+        return f"unknown type_code {type_code}"

dbt/adapters/iris/impl.py

@@ -36,10 +36,11 @@ def date_function(cls):
 
     @classmethod
     def convert_text_type(cls, agate_table, col_idx):
+        min_length = 65535
         column = agate_table.columns[col_idx]
         lengths = [len(d.encode("utf-8")) for d in column.values_without_nulls()]
-        max_len = max(lengths) if lengths else 64
-        length = max_len if max_len > 16 else 16
+        max_len = max(lengths) if lengths else min_length
+        length = max_len if max_len > min_length else min_length
         return "varchar({})".format(length)
 
     @classmethod
@@ -55,7 +56,9 @@ def convert_number_type(cls, agate_table, col_idx):
     def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str:
         return "datetime"
 
-    def timestamp_add_sql(self, add_to: str, number: int = 1, interval: str = "hour") -> str:
+    def timestamp_add_sql(
+        self, add_to: str, number: int = 1, interval: str = "hour"
+    ) -> str:
         return f"dateadd('{interval}', {number}, {add_to})"
 
     def create_schema(self, schema_relation: BaseRelation) -> None:
@@ -72,7 +75,9 @@ def drop_schema(self, schema_relation) -> None:
             raise
 
     def check_schema_exists(self, database, schema):
-        results = self.execute_macro(LIST_SCHEMAS_MACRO_NAME, kwargs={"database": database})
+        results = self.execute_macro(
+            LIST_SCHEMAS_MACRO_NAME, kwargs={"database": database}
+        )
 
         exists = True if schema in [row[0] for row in results] else False
         return exists
@@ -140,12 +145,16 @@ def get_missing_columns(
             col.name.lower(): col for col in self.get_columns_in_relation(from_relation)
         }
 
-        to_columns = {col.name.lower(): col for col in self.get_columns_in_relation(to_relation)}
+        to_columns = {
+            col.name.lower(): col for col in self.get_columns_in_relation(to_relation)
+        }
 
         missing_columns = set(from_columns.keys()) - set(to_columns.keys())
 
         return [
-            col for (col_name, col) in from_columns.items() if col_name.lower() in missing_columns
+            col
+            for (col_name, col) in from_columns.items()
+            if col_name.lower() in missing_columns
         ]
 
     def submit_python_job(self, parsed_model: dict, compiled_code: str):
@@ -166,9 +175,32 @@ def submit_python_job(self, parsed_model: dict, compiled_code: str):
 """
 
         response, _ = self.execute(create_procedure, auto_begin=False, fetch=False)
-        response, _ = self.execute(f"SELECT {proc_name}()", auto_begin=False, fetch=True)
-        self.execute(f"drop procedure if exists {proc_name}", auto_begin=False, fetch=False)
+        response, _ = self.execute(
+            f"SELECT {proc_name}()", auto_begin=False, fetch=True
+        )
+        self.execute(
+            f"drop procedure if exists {proc_name}", auto_begin=False, fetch=False
+        )
         return response
 
     def valid_incremental_strategies(self) -> List[str]:
         return ["append", "merge", "delete+insert"]
+
+    def drop_relation(self, relation: BaseRelation):
+        # if relation.type == "table":
+        #     print("!!!! Dropping table with temporary property", relation)
+        #     tmp_prop = "__dbt_tmp"
+        #     try:
+        #         self.execute(
+        #             f'alter table {relation} add "{tmp_prop}" bit calculated',
+        #             auto_begin=False,
+        #             fetch=False,
+        #         )
+        #     except Exception as ex:
+        #         print(
+        #             "!!!! Failed to add temporary property, continuing anyway",
+        #             relation,
+        #             ex,
+        #         )
+
+        super().drop_relation(relation)

dbt/adapters/iris/relation.py

@@ -28,6 +28,7 @@ class IRISRelation(BaseRelation):
         default_factory=lambda: frozenset(
             {
                 RelationType.Table,
+                RelationType.View,
             }
         )
     )
@@ -43,6 +44,9 @@ class IRISRelation(BaseRelation):
     quote_policy: Policy = field(default_factory=lambda: IRISQuotePolicy())
     include_policy: Policy = field(default_factory=lambda: IRISIncludePolicy())
 
+    def __repr__(self) -> str:
+        return "<{} {} {}>".format(self.__class__.__name__, self.type, self.render())
+
     def matches(
         self,
         database: Optional[str] = None,

dbt/include/iris/macros/adapters.sql

@@ -7,10 +7,6 @@ dbt docs: https://docs.getdbt.com/docs/contributing/building-a-new-adapter
   {{ return('?') }}
 {% endmacro %}
 
-{% macro iris__current_timestamp() -%}
-  current_timestamp
-{%- endmacro %}
-
 {% macro iris__list_schemas(database) -%}
     {% call statement('list_schemas', fetch_result=True, auto_begin=False) -%}
         select schema_name
@@ -95,7 +91,7 @@ dbt docs: https://docs.getdbt.com/docs/contributing/building-a-new-adapter
       select
           column_name,
           data_type,
-          character_maximum_length,
+          NULLIF(CAST(character_maximum_length as INT),0) character_maximum_length,
           numeric_precision,
           numeric_scale
       from information_schema.columns
@@ -108,18 +104,6 @@ dbt docs: https://docs.getdbt.com/docs/contributing/building-a-new-adapter
   {{ return(sql_convert_columns_in_relation(table)) }}
 {% endmacro %}
 
-
-{% macro iris__create_view_as(relation, sql) -%}
-  {%- set sql_header = config.get('sql_header', none) -%}
-
-  {{ sql_header if sql_header is not none }}
-  /* create_view_as */
-  {# create or replace view {{ relation }} #}
-  create table {{ relation }}
-  as {{ sql }}
-
-{%- endmacro %}
-
 {% macro iris__create_table_as(temporary, relation, compiled_code, language='sql') -%}
   {%- if language == 'sql' -%}
     {%- set sql_header = config.get('sql_header', none) -%}
@@ -130,11 +114,7 @@ dbt docs: https://docs.getdbt.com/docs/contributing/building-a-new-adapter
     {%- endif %}
     /* create_table_as */
     {{ sql_header if sql_header is not none }}
-    create
-    /* {% if temporary: -%}global temporary{%- endif %} */
-    table
-      {{ relation }}
-    as
+    create {% if temporary: -%}global temporary{%- endif %} table {{ relation }} as
       {{ compiled_code }}
   {%- elif language == 'python' -%}
     {{ py_write_table(compiled_code=compiled_code, target_relation=relation, temporary=temporary) }}
@@ -144,11 +124,56 @@ dbt docs: https://docs.getdbt.com/docs/contributing/building-a-new-adapter
 {%- endmacro %}
 
 {% macro iris__rename_relation(from_relation, to_relation) -%}
-  {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}
   {% call statement('drop_relation') %}
-    drop table if exists {{ to_relation }} cascade
+    drop {{ to_relation.type }} if exists {{ to_relation }} cascade
+  {% endcall %}
+  {% if not from_relation.type %}
+    {% do exceptions.raise_database_error("Cannot rename a relation with a blank type: " ~ from_relation.identifier) %}
+  {% elif from_relation.type == 'table' %}
+    {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}
+    {% do drop_related_view(from_relation) %}
+    {%- set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}
+    {% call statement('rename_relation') -%}
+      alter {{ from_relation.type }} {{ from_relation }} rename {{ target_name }}
+    {%- endcall %}
+  {%- elif from_relation.type == 'view' -%}
+    {% do adapter.dispatch('rename_view')(from_relation, to_relation) %}
+  {% else -%}
+    {% do exceptions.raise_database_error("Unknown type '" ~ from_relation.type ~ "' for relation: " ~ from_relation.identifier) %}
+  {% endif %}
+
+{% endmacro %}
+
+{% macro drop_related_view(relation) %}
+
+  {% set to_drop = get_related_views(relation) %}
+
+  {% if to_drop is not none and to_drop|length > 0 %}
+    {% for view in to_drop %}
+      {% set view_relation = api.Relation.create(
+          identifier=view['VIEW_NAME'],
+          schema=view['VIEW_SCHEMA'],
+          database=relation.database,
+          type='view') %}
+      {{ drop_relation_if_exists(view_relation) }}
+    {% endfor %}
+  {% endif %}
+
+{% endmacro %}
+
+{% macro get_related_views(relation) %}
+  {% call statement('list_tables', fetch_result=True) %}
+    SELECT VIEW_SCHEMA,VIEW_NAME
+    FROM INFORMATION_SCHEMA.VIEW_TABLE_USAGE
+    WHERE TABLE_SCHEMA='{{ relation.schema }}' AND TABLE_NAME='{{ relation.identifier }}'
   {% endcall %}
+  {{ return(load_result('list_tables').table) }}
+{% endmacro %}
+
+{# {% macro iris__rename_relation(from_relation, to_relation) -%}
+  {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}
+  {{ print("!!!! rename_relation: " ~ from_relation.type ~ ":" ~ from_relation.identifier ~ " to " ~ to_relation.type ~ ":" ~ to_relation) }}
   {% call statement('rename_relation') -%}
-    alter table {{ from_relation }} rename {{ target_name }}
+    alter table {{ from_relation.render() }} rename {{ target_name }}
   {%- endcall %}
-{% endmacro %}
+{% endmacro %} #}

dbt/include/iris/macros/adapters/columns.sql

@@ -8,3 +8,40 @@
 
     {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}
 {% endmacro %}
+
+{% macro iris__get_empty_subquery_sql(select_sql, select_sql_header=none) %}
+    {%- if select_sql_header is not none -%}
+    {{ select_sql_header }}
+    {%- endif -%}
+    select * from (
+        {{ select_sql }}
+    ) as __dbt_sbq
+    where 1=0
+    limit 0
+{% endmacro %}
+
+{% macro iris__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}
+
+  {% if add_columns %}
+
+    {% for column in add_columns %}
+      {% set sql -%}
+          alter {{ relation.type }} {{ relation }} add column {{ column.name }} {{ column.data_type }}
+      {% endset %}
+      {% do run_query(sql) %}
+    {% endfor %}
+
+  {% endif %}
+
+  {% if remove_columns %}
+
+    {% for column in remove_columns %}
+      {% set sql -%}
+          alter {{ relation.type }} {{ relation }} drop column {{ column.name }}
+      {% endset %}
+      {% do run_query(sql) %}
+    {% endfor %}
+
+  {% endif %}
+
+{% endmacro %}

dbt/include/iris/macros/datatypes.sql

@@ -1,3 +1,11 @@
-{%- macro iris__type_string() -%}
-    VARCHAR('')
+{%- macro iris__type_float() -%}
+    DOUBLE
+{%- endmacro -%}
+
+{%- macro iris__type_numeric() -%}
+    NUMERIC
+{%- endmacro -%}
+
+{%- macro iris__type_int() -%}
+    INTEGER
 {%- endmacro -%}