Skip to content

Commit dd70f22

Browse files
ryantimjohnryantimjohn
authored
Merge pull request #208 from bluelabsio/RM-39-Bump-SQLAlchemy-to-version-1.4
RM-39 update to sqlalchemy 1.4
2 parents cd80d53 + 5393016 commit dd70f22

File tree

9 files changed

+78
-54
lines changed

9 files changed

+78
-54
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,3 +73,6 @@ target/
7373

7474
# PyCharm
7575
.idea/
76+
77+
# VSCode
78+
.vscode/settings.json

‎metrics/coverage_high_water_mark‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
93.5400
1+
93.5000

‎metrics/mypy_high_water_mark‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
92.3700
1+
92.3500

‎records_mover/records/sources/table.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -77,9 +77,12 @@ def to_dataframes_source(self,
7777
db = self.driver.db
7878
records_schema = self.pull_records_schema()
7979

80-
columns = db.dialect.get_columns(db,
81-
self.table_name,
82-
schema=self.schema_name)
80+
if isinstance(db, Engine):
81+
connection = db.connect()
82+
columns = db.dialect.get_columns(connection, self.table_name, schema=self.schema_name)
83+
connection.close()
84+
else:
85+
columns = db.dialect.get_columns(db, self.table_name, schema=self.schema_name)
8386

8487
num_columns = len(columns)
8588
if num_columns == 0:

‎setup.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ def initialize_options(self) -> None:
144144
]
145145

146146
db_dependencies = [
147-
'sqlalchemy>=1.3.18,<1.4',
147+
'sqlalchemy>=1.4,<2.0',
148148
]
149149

150150
smart_open_dependencies = [

‎tests/integration/records/expected_column_types.py‎

Lines changed: 29 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -9,32 +9,32 @@
99
'redshift': [
1010
'INTEGER', 'VARCHAR(3)', 'VARCHAR(3)', 'VARCHAR(1)', 'VARCHAR(1)',
1111
'VARCHAR(3)', 'VARCHAR(111)', 'DATE', 'VARCHAR(8)',
12-
'TIMESTAMP WITHOUT TIME ZONE', 'TIMESTAMPTZ'
12+
'TIMESTAMP', 'TIMESTAMPTZ'
1313
],
1414
'postgresql': [
1515
'INTEGER', 'VARCHAR(3)', 'VARCHAR(3)', 'VARCHAR(1)', 'VARCHAR(1)',
16-
'VARCHAR(3)', 'VARCHAR(111)', 'DATE', 'TIME WITHOUT TIME ZONE',
17-
'TIMESTAMP WITHOUT TIME ZONE', 'TIMESTAMP WITH TIME ZONE'
16+
'VARCHAR(3)', 'VARCHAR(111)', 'DATE', 'TIME',
17+
'TIMESTAMP', 'TIMESTAMP'
1818
],
1919
'bigquery': [
2020
'INTEGER', 'VARCHAR(3)', 'VARCHAR(3)', 'VARCHAR(1)', 'VARCHAR(1)', 'VARCHAR(3)',
2121
'VARCHAR(111)', 'DATE', 'TIME', 'DATETIME', 'TIMESTAMP'
2222
],
2323
'mysql': [
24-
'INTEGER(11)', 'VARCHAR(3)', 'VARCHAR(3)', 'VARCHAR(1)', 'VARCHAR(1)', 'VARCHAR(3)',
25-
'VARCHAR(111)', 'DATE', 'TIME', 'DATETIME(6)', 'DATETIME(6)'
24+
'INTEGER', 'VARCHAR(3)', 'VARCHAR(3)', 'VARCHAR(1)', 'VARCHAR(1)', 'VARCHAR(3)',
25+
'VARCHAR(111)', 'DATE', 'TIME', 'DATETIME', 'DATETIME'
2626
],
2727
}
2828

2929
expected_df_loaded_database_column_types = {
3030
'postgresql': [
3131
'BIGINT', 'VARCHAR(12)', 'VARCHAR(12)', 'VARCHAR(4)', 'VARCHAR(4)',
32-
'VARCHAR(12)', 'VARCHAR(444)', 'DATE', 'TIME WITHOUT TIME ZONE',
33-
'TIMESTAMP WITHOUT TIME ZONE', 'TIMESTAMP WITH TIME ZONE'
32+
'VARCHAR(12)', 'VARCHAR(444)', 'DATE', 'TIME',
33+
'TIMESTAMP', 'TIMESTAMP'
3434
],
3535
'mysql': [
36-
'BIGINT(20)', 'VARCHAR(3)', 'VARCHAR(3)', 'VARCHAR(1)', 'VARCHAR(1)', 'VARCHAR(3)',
37-
'VARCHAR(111)', 'DATE', 'TIME', 'DATETIME(6)', 'DATETIME(6)'
36+
'BIGINT', 'VARCHAR(3)', 'VARCHAR(3)', 'VARCHAR(1)', 'VARCHAR(1)', 'VARCHAR(3)',
37+
'VARCHAR(111)', 'DATE', 'TIME', 'DATETIME', 'DATETIME'
3838
],
3939
'vertica': [
4040
'INTEGER', 'VARCHAR(12)', 'VARCHAR(12)', 'VARCHAR(4)', 'VARCHAR(4)',
@@ -44,7 +44,7 @@
4444
'redshift': [
4545
'BIGINT', 'VARCHAR(12)', 'VARCHAR(12)', 'VARCHAR(4)', 'VARCHAR(4)',
4646
'VARCHAR(12)', 'VARCHAR(444)', 'DATE', 'VARCHAR(8)',
47-
'TIMESTAMP WITHOUT TIME ZONE', 'TIMESTAMPTZ'
47+
'TIMESTAMP', 'TIMESTAMPTZ'
4848
],
4949
'bigquery': [
5050
'INTEGER', 'VARCHAR(12)', 'VARCHAR(12)', 'VARCHAR(4)', 'VARCHAR(4)',
@@ -116,8 +116,8 @@
116116
# date/time/timestamp/timestamptz all get turned into strings.
117117
('postgresql', 'postgresql'): [
118118
'INTEGER', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)',
119-
'VARCHAR(256)', 'VARCHAR(256)', 'DATE', 'TIME WITHOUT TIME ZONE',
120-
'TIMESTAMP WITHOUT TIME ZONE', 'TIMESTAMP WITH TIME ZONE'
119+
'VARCHAR(256)', 'VARCHAR(256)', 'DATE', 'TIME',
120+
'TIMESTAMP', 'TIMESTAMP'
121121
],
122122
('postgresql', 'vertica'): [
123123
'INTEGER', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)',
@@ -127,7 +127,7 @@
127127
('postgresql', 'redshift'): [
128128
'INTEGER', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)',
129129
'VARCHAR(256)', 'VARCHAR(256)', 'DATE', 'VARCHAR(8)',
130-
'TIMESTAMP WITHOUT TIME ZONE', 'TIMESTAMPTZ'
130+
'TIMESTAMP', 'TIMESTAMPTZ'
131131
],
132132
('postgresql', 'bigquery'): [
133133
'INTEGER', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)',
@@ -141,7 +141,7 @@
141141
('redshift', 'postgresql'): [
142142
'INTEGER', 'VARCHAR(3)', 'VARCHAR(3)', 'VARCHAR(1)', 'VARCHAR(1)',
143143
'VARCHAR(3)', 'VARCHAR(111)', 'DATE', 'VARCHAR(8)',
144-
'TIMESTAMP WITHOUT TIME ZONE', 'TIMESTAMP WITH TIME ZONE'
144+
'TIMESTAMP', 'TIMESTAMP'
145145
],
146146
('bigquery', 'redshift'): [
147147
'BIGINT', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)',
@@ -154,8 +154,8 @@
154154
],
155155
('bigquery', 'postgresql'): [
156156
'BIGINT', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)',
157-
'VARCHAR(256)', 'VARCHAR(256)', 'DATE', 'TIME WITHOUT TIME ZONE',
158-
'TIMESTAMP WITHOUT TIME ZONE', 'TIMESTAMP WITH TIME ZONE'
157+
'VARCHAR(256)', 'VARCHAR(256)', 'DATE', 'TIME',
158+
'TIMESTAMP', 'TIMESTAMP'
159159
],
160160
('bigquery', 'vertica'): [
161161
'INTEGER', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)',
@@ -170,37 +170,37 @@
170170
'VARCHAR(444)', 'DATE', 'TIME', 'DATETIME', 'DATETIME',
171171
],
172172
('redshift', 'mysql'): [
173-
'INTEGER(11)', 'VARCHAR(3)', 'VARCHAR(3)', 'VARCHAR(1)', 'VARCHAR(1)', 'VARCHAR(3)',
174-
'VARCHAR(111)', 'DATE', 'VARCHAR(8)', 'DATETIME(6)', 'DATETIME(6)'
173+
'INTEGER', 'VARCHAR(3)', 'VARCHAR(3)', 'VARCHAR(1)', 'VARCHAR(1)', 'VARCHAR(3)',
174+
'VARCHAR(111)', 'DATE', 'VARCHAR(8)', 'DATETIME', 'DATETIME'
175175
],
176176
('postgresql', 'mysql'): [
177-
'INTEGER(11)', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)',
177+
'INTEGER', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)',
178178
'VARCHAR(256)',
179-
'VARCHAR(256)', 'DATE', 'TIME', 'DATETIME(6)', 'DATETIME(6)'
179+
'VARCHAR(256)', 'DATE', 'TIME', 'DATETIME', 'DATETIME'
180180
],
181181
('bigquery', 'mysql'): [
182-
'BIGINT(20)', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)',
182+
'BIGINT', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)', 'VARCHAR(256)',
183183
'VARCHAR(256)',
184-
'VARCHAR(256)', 'DATE', 'TIME', 'DATETIME(6)', 'DATETIME(6)'
184+
'VARCHAR(256)', 'DATE', 'TIME', 'DATETIME', 'DATETIME'
185185
],
186186
('mysql', 'postgresql'): [
187187
'INTEGER', 'VARCHAR(12)', 'VARCHAR(12)', 'VARCHAR(4)', 'VARCHAR(4)', 'VARCHAR(12)',
188-
'VARCHAR(444)', 'DATE', 'TIME WITHOUT TIME ZONE', 'TIMESTAMP WITHOUT TIME ZONE',
189-
'TIMESTAMP WITHOUT TIME ZONE'
188+
'VARCHAR(444)', 'DATE', 'TIME', 'TIMESTAMP',
189+
'TIMESTAMP'
190190
],
191191
('mysql', 'redshift'): [
192192
'INTEGER', 'VARCHAR(12)', 'VARCHAR(12)', 'VARCHAR(4)', 'VARCHAR(4)', 'VARCHAR(12)',
193-
'VARCHAR(444)', 'DATE', 'VARCHAR(8)', 'TIMESTAMP WITHOUT TIME ZONE',
194-
'TIMESTAMP WITHOUT TIME ZONE'
193+
'VARCHAR(444)', 'DATE', 'VARCHAR(8)', 'TIMESTAMP',
194+
'TIMESTAMP'
195195
],
196196
('vertica', 'postgresql'): [
197197
'BIGINT', 'VARCHAR(3)', 'VARCHAR(3)', 'VARCHAR(1)', 'VARCHAR(1)',
198-
'VARCHAR(3)', 'VARCHAR(111)', 'DATE', 'TIME WITHOUT TIME ZONE',
199-
'TIMESTAMP WITHOUT TIME ZONE', 'TIMESTAMP WITH TIME ZONE'
198+
'VARCHAR(3)', 'VARCHAR(111)', 'DATE', 'TIME',
199+
'TIMESTAMP', 'TIMESTAMP'
200200
],
201201
('vertica', 'redshift'): [
202202
'BIGINT', 'VARCHAR(3)', 'VARCHAR(3)', 'VARCHAR(1)', 'VARCHAR(1)',
203203
'VARCHAR(3)', 'VARCHAR(111)', 'DATE', 'VARCHAR(8)',
204-
'TIMESTAMP WITHOUT TIME ZONE', 'TIMESTAMPTZ'
204+
'TIMESTAMP', 'TIMESTAMPTZ'
205205
],
206206
}

‎tests/integration/records/single_db/numeric_expectations.py‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -120,11 +120,11 @@
120120
'uint64': 'NUMERIC(20, 0)',
121121
'float16': 'REAL',
122122
'float32': 'REAL',
123-
'float64': 'DOUBLE PRECISION',
124-
'float128': 'DOUBLE PRECISION', # Redshift doesn't support >float64
123+
'float64': 'DOUBLE_PRECISION',
124+
'float128': 'DOUBLE_PRECISION', # Redshift doesn't support >float64
125125
'fixed_6_2': 'NUMERIC(6, 2)',
126126
'fixed_38_9': 'NUMERIC(38, 9)',
127-
'fixed_100_4': 'DOUBLE PRECISION' # Redshift doesn't support fixed precision > 38
127+
'fixed_100_4': 'DOUBLE_PRECISION' # Redshift doesn't support fixed precision > 38
128128
},
129129
'vertica': {
130130
'int8': 'INTEGER',
@@ -180,8 +180,8 @@
180180
'uint64': 'NUMERIC(20, 0)',
181181
'float16': 'REAL',
182182
'float32': 'REAL',
183-
'float64': 'DOUBLE PRECISION',
184-
'float128': 'DOUBLE PRECISION', # Postgres doesn't support >float64
183+
'float64': 'DOUBLE_PRECISION',
184+
'float128': 'DOUBLE_PRECISION', # Postgres doesn't support >float64
185185
'fixed_6_2': 'NUMERIC(6, 2)',
186186
'fixed_38_9': 'NUMERIC(38, 9)',
187187
'fixed_100_4': 'NUMERIC(100, 4)',
@@ -191,15 +191,15 @@
191191
# relevant and records-mover just uses the defaults which end up
192192
# as the below.
193193
'mysql': {
194-
'int8': 'TINYINT(4)',
195-
'int16': 'SMALLINT(6)',
196-
'int32': 'INTEGER(11)',
197-
'int64': 'BIGINT(20)',
198-
'ubyte': 'TINYINT(3) UNSIGNED',
199-
'uint8': 'TINYINT(3) UNSIGNED',
200-
'uint16': 'SMALLINT(5) UNSIGNED',
201-
'uint32': 'INTEGER(10) UNSIGNED',
202-
'uint64': 'BIGINT(20) UNSIGNED',
194+
'int8': 'TINYINT',
195+
'int16': 'SMALLINT',
196+
'int32': 'INTEGER',
197+
'int64': 'BIGINT',
198+
'ubyte': 'TINYINT',
199+
'uint8': 'TINYINT',
200+
'uint16': 'SMALLINT',
201+
'uint32': 'INTEGER',
202+
'uint64': 'BIGINT',
203203
'float16': 'FLOAT',
204204
'float32': 'FLOAT',
205205
'float64': 'DOUBLE',

‎tests/integration/records/single_db/test_records_numeric.py‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
RecordsSchema, DelimitedRecordsFormat, ProcessingInstructions
1111
)
1212
from ..records_numeric_database_fixture import RecordsNumericDatabaseFixture
13+
from sqlalchemy.engine import Engine
1314

1415
logger = logging.getLogger(__name__)
1516

@@ -67,8 +68,17 @@ def test_numeric_schema_fields_created(self) -> None:
6768
self.validate_records_schema(tempdir)
6869

6970
def validate_table(self):
70-
columns = self.engine.dialect.get_columns(self.engine, self.table_name,
71-
schema=self.schema_name)
71+
if isinstance(self.engine, Engine):
72+
connection = self.engine.connect()
73+
columns = self.engine.dialect.get_columns(connection,
74+
self.table_name,
75+
schema=self.schema_name)
76+
connection.close()
77+
else:
78+
columns = self.engine.dialect.get_columns(self.engine,
79+
self.table_name,
80+
schema=self.schema_name)
81+
7282
# Note that Redshift doesn't support TIME type:
7383
# https://docs.aws.amazon.com/redshift/latest/dg/r_Datetime_types.html
7484
actual_column_types = {

‎tests/integration/records/table_validator.py‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,9 +62,17 @@ def validate(self,
6262
self.validate_data_values(schema_name, table_name)
6363

6464
def validate_data_types(self, schema_name: str, table_name: str) -> None:
65-
columns = self.target_db_engine.dialect.get_columns(self.target_db_engine,
66-
table_name,
67-
schema=schema_name)
65+
if isinstance(self.target_db_engine, Engine):
66+
connection = self.target_db_engine.connect()
67+
columns = self.target_db_engine.dialect.get_columns(connection,
68+
table_name,
69+
schema=schema_name)
70+
connection.close()
71+
else:
72+
columns = self.target_db_engine.dialect.get_columns(self.target_db_engine,
73+
table_name,
74+
schema=schema_name)
75+
6876
expected_column_names = [
6977
'num', 'numstr', 'str', 'comma', 'doublequote', 'quotecommaquote',
7078
'newlinestr', 'date', 'time', 'timestamp', 'timestamptz'

0 commit comments

Comments
 (0)