|
66 | 66 | "cell_type": "markdown", |
67 | 67 | "metadata": {}, |
68 | 68 | "source": [ |
69 | | - "## Checking Glue Catalog Databases" |
70 | | - ] |
71 | | - }, |
72 | | - { |
73 | | - "cell_type": "code", |
74 | | - "execution_count": 3, |
75 | | - "metadata": {}, |
76 | | - "outputs": [ |
77 | | - { |
78 | | - "name": "stdout", |
79 | | - "output_type": "stream", |
80 | | - "text": [ |
81 | | - " Database Description\n", |
82 | | - "0 aws_data_wrangler AWS Data Wrangler Test Arena - Glue Database\n", |
83 | | - "1 default Default Hive database\n" |
84 | | - ] |
85 | | - } |
86 | | - ], |
87 | | - "source": [ |
88 | | - "databases = wr.catalog.databases()\n", |
89 | | - "print(databases)" |
| 69 | + "## Checking/Creating Glue Catalog Databases" |
90 | 70 | ] |
91 | 71 | }, |
92 | 72 | { |
|
106 | 86 | } |
107 | 87 | ], |
108 | 88 | "source": [ |
109 | | - "if \"awswrangler_test\" not in databases.values:\n", |
110 | | - " wr.catalog.create_database(\"awswrangler_test\")\n", |
111 | | - " print(wr.catalog.databases())\n", |
112 | | - "else:\n", |
113 | | - " print(\"Database awswrangler_test already exists\")" |
| 89 | + "if \"awswrangler_test\" not in wr.catalog.databases().values:\n", |
| 90 | + " wr.catalog.create_database(\"awswrangler_test\")" |
114 | 91 | ] |
115 | 92 | }, |
116 | 93 | { |
|
324 | 301 | "metadata": {}, |
325 | 302 | "outputs": [], |
326 | 303 | "source": [ |
327 | | - "res = wr.s3.to_parquet(\n", |
| 304 | + "wr.s3.to_parquet(\n", |
328 | 305 | " df=df,\n", |
329 | 306 | " path=path,\n", |
330 | 307 | " dataset=True,\n", |
331 | 308 | " mode=\"overwrite\",\n", |
332 | 309 | " database=\"awswrangler_test\",\n", |
333 | 310 | " table=\"noaa\"\n", |
334 | | - ")" |
| 311 | + ");" |
335 | 312 | ] |
336 | 313 | }, |
337 | 314 | { |
|
1120 | 1097 | " \"SELECT * FROM noaa\",\n", |
1121 | 1098 | " database=\"awswrangler_test\",\n", |
1122 | 1099 | " ctas_approach=False,\n", |
1123 | | - " chunksize=10_000_000\n", |
| 1100 | + " chunksize=500_000\n", |
1124 | 1101 | ")\n", |
1125 | 1102 | "\n", |
1126 | 1103 | "for df in dfs: # Batching\n", |
|
1147 | 1124 | "cell_type": "markdown", |
1148 | 1125 | "metadata": {}, |
1149 | 1126 | "source": [ |
1150 | | - "## Cleaning Up the Database" |
| 1127 | + "## Delete table" |
1151 | 1128 | ] |
1152 | 1129 | }, |
1153 | 1130 | { |
|
1156 | 1133 | "metadata": {}, |
1157 | 1134 | "outputs": [], |
1158 | 1135 | "source": [ |
1159 | | - "for table in wr.catalog.get_tables(database=\"awswrangler_test\"):\n", |
1160 | | - " wr.catalog.delete_table_if_exists(database=\"awswrangler_test\", table=table[\"Name\"])" |
| 1136 | + "wr.catalog.delete_table_if_exists(database=\"awswrangler_test\", table=\"noaa\")" |
1161 | 1137 | ] |
1162 | 1138 | }, |
1163 | 1139 | { |
1164 | 1140 | "cell_type": "markdown", |
1165 | 1141 | "metadata": {}, |
1166 | 1142 | "source": [ |
1167 | | - "### Delete Database" |
| 1143 | + "## Delete Database" |
1168 | 1144 | ] |
1169 | 1145 | }, |
1170 | 1146 | { |
|
1193 | 1169 | "name": "python", |
1194 | 1170 | "nbconvert_exporter": "python", |
1195 | 1171 | "pygments_lexer": "ipython3", |
1196 | | - "version": "3.7.7" |
| 1172 | + "version": "3.6.10" |
1197 | 1173 | }, |
1198 | 1174 | "pycharm": { |
1199 | 1175 | "stem_cell": { |
|
0 commit comments