Migration monorepo

README.md

@@ -33,7 +33,6 @@ export BD_DJANGO_PASSWORD="password"
 ```python
 from databasers_utils import (
     TableArchitecture,
-    copy_models_from_dev_to_prod,
     get_architecture_table_from_api,
 )
 
@@ -61,9 +60,6 @@ arch.update_dbt_project()
 # Faz o upload das colunas para o DJango
 arch.upload_columns()
 
-# Copia os modelos em dev para prod
-copy_models_from_dev_to_prod(["br_ibge_ppm", "br_ibge_pam"])
-
 # Retorna um DataFrame da arquitetura obtida na API
 # Util para gerar arquitetura quando ela não está no Drive
 get_architecture_table_from_api("br_ms_sinasc", "microdados")

pyproject.toml

@@ -17,11 +17,11 @@ lint = "scripts.lint:main"
 
 [tool.poetry.dependencies]
 python = ">=3.9,<4"
-basedosdados = {version = "2.0.0-beta.27", extras = ["all"]}
+basedosdados = {extras = ["all"], version = "^2.0.2"}
 pandas = "^2.2.2"
 numpy = "^1.26.4"
-requests = "^2.32.3"
-ruamel-yaml = "^0.18.6"
+requests = ">=2.26.0"
+ruamel-yaml = ">=0.17.10"
 
 
 [tool.poetry.group.dev.dependencies]

src/databasers_utils/__init__.py

@@ -1,9 +1,7 @@
-from .copy_models_from_dev_to_prod import copy_models_from_dev_to_prod
 from .get_architecture_table_from_api import get_architecture_table_from_api
 from .table_architecture import TableArchitecture
 
 __all__ = [
-    "copy_models_from_dev_to_prod",
     "get_architecture_table_from_api",
     "TableArchitecture",
 ]

src/databasers_utils/table_architecture.py

@@ -77,7 +77,7 @@ def create_sql_files(
                     sql_line = f"    safe_cast({original_name} as {bq_type}) {column_name},\n"
                     file.write(sql_line)
 
-                sql_last_line = f"from `basedosdados-dev.{self.dataset_id}_staging.{table_id}` as t\n"
+                sql_last_line = f'from {{{{ set_datalake_project("{self.dataset_id}_staging.{table_id}") }}}} as t\n'
                 file.write(sql_last_line)
 
         print("SQL files created!")

src/databasers_utils/utils.py

@@ -25,10 +25,21 @@ def read_architecture_table(url: str) -> pd.DataFrame:
         io.StringIO(
             requests.get(export_url, timeout=10).content.decode("utf-8")
         )
-    )
+    ).apply(lambda x: x.str.strip() if x.dtype == object else x)
 
     df_architecture = df_architecture.loc[
-        df_architecture["name"] != "(excluido)"
+        ~df_architecture["name"].isin(  # type: ignore
+            [
+                "(excluido)",
+                "excluido",
+                "(excluído)",
+                "excluído",
+                "removido",
+                "(removido)",
+                "deletado",
+                "(deletado)",
+            ]
+        )
     ]
 
     return df_architecture.replace(np.nan, "", regex=True)

tests/queries-basedosdados-dev/models/br_ibge_pib/br_ibge_pib__gini.sql

@@ -8,4 +8,4 @@ select
     safe_cast(gini_va_industria as int64) gini_va_industria,
     safe_cast(gini_va_servicos as int64) gini_va_servicos,
     safe_cast(gini_va_adespss as int64) gini_va_adespss,
-from `basedosdados-dev.br_ibge_pib_staging.gini` as t
+from {{ set_datalake_project("br_ibge_pib_staging.gini") }} as t

tests/queries-basedosdados-dev/models/br_ibge_pib/br_ibge_pib__uf.sql

@@ -10,4 +10,4 @@ select
     safe_cast(va_industria as int64) va_industria,
     safe_cast(va_servicos as int64) va_servicos,
     safe_cast(va_adespss as int64) va_adespss,
-from `basedosdados-dev.br_ibge_pib_staging.uf` as t
+from {{ set_datalake_project("br_ibge_pib_staging.uf") }} as t