chore(CI): fix CI workflows (#214)

* fix(CI): fix CI workflows

* run pre-commit

* include pre-commit before testing

* include pre-commit before commiting

* linting pt.1

* linting pt.2

* replace mocked tests by endpoint tests

* remove compose-go

* linting pt.3 final

Author: luabida

.github/workflows/python-package.yml

@@ -35,8 +35,10 @@ jobs:
     - name: Run jupyterlab with PySUS
       run: |
         make run-jupyter-pysus
+        # make test-jupyter-pysus  ## takes too long
 
-    - name: Test with pytest
+    - name: Linting & Tests
       run: |
         poetry install
+        pre-commit run --all-files
         make test-pysus

.idea/PySUS.iml

@@ -1,3 +1,5 @@
+default_stages: [commit, push]
+
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.1.0
@@ -16,8 +18,6 @@ repos:
       files: ""
       language: system
       pass_filenames: true
-      stages:
-        - commit
       types:
         - python
         - file
@@ -30,8 +30,6 @@ repos:
       language: python
       name: flake8
       pass_filenames: true
-      stages:
-        - commit
       types:
         - python
 
@@ -42,7 +40,5 @@ repos:
       language: python
       name: isort
       pass_filenames: true
-      stages:
-        - commit
       types:
         - python

.idea/misc.xml

@@ -22,7 +22,7 @@ export PRINT_HELP_PYSCRIPT
 help:
 	@python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST)
 
-DOCKER = docker-compose -p pysus -f docker/docker-compose.yaml
+DOCKER = docker compose -p pysus -f docker/docker-compose.yaml
 SERVICE :=
 SEMANTIC_RELEASE = npx --yes \
           -p semantic-release \
@@ -48,11 +48,10 @@ down-jupyter-pysus: ## stop and remove containers for all services
 #* Tests
 .PHONY: test-jupyter-pysus
 test-jupyter-pysus: ## run pytest for notebooks inside jupyter container
-	$(DOCKER) exec -T jupyter bash /test_notebooks.sh
+	$(DOCKER) exec -T jupyter pytest -vv --nbmake
 
 .PHONY: test-pysus
 test-pysus: ## run tests quickly with the default Python
-	cp docs/source/**/*.ipynb pysus/Notebooks
 	poetry run pytest -vv pysus/tests/
 
 # RELEASE

.pre-commit-config.yaml

@@ -55,7 +55,7 @@ First, clone the Pysus repository:
 
 ```bash
 git clone https://github.com/fccoelho/PySUS.git
-``` 
+```
 then from within the PySUS directory build the container
 
 ```bash
@@ -72,7 +72,7 @@ Point your browser to [http://127.0.0.1:8888](http://127.0.0.1:8888) and have fu
 Once you are done, you can stop the container with a simple `ctrl-c` from the terminal you started it or use the following command:
 ```bash
 # to find the container ID
-docker ps 
+docker ps
 docker stop <CONTAINER ID>
 ```
 ### Mounting your working directory in the container
@@ -105,7 +105,7 @@ setx PYSUS_CACHEPATH "C:\Users\Me\desired\path\.pysus"
 In Docker, just add an extra parameter `-e PYSUS_CACHEPATH="/home/me/desired/path/.pysus"` when starting the container:
 
 ```bash
-docker run -p 8888:8888 -e PYSUS_CACHEPATH="/home/me/desired/path/.pysus" pysus:latest 
+docker run -p 8888:8888 -e PYSUS_CACHEPATH="/home/me/desired/path/.pysus" pysus:latest
 ```
 
 ## Examples
@@ -164,7 +164,7 @@ Downloading and reading SINASC data:
 In[1]: from pysus.online_data.sinasc import download
 In[2]: df = download('SE', 2015)
 In[3]: df.head()
-Out[3]: 
+Out[3]:
    NUMERODN        CODINST ORIGEM    ...     TPROBSON PARIDADE KOTELCHUCK
 0  19533794  MSE2805100001      1    ...           11        1          9
 1  52927108  MSE2802700001      1    ...           11        1          9
@@ -180,7 +180,7 @@ Dowloading and reading SIM data:
 In[1]: from pysus.online_data.SIM import download
 In[2]: df = download('ba', 2007)
 In[3]: df.head()
-Out[3]: 
+Out[3]:
    NUMERODO TIPOBITO   DTOBITO  ...   UFINFORM        CODINST CB_PRE
 0  01499664        2  30072007  ...         29  RBA2914800001   C229
 1  09798190        2  04072007  ...         29  RBA2914800001    R98
@@ -196,7 +196,7 @@ Dowloading and reading CIHA data:
 In[1]: from pysus.online_data.CIHA import download
 In[2]: df = download('mg', 2009, 7)
 In[3]: df.head()
-Out[3]: 
+Out[3]:
   ANO_CMPT MES_CMPT ESPEC        CGC_HOSP  ...  CAR_INT HOMONIMO     CNES FONTE
 0     2009       07        16505851000126  ...                    2126796     1
 1     2009       07        16505851000126  ...                    2126796     2
@@ -212,7 +212,7 @@ Dowloading and reading SIA data:
 In[1]: from pysus.online_data.SIA import download
 In[2]: bi, ps = download('AC', 2020, 3, group=["BI", "PS"])
 In[3]: bi.head()
-Out[3]: 
+Out[3]:
     CODUNI  GESTAO CONDIC   UFMUN TPUPS  ... VL_APROV UFDIF MNDIF ETNIA NAT_JUR
 0  2000733  120000     EP  120040    07  ...     24.2     0     0          1023
 1  2001063  120000     EP  120040    36  ...      7.3     0     0          1023

Makefile

@@ -3,6 +3,7 @@ channels:
   - conda-forge
   - defaults
 dependencies:
+  - docker-compose
   - python >=3.10,<3.12
   - numpy>=1.26.2
   - cffi

README.md

@@ -47,10 +47,10 @@ about:
   home: "https://github.com/fccoelho/PySUS"
   license: gpl-v3
   license_family: GPL3
-  license_file: 
+  license_file:
   summary: "Tools for dealing with Brazil's Public health data"
-  doc_url: 
-  dev_url: 
+  doc_url:
+  dev_url:
 
 extra:
   recipe-maintainers:

conda/dev.yaml

@@ -30,6 +30,10 @@ RUN useradd -ms /bin/bash pysus \
   && chmod -R a+rwx /opt/conda /tmp \
   && sudo chown -R pysus:pysus /usr/src
 
+USER pysus
+
+RUN mkdir -p /home/pysus/Notebooks/
+
 COPY --chown=pysus:pysus conda/dev.yaml /tmp/dev.yaml
 COPY --chown=pysus:pysus docker/scripts/entrypoint.sh /entrypoint.sh
 COPY --chown=pysus:pysus docker/scripts/poetry-install.sh /tmp/poetry-install.sh
@@ -38,8 +42,6 @@ COPY --chown=pysus:pysus pysus /usr/src/pysus
 COPY --chown=pysus:pysus docs/source/**/*.ipynb /home/pysus/Notebooks/
 COPY --chown=pysus:pysus docs/source/data /home/pysus/Notebooks/
 
-USER pysus
-
 RUN mamba env create -n $ENV_NAME --file /tmp/dev.yaml \
   && mamba clean -afy
 

condarecipe/pysus/meta.yaml

@@ -1,5 +1,5 @@
 version: '3.3'
-services: 
+services:
   jupyter:
     build:
       context: ".."