diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 971a7a7..c8accaa 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -49,3 +49,4 @@ jobs: pip install -r src/requirements.txt - name: Run build run: kedro package + diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..15e057d --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: check-yaml + - id: check-json + - id: check-ast + - id: check-docstring-first + - id: name-tests-test + - id: end-of-file-fixer + - id: trailing-whitespace + - id: detect-private-key + - id: fix-encoding-pragma + - id: requirements-txt-fixer +- repo: https://github.com/psf/black + rev: 21.9b0 + hooks: + - id: black diff --git a/README.md b/README.md index 5344531..a3afc8d 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,56 @@ +![example branch parameter](https://github.com/elcubonegro/kedro-devops/actions/workflows/pipeline.yml/badge.svg?branch=main) + # kedro-devops This project intends to demonstrate and define a Dev(ML)Ops pipeline for Kedro - ## Setup To install de project you must have [conda](https://conda.io/projects/conda/en/latest/user-guide/install/index.html) and setup a Python3.7 environment as follows: ```bash -conda create --name kedro-devops python=3.7 -y +conda create --name Aldebaran-Learning python=3.7 -y ``` Then you need to activate your virtualenv ```bash -conda activate kedro-devops +conda activate Aldebaran-Learning ``` - > Note: if you are using windows you may need to use a cmd shell instead of a powershell to activate a conda environment +Init the git flow + +```bash +git clone -b master git@github.com:elcubonegro/gitflow-avh.git +cd gitflow-avh +make && make install +``` + +Init the git pre-commit hooks + +```bash +conda activate Aldebaran-Learning +``` + +After that initialize the local gitflow repository with gitflow itself: + +```bash +git flow init -d +git flow feature start +``` + +Then, do work and commit your changes. + +```bash +git flow feature publish +When done, open a pull request to your feature branch. +``` + ## Overview -This is your new Kedro project, which was generated using `Kedro 0.17.5`. +this is a Kedro project, builded using `Kedro 0.17.5`. -Take a look at the [Kedro documentation](https://kedro.readthedocs.io) to get started. +For more information please reffer to [Kedro documentation](https://kedro.readthedocs.io). ## Rules and guidelines diff --git a/src/kedro_devops/common/.gitkeep b/src/kedro_devops/common/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/kedro_devops/pipelines/data_engineering/nodes/transform_uppercase.py b/src/kedro_devops/pipelines/data_engineering/nodes/transform_uppercase.py index 5a3b3e1..5b416c3 100644 --- a/src/kedro_devops/pipelines/data_engineering/nodes/transform_uppercase.py +++ b/src/kedro_devops/pipelines/data_engineering/nodes/transform_uppercase.py @@ -3,12 +3,11 @@ def transform_uppercase(data: pd.DataFrame) -> pd.DataFrame: """ - Transform a lowercase dataframe to uppercase. - + Transform the data to uppercase. Args: - data (pd.DataFrame): A raw dataframe + data (DataFrame): Data to be transformed. Returns: - pd.DataFrame: An uppercase dataframe + (DataFrame) Transformed data to uppercase. """ - return data.applymap(lambda x: x.upper()) + return data.applymap(lambda row: row.upper()) \ No newline at end of file diff --git a/src/requirements.in b/src/requirements.in index 5d3de76..d05df99 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -11,12 +11,13 @@ kedro-telemetry~=0.1.0 mypy~=0.910 nbstripout~=0.4 pandas-stubs +pre-commit==2.15.0 pydocstyle~=6.1.1 pytest-cov~=2.5 pytest-mock>=1.7.1, <2.0 pytest~=6.2 -types-atomicwrites -types-cachetools -types-setuptools +types-atomicwrites +types-cachetools +types-setuptools types-toml wheel>=0.35, <0.37 diff --git a/src/requirements.txt b/src/requirements.txt index 3f90e92..e69de29 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,437 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile 'C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in' -# -anyconfig==0.10.1 - # via kedro -anyio==3.3.3 - # via jupyter-server -appdirs==1.4.4 - # via black -argcomplete==1.12.3 - # via ipykernel -argon2-cffi==21.1.0 - # via - # jupyter-server - # notebook -arrow==1.2.0 - # via jinja2-time -atomicwrites==1.4.0 - # via pytest -attrs==21.2.0 - # via - # jsonschema - # pytest -babel==2.9.1 - # via jupyterlab-server -backcall==0.2.0 - # via ipython -binaryornot==0.4.4 - # via cookiecutter -black==21.5b1 - # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in -bleach==4.1.0 - # via nbconvert -cachetools==4.2.4 - # via kedro -certifi==2021.10.8 - # via requests -cffi==1.14.6 - # via argon2-cffi -chardet==4.0.0 - # via - # binaryornot - # requests -click==7.1.2 - # via - # black - # cookiecutter - # kedro - # kedro-telemetry - # pip-tools -colorama==0.4.4 - # via - # ipython - # pytest -cookiecutter==1.7.3 - # via kedro -coverage==6.0.2 - # via pytest-cov -debugpy==1.5.0 - # via ipykernel -decorator==5.1.0 - # via ipython -defusedxml==0.7.1 - # via nbconvert -dynaconf==3.1.5 - # via kedro -entrypoints==0.3 - # via - # jupyterlab-server - # nbconvert -flake8==3.9.2 - # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in -fsspec==2021.10.0 - # via kedro -gitdb==4.0.7 - # via gitpython -gitpython==3.1.24 - # via kedro -idna==2.10 - # via - # anyio - # requests -importlib-metadata==4.8.1 - # via - # argcomplete - # flake8 - # ipykernel - # jsonschema - # pluggy - # pytest -iniconfig==1.1.1 - # via pytest -ipykernel==6.4.1 - # via - # ipywidgets - # jupyter - # jupyter-console - # notebook - # qtconsole -ipython-genutils==0.2.0 - # via - # ipykernel - # ipywidgets - # jupyter-server - # nbformat - # notebook - # qtconsole -ipython==7.28.0 - # via - # -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in - # ipykernel - # ipywidgets - # jupyter-console - # jupyterlab -ipywidgets==7.6.5 - # via jupyter -isort==5.9.3 - # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in -jedi==0.18.0 - # via ipython -jinja2-time==0.2.0 - # via cookiecutter -jinja2==3.0.2 - # via - # cookiecutter - # jinja2-time - # jupyter-server - # jupyterlab - # jupyterlab-server - # nbconvert - # notebook -jmespath==0.10.0 - # via kedro -json5==0.9.6 - # via jupyterlab-server -jsonschema==4.1.0 - # via - # jupyterlab-server - # nbformat -jupyter-client==6.1.12 - # via - # -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in - # ipykernel - # jupyter-console - # jupyter-server - # kedro - # nbclient - # notebook - # qtconsole -jupyter-console==6.4.0 - # via jupyter -jupyter-core==4.8.1 - # via - # jupyter-client - # jupyter-server - # jupyterlab - # nbconvert - # nbformat - # notebook - # qtconsole -jupyter-server==1.11.1 - # via - # jupyterlab - # jupyterlab-server - # nbclassic -jupyter==1.0.0 - # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in -jupyterlab-pygments==0.1.2 - # via nbconvert -jupyterlab-server==2.8.2 - # via jupyterlab -jupyterlab-widgets==1.0.2 - # via ipywidgets -jupyterlab==3.1.18 - # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in -kedro-airflow==0.4.1 - # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in -kedro-telemetry==0.1.2 - # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in -kedro[pandas.csvdataset]==0.17.5 - # via - # -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in - # kedro-airflow - # kedro-telemetry -markupsafe==2.0.1 - # via jinja2 -matplotlib-inline==0.1.3 - # via - # ipykernel - # ipython -mccabe==0.6.1 - # via flake8 -mistune==0.8.4 - # via nbconvert -mypy-extensions==0.4.3 - # via - # black - # mypy -mypy==0.910 - # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in -nbclassic==0.3.2 - # via jupyterlab -nbclient==0.5.4 - # via nbconvert -nbconvert==6.2.0 - # via - # jupyter - # jupyter-server - # notebook -nbformat==5.1.3 - # via - # ipywidgets - # jupyter-server - # nbclient - # nbconvert - # nbstripout - # notebook -nbstripout==0.5.0 - # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in -nest-asyncio==1.5.1 - # via nbclient -notebook==6.4.4 - # via - # jupyter - # nbclassic - # widgetsnbextension -numpy==1.21.2 - # via pandas -packaging==21.0 - # via - # bleach - # jupyterlab - # jupyterlab-server - # pytest -pandas-stubs==1.2.0.29 - # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in -pandas==1.3.3 - # via kedro -pandocfilters==1.5.0 - # via nbconvert -parso==0.8.2 - # via jedi -pathspec==0.9.0 - # via black -pickleshare==0.7.5 - # via ipython -pip-tools==5.5.0 - # via kedro -pluggy==0.13.1 - # via - # kedro - # pytest -poyo==0.5.0 - # via cookiecutter -prometheus-client==0.11.0 - # via - # jupyter-server - # notebook -prompt-toolkit==3.0.20 - # via - # ipython - # jupyter-console -py==1.10.0 - # via pytest -pycodestyle==2.7.0 - # via flake8 -pycparser==2.20 - # via cffi -pydocstyle==6.1.1 - # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in -pyflakes==2.3.1 - # via flake8 -pygments==2.10.0 - # via - # ipython - # jupyter-console - # jupyterlab-pygments - # nbconvert - # qtconsole -pyparsing==2.4.7 - # via packaging -pyrsistent==0.18.0 - # via jsonschema -pytest-cov==2.12.1 - # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in -pytest-mock==1.13.0 - # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in -pytest==6.2.5 - # via - # -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in - # pytest-cov - # pytest-mock -python-dateutil==2.8.2 - # via - # arrow - # jupyter-client - # pandas -python-json-logger==2.0.2 - # via kedro -python-slugify==4.0.1 - # via - # cookiecutter - # kedro-airflow -pytz==2021.3 - # via - # babel - # pandas -pywin32==302 - # via jupyter-core -pywinpty==1.1.4 - # via terminado -pyyaml==5.4.1 - # via - # kedro - # kedro-telemetry -pyzmq==22.3.0 - # via - # jupyter-client - # jupyter-server - # notebook - # qtconsole -qtconsole==5.1.1 - # via jupyter -qtpy==1.11.2 - # via qtconsole -regex==2021.10.8 - # via black -requests-unixsocket==0.2.0 - # via jupyter-server -requests==2.25.1 - # via - # cookiecutter - # jupyterlab-server - # kedro-telemetry - # requests-unixsocket -rope==0.19.0 - # via kedro -semver==2.13.0 - # via kedro-airflow -send2trash==1.8.0 - # via - # jupyter-server - # notebook -six==1.16.0 - # via - # bleach - # cookiecutter - # python-dateutil -smmap==4.0.0 - # via gitdb -sniffio==1.2.0 - # via anyio -snowballstemmer==2.1.0 - # via pydocstyle -terminado==0.12.1 - # via - # jupyter-server - # notebook -testpath==0.5.0 - # via nbconvert -text-unidecode==1.3 - # via python-slugify -toml==0.10.2 - # via - # black - # kedro - # mypy - # pytest - # pytest-cov -toposort==1.7 - # via kedro -tornado==6.1 - # via - # ipykernel - # jupyter-client - # jupyter-server - # jupyterlab - # notebook - # terminado -traitlets==5.1.0 - # via - # ipykernel - # ipython - # ipywidgets - # jupyter-client - # jupyter-core - # jupyter-server - # matplotlib-inline - # nbclient - # nbconvert - # nbformat - # notebook - # qtconsole -typed-ast==1.4.3 - # via - # black - # mypy -types-atomicwrites==1.4.1 - # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in -types-cachetools==4.2.4 - # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in -types-setuptools==57.4.2 - # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in -types-toml==0.10.1 - # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in -typing-extensions==3.10.0.2 - # via - # anyio - # arrow - # black - # gitpython - # importlib-metadata - # mypy - # pandas-stubs -urllib3==1.26.7 - # via - # requests - # requests-unixsocket -wcwidth==0.2.5 - # via prompt-toolkit -webencodings==0.5.1 - # via bleach -websocket-client==1.2.1 - # via jupyter-server -wheel==0.36.2 - # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in -widgetsnbextension==3.5.1 - # via ipywidgets -zipp==3.6.0 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools