diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 971a7a7..c05f05a 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -48,4 +48,4 @@ jobs: kedro build-reqs pip install -r src/requirements.txt - name: Run build - run: kedro package + run: kedro package \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..15e057d --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: check-yaml + - id: check-json + - id: check-ast + - id: check-docstring-first + - id: name-tests-test + - id: end-of-file-fixer + - id: trailing-whitespace + - id: detect-private-key + - id: fix-encoding-pragma + - id: requirements-txt-fixer +- repo: https://github.com/psf/black + rev: 21.9b0 + hooks: + - id: black diff --git a/README.md b/README.md index 5344531..940008a 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,56 @@ +![example branch parameter](https://github.com/elcubonegro/kedro-devops/actions/workflows/pipeline.yml/badge.svg?branch=main) + # kedro-devops This project intends to demonstrate and define a Dev(ML)Ops pipeline for Kedro - ## Setup To install de project you must have [conda](https://conda.io/projects/conda/en/latest/user-guide/install/index.html) and setup a Python3.7 environment as follows: ```bash -conda create --name kedro-devops python=3.7 -y +conda create --name Aldebaran_Learning python=3.7 -y ``` Then you need to activate your virtualenv ```bash -conda activate kedro-devops +conda activate Aldebaran_Learning ``` - > Note: if you are using windows you may need to use a cmd shell instead of a powershell to activate a conda environment +Init the git flow + +```bash +git clone -b master git@github.com:elcubonegro/gitflow-avh.git +cd gitflow-avh +make && make install +``` + +Init the git pre-commit hooks + +```bash +conda activate Aldebaran-Learning +``` + +After that initialize the local gitflow repository with gitflow itself: + +```bash +git flow init -d +git flow feature start +``` + +Then, do work and commit your changes. + +```bash +git flow feature publish +When done, open a pull request to your feature branch. +``` + ## Overview -This is your new Kedro project, which was generated using `Kedro 0.17.5`. +this is a Kedro project, builded using `Kedro 0.17.5`. -Take a look at the [Kedro documentation](https://kedro.readthedocs.io) to get started. +For more information please reffer to [Kedro documentation](https://kedro.readthedocs.io). ## Rules and guidelines diff --git a/src/kedro_devops/common/.gitkeep b/src/kedro_devops/common/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/kedro_devops/pipelines/data_engineering/nodes/transform_uppercase.py b/src/kedro_devops/pipelines/data_engineering/nodes/transform_uppercase.py index 5a3b3e1..5b416c3 100644 --- a/src/kedro_devops/pipelines/data_engineering/nodes/transform_uppercase.py +++ b/src/kedro_devops/pipelines/data_engineering/nodes/transform_uppercase.py @@ -3,12 +3,11 @@ def transform_uppercase(data: pd.DataFrame) -> pd.DataFrame: """ - Transform a lowercase dataframe to uppercase. - + Transform the data to uppercase. Args: - data (pd.DataFrame): A raw dataframe + data (DataFrame): Data to be transformed. Returns: - pd.DataFrame: An uppercase dataframe + (DataFrame) Transformed data to uppercase. """ - return data.applymap(lambda x: x.upper()) + return data.applymap(lambda row: row.upper()) \ No newline at end of file diff --git a/src/requirements.in b/src/requirements.in index 5d3de76..d05df99 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -11,12 +11,13 @@ kedro-telemetry~=0.1.0 mypy~=0.910 nbstripout~=0.4 pandas-stubs +pre-commit==2.15.0 pydocstyle~=6.1.1 pytest-cov~=2.5 pytest-mock>=1.7.1, <2.0 pytest~=6.2 -types-atomicwrites -types-cachetools -types-setuptools +types-atomicwrites +types-cachetools +types-setuptools types-toml wheel>=0.35, <0.37 diff --git a/src/requirements.txt b/src/requirements.txt index 3f90e92..6854c9e 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -2,6 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # + # pip-compile 'C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in' # anyconfig==0.10.1 @@ -10,8 +11,6 @@ anyio==3.3.3 # via jupyter-server appdirs==1.4.4 # via black -argcomplete==1.12.3 - # via ipykernel argon2-cffi==21.1.0 # via # jupyter-server @@ -28,10 +27,12 @@ babel==2.9.1 # via jupyterlab-server backcall==0.2.0 # via ipython +backports.entry-points-selectable==1.1.0 + # via virtualenv binaryornot==0.4.4 # via cookiecutter black==21.5b1 - # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in + # via -r C:\Users\j.alban\Desktop\Sumz\Decameron\kedro-devops\src\requirements.in bleach==4.1.0 # via nbconvert cachetools==4.2.4 @@ -40,6 +41,8 @@ certifi==2021.10.8 # via requests cffi==1.14.6 # via argon2-cffi +cfgv==3.3.1 + # via pre-commit chardet==4.0.0 # via # binaryornot @@ -65,12 +68,16 @@ decorator==5.1.0 # via ipython defusedxml==0.7.1 # via nbconvert +distlib==0.3.3 + # via virtualenv dynaconf==3.1.5 # via kedro entrypoints==0.3 # via # jupyterlab-server # nbconvert +filelock==3.3.1 + # via virtualenv flake8==3.9.2 # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in fsspec==2021.10.0 @@ -79,28 +86,15 @@ gitdb==4.0.7 # via gitpython gitpython==3.1.24 # via kedro +identify==2.3.0 + # via pre-commit idna==2.10 # via # anyio # requests -importlib-metadata==4.8.1 - # via - # argcomplete - # flake8 - # ipykernel - # jsonschema - # pluggy - # pytest iniconfig==1.1.1 # via pytest ipykernel==6.4.1 - # via - # ipywidgets - # jupyter - # jupyter-console - # notebook - # qtconsole -ipython-genutils==0.2.0 # via # ipykernel # ipywidgets @@ -120,10 +114,10 @@ ipywidgets==7.6.5 isort==5.9.3 # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in jedi==0.18.0 - # via ipython -jinja2-time==0.2.0 # via cookiecutter jinja2==3.0.2 + # via ipython +jinja2-time==0.2.0 # via # cookiecutter # jinja2-time @@ -137,6 +131,11 @@ jmespath==0.10.0 json5==0.9.6 # via jupyterlab-server jsonschema==4.1.0 + # via + # jupyterlab + # jupyterlab-server + # nbclassic +jupyter==1.0.0 # via # jupyterlab-server # nbformat @@ -173,6 +172,7 @@ jupyterlab-pygments==0.1.2 jupyterlab-server==2.8.2 # via jupyterlab jupyterlab-widgets==1.0.2 + # via ipywidgets jupyterlab==3.1.18 # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in @@ -194,8 +194,6 @@ matplotlib-inline==0.1.3 mccabe==0.6.1 # via flake8 mistune==0.8.4 - # via nbconvert -mypy-extensions==0.4.3 # via # black # mypy @@ -222,6 +220,8 @@ nbstripout==0.5.0 # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in nest-asyncio==1.5.1 # via nbclient +nodeenv==1.6.0 + # via pre-commit notebook==6.4.4 # via # jupyter @@ -230,6 +230,8 @@ notebook==6.4.4 numpy==1.21.2 # via pandas packaging==21.0 + # via -r C:\Users\j.alban\Desktop\Sumz\Decameron\kedro-devops\src\requirements.in +pandas==1.3.3 # via # bleach # jupyterlab @@ -249,12 +251,16 @@ pickleshare==0.7.5 # via ipython pip-tools==5.5.0 # via kedro +platformdirs==2.4.0 + # via virtualenv pluggy==0.13.1 # via # kedro # pytest poyo==0.5.0 # via cookiecutter +pre-commit==2.15.0 + # via -r C:\Users\j.alban\Desktop\Sumz\Decameron\kedro-devops\src\requirements.in prometheus-client==0.11.0 # via # jupyter-server @@ -283,6 +289,8 @@ pygments==2.10.0 pyparsing==2.4.7 # via packaging pyrsistent==0.18.0 + # via -r C:\Users\j.alban\Desktop\Sumz\Decameron\kedro-devops\src\requirements.in +pytest==6.2.5 # via jsonschema pytest-cov==2.12.1 # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in @@ -316,6 +324,7 @@ pyyaml==5.4.1 # via # kedro # kedro-telemetry + # pre-commit pyzmq==22.3.0 # via # jupyter-client @@ -327,10 +336,10 @@ qtconsole==5.1.1 qtpy==1.11.2 # via qtconsole regex==2021.10.8 - # via black -requests-unixsocket==0.2.0 # via jupyter-server requests==2.25.1 + # via black +requests-unixsocket==0.2.0 # via # cookiecutter # jupyterlab-server @@ -349,6 +358,7 @@ six==1.16.0 # bleach # cookiecutter # python-dateutil + # virtualenv smmap==4.0.0 # via gitdb sniffio==1.2.0 @@ -368,6 +378,7 @@ toml==0.10.2 # black # kedro # mypy + # pre-commit # pytest # pytest-cov toposort==1.7 @@ -394,10 +405,6 @@ traitlets==5.1.0 # nbformat # notebook # qtconsole -typed-ast==1.4.3 - # via - # black - # mypy types-atomicwrites==1.4.1 # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in types-cachetools==4.2.4 @@ -408,17 +415,14 @@ types-toml==0.10.1 # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in typing-extensions==3.10.0.2 # via - # anyio - # arrow - # black # gitpython - # importlib-metadata # mypy - # pandas-stubs urllib3==1.26.7 # via # requests # requests-unixsocket +virtualenv==20.8.1 + # via pre-commit wcwidth==0.2.5 # via prompt-toolkit webencodings==0.5.1 @@ -429,8 +433,6 @@ wheel==0.36.2 # via -r C:\Users\juanm\Documents\demos\kedro-devops\src\requirements.in widgetsnbextension==3.5.1 # via ipywidgets -zipp==3.6.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip