Merge pull request #7 from TatraDev/venv_logic

Docker Compose (fastapi, milvus) logic

Author: LaoGunz

.github/workflows/test.yaml

@@ -1,4 +1,4 @@
-name: Python package
+name: test
 
 on: [push]
 
@@ -27,6 +27,5 @@ jobs:
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-      - name: Test with pytest
-        run: |
-           pytest --verbose tests/running_piper_test.py::TestVenv
+      - name: Test with makefile
+        run: make test

MANIFEST.in

@@ -0,0 +1,4 @@
+include piper/base/backend/templates/*.j2
+include piper/base/docker/images/*.j2
+include piper/base/docker_compose/templates/*.j2
+include piper/base/virtualenv/templates/*.j2

Makefile

@@ -0,0 +1,16 @@
+SHELL:=/usr/bin/env bash
+
+.PHONY: lint
+lint:
+	flake8 piper
+
+.PHONY: unit
+unit:
+	pytest -vs tests/running_piper_test.py::TestCompose
+
+.PHONY: package
+package:
+	pip check
+
+.PHONY: test
+test: package unit

Readme.rst

@@ -5,6 +5,7 @@
 • `Chat (Community & Support) <https://t.me/pipertool>`_
 • `Tutorials <http://pipertool.org>`_
 
+|Contribs| |License| |Release| |Python Version| |Docs| |Build status| |Contact| |Blog|
 
 **Piper** is an **open-source** platform for data science and machine
 learning prototyping. Concentrate only on your goals. Key features:
@@ -31,6 +32,48 @@ How Piper works
 
 Quick start
 ===========
+Quick start pipertool package compose env
+===========
+
+In root directory project run command in terminal
+
+- sudo -u root /bin/bash
+
+- create and activate venv
+
+- pip install -r requirements.txt
+
+- in configuration.py rename for correctly path for new directory
+
+- python setup.py install
+
+- piper --env-type compose start
+
+- 0.0.0.0:7585 - FastApi
+
+- 0.0.0.0:9001 - Milvus Console (minioadmin/minioadmin)
+
+- piper --env-type compose stop
+
+- pip uninstall piper
+
+Quick start pipertool package compose env
+===========
+
+In root directory project run command in terminal
+
+- sudo -u root /bin/bash
+
+- create and activate venv
+
+- pip install -r requirements.txt
+
+- in configuration.py rename for correctly path for new directory
+
+- python main.py
+
+- await click CTRL+C from compose env
+
 
 
 Installation
@@ -88,6 +131,28 @@ By submitting a pull request to this project, you agree to license your contribu
    :alt: Piper logo
 
 
-.. |Contribs| image:: https://tatradev.com
-   :target: https://github.com/TatraDev/piper/graphs/contributors
+.. |Contribs| image:: https://img.shields.io/badge/contributors-3-brightgreen
+   :target: https://github.com/TatraDev/pipertool/graphs/contributors
    :alt: Contributors
+
+.. |Python Version| image:: https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10-blue
+   :target: https://pypi.org/project/pipertool
+   :alt: Python Version
+
+.. |Release| image:: https://img.shields.io/badge/release-v0.0.2-blue
+   :target: https://github.com/TatraDev/pipertool/releases
+
+.. |Build status| image:: https://github.com/TatraDev/pipertool/workflows/test/badge.svg?branch=venv_logic&event=push
+    :target: https://github.com/TatraDev/pipertool/actions?query=workflow%3Atest
+
+.. |Contact| image:: https://img.shields.io/badge/telegram-write%20me-blue.svg
+    :target:  https://t.me/pipertool
+
+.. |Blog| image:: https://img.shields.io/badge/site-my%20blog-yellow.svg
+    :target:  https://tatradev.com
+
+.. |License| image:: https://img.shields.io/badge/License-Apache_2.0-yellow.svg
+    :target:  https://www.apache.org/licenses/LICENSE-2.0
+
+.. |Docs| image:: https://img.shields.io/badge/docs-passing-brightgreen
+    :target:  http://pipertool.org

main.py

@@ -5,7 +5,8 @@
 from loguru import logger
 
 from piper.configurations import get_configuration
-from piper.envs import CurrentEnv, DockerEnv, VirtualEnv
+from piper.envs import ComposeEnv, CurrentEnv, DockerEnv, VirtualEnv
+
 # from piper.services import (SpacyNER, StringValue, TesseractRecognizer,
 #                             TestMessageAdder)
 # from piper.utils import tesrct_utils as tu
@@ -21,7 +22,7 @@
     #     result = loop.run_until_complete(adder(x))
     #     print(result)
 
-    # x = StringValue(value="hello, world") 
+    # x = StringValue(value="hello, world")
     # adder = TestMessageAdder(appender="!", port=cfg.docker_app_port)
     # result = loop.run_until_complete(adder(x))
     # print(result)
@@ -37,11 +38,21 @@
     #     result = loop.run_until_complete(recognizer())
     #     logger.info(f'result of recognition is {result}')
 
-    with VirtualEnv() as env:
-        env.copy_struct_project()
-        env.create_files_for_venv()
-        env.create_files_for_tests()
-        
+    # with VirtualEnv() as env:
+    #     env.copy_struct_project()
+    #     env.create_files_for_venv()
+    #     env.create_files_for_tests()
+
+    with ComposeEnv() as env:
+        try:
+            env.copy_struct_project()
+            env.create_files_for_compose(testing=True)
+            env.start_compose()
+        except KeyboardInterrupt:
+            logger.info('Ctrl+C pressed. Except KeyboardInterrupt.')
+            env.stop_compose()
+            sys.exit(1)
+
         # sys.exit()
 
         # sn = SpacyNER()
@@ -55,4 +66,4 @@
         #         result1_str = "\n".join(str(x) for x in result1)
         #         logger.info(f'result of NER for model {avalable_model} is {result1_str}')
         #     else:
-        #         logger.info(f'module didn`t get NER data')
+        #         logger.info(f'module didn`t get NER data')

piper/__init__.py

@@ -0,0 +1,34 @@
+import sys
+
+import click
+
+from piper.envs import ComposeEnv
+
+
+@click.command()
+@click.argument('type_command')
+@click.option(
+    '--env-type', '-e',
+    help='your current interpretation',
+)
+def main(type_command: str, env_type: str):
+    if env_type == 'compose':
+        if type_command == 'start':
+            print("type_command == 'start'")
+            with ComposeEnv() as env:
+                env.copy_struct_project()
+                env.create_files_for_compose()
+                env.start_compose()
+        elif type_command == 'stop':
+            print("type_command == 'stop'")
+            with ComposeEnv() as env:
+                env.stop_compose()
+    else:
+        raise NotImplementedError(f'{env_type} not released in this version pipertool')
+
+
+if __name__ == '__main__':
+    args = sys.argv
+    if "--help" in args or len(args) == 1:
+        print("CVE")
+    main()

piper/__main__.py

@@ -0,0 +1,79 @@
+import inspect
+import subprocess
+from typing import Dict
+
+from piper.base.docker_compose.utils import ComposeServices
+from piper.configurations import get_configuration
+from piper.utils.logger_utils import logger
+
+
+def copy_piper(path: str):
+    cfg = get_configuration()
+    from distutils.dir_util import copy_tree
+    copy_tree(cfg.piper_path, f"{path}/piper")
+
+
+def copy_scripts(path: str, scripts: Dict[str, str]):
+    for script_name, script_path in scripts.items():
+        with open(f"{path}/{script_name}.py", "w") as output:
+            with open(script_path, "r") as current_file:
+                output.write(current_file.read())
+
+
+def write_requirements(path, requirements):
+    with open(f"{path}/requirements.txt", "w") as output:
+        output.write("\n".join(requirements))
+
+
+class ComposeExecutor:
+    requirements = ["gunicorn", "fastapi", "uvicorn", "aiohttp", "Jinja2", "pydantic", "pymilvus", "numpy", "loguru"]
+
+    def __init__(self):
+        logger.info('ComposeExecutor init with is_compose_env()')
+
+        cfg = get_configuration()
+        self.project_output_path = cfg.path
+
+    def scripts(self):
+        return {"service": inspect.getfile(self.__class__)}
+
+    def copy_struct_project(self):
+        copy_piper(self.project_output_path)
+        copy_scripts(self.project_output_path, self.scripts())
+
+    def create_files_for_compose(self, testing: bool = False):
+        logger.info('ComposeExecutor create_fast_api_files_venv()')
+
+        compose_service = ComposeServices(
+            name_path=self.project_output_path,
+        )
+
+        main_fastapi = compose_service.render_script_fastapi()
+        with open(f"{self.project_output_path}/main.py", "w") as output:
+            output.write(main_fastapi)
+
+        docker_compose = compose_service.render_compose_services()
+        with open(f"{self.project_output_path}/docker-compose.yaml", "w") as output:
+            output.write(docker_compose)
+
+        bash_start = compose_service.render_bash_start(testing=testing)
+        with open(f"{self.project_output_path}/bash-start.sh", "w") as output:
+            output.write(bash_start)
+
+        bash_stop = compose_service.render_bash_stop()
+        with open(f"{self.project_output_path}/bash-stop.sh", "w") as output:
+            output.write(bash_stop)
+
+        dockerfile = compose_service.render_dockerfile()
+        with open(f"{self.project_output_path}/Dockerfile", "w") as output:
+            output.write(dockerfile)
+
+        write_requirements(self.project_output_path, self.requirements)
+
+    def start_compose(self):
+        process_chmod_start = subprocess.run(f'chmod +x {self.project_output_path}bash-start.sh', shell=True)
+        process_run = subprocess.run(f'{self.project_output_path}bash-start.sh', shell=True)
+
+    def stop_compose(self):
+        process_chmod_stop = subprocess.run(f'chmod +x {self.project_output_path}bash-stop.sh', shell=True)
+        process_run = subprocess.run(f'{self.project_output_path}bash-stop.sh', shell=True)

piper/base/docker_compose/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+sudo docker-compose -f {{ name_path }}docker-compose.yaml up --build {{ compose_flag }}