Add tests for tern-rest-api reports implementation

Kairo de Araujo · Kairo de Araujo · commit 45d834203865 · 2022-03-18T16:51:08.000+01:00
This commit adds all the tern-rest-api reports implementation tests and
adjusts the repository to use Docker.
diff --git a/.coveragerc b/.coveragerc
@@ -0,0 +1,2 @@
+[run]
+omit = *tests*,app.py
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -22,4 +22,4 @@ jobs:
       run: pip install tox tox-gh-actions
 
     - name: Run Python tests
-      run: tox
+      run: tox -r
diff --git a/Dockerfile b/Dockerfile
@@ -16,14 +16,15 @@ RUN echo "deb http://deb.debian.org/debian bullseye main" > /etc/apt/sources.lis
     skopeo \
     && rm -rf /var/lib/apt/lists/*
 
+COPY requirements.txt requirements.txt
+RUN pip install --no-cache -r ./requirements.txt
+
 RUN mkdir /opt/tern-rest-api
 
 ADD . /opt/tern-rest-api
 WORKDIR /opt/tern-rest-api
-RUN pip install --no-cache -r ./requirements.txt
 
-ENV TERN_API_CACHE_DIR=/var/opt/tern-rest-api/cached
+ENV TERN_API_CACHE_DIR=/var/opt/tern-rest-api/cache
 ENV TERN_DEFAULT_REGISTRY="registry.hub.docker.com"
 
-ENV FLASK_APP=/opt/tern-rest-api/app.py
-CMD ["bash", "docker_start.sh"]
+ENTRYPOINT [ "bash", "docker_start.sh" ]
diff --git a/Makefile b/Makefile
@@ -5,7 +5,17 @@
 # SPDX-License-Identifier: BSD-2-Clause
 
 build-dev:
-	docker build -t tern-rest-api:dev .
+	docker-compose build --force-rm tern-rest-api
 
 serve-dev: build-dev
-	docker run --rm --name tern-rest-api -e ENVIRONMENT=DEVELOPMENT --privileged -v /var/run/docker.sock:/var/run/docker.sock -v $(PWD):/opt/tern-rest-api -p 5001:80 tern-rest-api:dev
+	docker-compose up --remove-orphans
+
+tests: build-dev
+	docker-compose run --rm --volume=$(PWD):/opt/tern-rest-api --entrypoint="/bin/sh" tern-rest-api -c 'pip install tox && tox'
+
+stop:
+	docker-compose down -v
+
+update-requirements:
+	pipenv lock -r > requirements.txt
+	pipenv lock -r -d > requirements-dev.txt
diff --git a/README.md b/README.md
@@ -68,6 +68,8 @@ Open http://localhost/ in your browser.
 Changing the source code will automatically reload the server inside the
 container and makes the development easier.
 
+You can stop the sever using ``Ctrl+C`` and running ``make stop``
+
 ## On your local machine
 Runing the API locally
 
@@ -81,7 +83,12 @@ Open http://localhost:5000/ in your browser.
 
 We use [Tox](https://tox.wiki/en/latest/) to manage running the tests.
 
-Running tests
+## As a Docker Container
+```shell
+$ make tests
+```
+
+## On your local machine
 ```shell
 $ tox
 ```
@@ -106,6 +113,5 @@ $ pipenv update
 
 Updating the ``requirements.txt`` and ``requirements-dev.txt``
 ```shell
-$ pipenv lock -r > requirements.txt
-$ pipenv lock -r -d > requirements-dev.txt
+$ make update-requirements
 ```
diff --git a/app.py b/app.py
@@ -10,7 +10,7 @@
 
 from tern_api import __version__, tern_app
 from tern_api.api.v1.common_models import api_models_namespace
-from tern_api.api.v1.reports import ns as report_v1
+from tern_api.api.v1.reports import ns as reports_v1
 from tern_api.api.v1.version import ns as version_v1
 
 logging.basicConfig(
@@ -35,7 +35,7 @@
 
 api.add_namespace(api_models_namespace)
 api.add_namespace(version_v1, path="/api/v1/version")
-api.add_namespace(report_v1, path="/api/v1/report")
+api.add_namespace(reports_v1, path="/api/v1/reports")
 
 
 def export_swagger_json(filepath):
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -0,0 +1,21 @@
+version: '3'
+
+volumes:
+  tern-cache:
+
+services:
+  tern-rest-api:
+    build:
+      context: .
+      args:
+        DEVEL: "yes"
+    command: "bash bash_start.sh"
+    environment:
+      - ENVIRONMENT=development
+      - TERN_DEFAULT_REGISTRY=registry.hub.docker.com
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - tern-cache:/var/opt/tern-rest-api/cache:z
+      - ./tern_api:/opt/tern-rest-api/tern_api:z
+    ports:
+      - "5001:80"
diff --git a/docker_start.sh b/docker_start.sh
@@ -1,7 +1,8 @@
 #!/bin/bash
 
 if [[ ${ENVIRONMENT^^} == "DEVELOPMENT" ]]; then
-    flask run --reload --debugger -h 0.0.0.0 -p 80
+    echo "Starting tern-rest-api in development mode"
+    gunicorn --reload -b 0.0.0.0:80 app:tern_app
 else
-    gunicorn --workers=1 -b 0.0.0.0:80 app:tern_api
+    gunicorn -b 0.0.0.0:80 app:tern_app
 fi
diff --git a/tern_api/__init__.py b/tern_api/__init__.py
@@ -13,3 +13,4 @@
 tern_app.config["TERN_DEFAULT_REGISTRY"] = os.getenv("TERN_DEFAULT_REGISTRY")
 
 tern_tasks = Executor(tern_app)
+tern_app.config["TERN_TASKS"] = tern_tasks
diff --git a/tern_api/api/v1/reports.py b/tern_api/api/v1/reports.py
@@ -6,7 +6,8 @@
 from flask import request
 from flask_restx import Namespace, Resource, fields
 
-from tern_api import reports, tern_app
+from tern_api import tern_app, constants
+from tern_api.reports import submit, status
 from tern_api.api.v1.common_models import (
     async_response_model,
     error_model,
@@ -53,14 +54,14 @@ class Report(Resource):
     )
 
     @ns.response(200, "OK", report_response_request)
-    @ns.expect(report_parameters)
+    @ns.expect(report_parameters, validate=True)
     def post(self):
         """Tern BoM report
 
         **Note**: This request will be processed assynchronous.
         """
         payload = request.json
-        response = reports.request(payload)
+        response = submit(payload)
         return response.to_response()
 
 
@@ -79,13 +80,28 @@ class ReportStatus(Resource):
     data_status_response = ns.model(
         "data_status_response",
         {
+            "cache": fields.Boolean(
+                description="Requested using cache?",
+                required=True,
+                example=True,
+            ),
+            "id": fields.String(
+                description="Unique Identification for request",
+                required=False,
+                example="19f035a711644eab84ef5a38ceb5572e",
+            ),
+            "message": fields.String(
+                description="Message",
+                required=False,
+                exampple="Request is running",
+            ),
+            "report": fields.Nested(report_model),
             "status": fields.String(
                 description="Status of request",
                 required=True,
-                example="FINISH",
-                enum=["UNKNOWN", "FINISH", "RUNNING", "FAIL"],
+                example=constants.task_status.SUCCESS.value,
+                enum=[s.value for s in constants.task_status],
             ),
-            "result": fields.Nested(report_model),
         },
     )
     report_status_response = ns.model(
@@ -97,10 +113,10 @@ class ReportStatus(Resource):
     )
 
     @ns.response(200, "OK", report_status_response)
-    @ns.expect(report_status_parameters)
+    @ns.expect(report_status_parameters, validate=True)
     def post(self):
         """Request Tern BoM report status/result"""
 
         payload = request.json
-        response = reports.status(payload.get("id"))
+        response = status(payload.get("id"))
         return response.to_response()
diff --git a/tern_api/reports.py b/tern_api/reports.py
@@ -50,6 +50,9 @@ def tern(command: list) -> Dict[str, Any]:
     """
     logging.debug(command)
 
+    if type(command) != list:
+        raise TypeError("command must be a list")
+
     tern_cmd = subprocess.run(command, capture_output=True)
     if tern_cmd.stdout:
         json_report = json.loads(tern_cmd.stdout)
@@ -61,7 +64,7 @@ def tern(command: list) -> Dict[str, Any]:
             # Is important to return a specific error if the error comes from
             # tern CLI, it means the task FINISH, for example, invalid image.
             logging.info(tern_cmd.stderr)
-            raise TernError(tern_cmd.stderr)
+            raise TernError(tern_cmd.stderr.decode())
 
 
 @tern_tasks.job
@@ -79,7 +82,6 @@ def tern_report(
     Return:
         Report as Dictionary
     """
-
     if cache:
         # If a API user is using the cache, first try to load the cached
         # instead doing a new call to the tern.
@@ -88,6 +90,7 @@ def tern_report(
                 report = json.load(f)
             return report
         except FileNotFoundError:
+            logging.debug(f"Cache file not found: {cache_file}")
             # call the tern and dump it to the cache
             report = tern(command)
             with open(cache_file, "w") as f:
@@ -98,7 +101,7 @@ def tern_report(
     return report
 
 
-def request(payload: dict) -> TernAPIResponse:
+def submit(payload: dict) -> TernAPIResponse:
     """
     Get the Payload from API and prepare to request the report.
     The request will be handled in the background as tern tasks.
@@ -111,7 +114,6 @@ def request(payload: dict) -> TernAPIResponse:
     """
     TERN_API_CACHE_DIR = tern_app.config["TERN_API_CACHE_DIR"]
     TERN_DEFAULT_REGISTRY = tern_app.config["TERN_DEFAULT_REGISTRY"]
-
     task_id = uuid4().hex
     registry = payload.get(
         "registry",
@@ -131,7 +133,7 @@ def request(payload: dict) -> TernAPIResponse:
         registry_image_tag = f"{image}:{tag}"
 
     command = ["tern", "report", "-i", registry_image_tag, "-f", "json"]
-    logging.info(command)
+    logging.debug(command)
 
     tern_report.submit_stored(
         task_id, command=command, cache=cache, cache_file=cache_file
@@ -146,10 +148,10 @@ def status(task_id: str) -> TernAPIResponse:
     Request to the status/result from the tern tasks.
 
     The tern tasks can have basically the following status:
-    - UNKNOWN: Not known (yet) by the task manager (initial status).
+    - PENDING: Not known (yet) by the task manager (initial status).
     - RUNNING: Task is running by the task manager.
-    - FINISH : Task has fineshed in the task manager.
-    - FAIL   : Task has failed before finished.
+    - SUCCESS: Task has fineshed in the task manager.
+    - FAILURE: Task has failed before finished.
 
     Args:
         task_id: the unique task ID
@@ -159,9 +161,16 @@ def status(task_id: str) -> TernAPIResponse:
     try:
         if not tern_tasks.futures.done(task_id):
             status = tern_tasks.futures._state(task_id)
-            if status:
+            if status == task_status.RUNNING.value:
                 data_response.status = status
 
+            else:
+                # if the task state is not done, then the task is still running
+                # we can't know the status, so we return the status as pending
+                # and remove the task from the task manager
+                data_response.status = task_status.PENDING.value
+                tern_tasks.futures.pop(task_id)
+
             return TernAPIResponse(data_response.to_dict())
 
         report = tern_tasks.futures.pop(task_id)
@@ -181,7 +190,7 @@ def status(task_id: str) -> TernAPIResponse:
     except:  # noqa
         data_response.status = task_status.FAILURE.value
         data_response.message = (
-            f"Task couln't finish due: {str(sys.exc_info())}"
+            f"Task could not finish due: {str(sys.exc_info())}"
         )
 
     return TernAPIResponse(data_response.to_dict())
diff --git a/tern_api/utils.py b/tern_api/utils.py
@@ -17,14 +17,6 @@ class TernAPIResponse:
     status_code: int = 200
     errors: Dict[str, Any] = field(default_factory=dict)
 
-    def to_dict(self) -> Dict[str, Any]:
-        """Converts the dataclass data to a dictionary.
-
-        :return: Response data as a dictionary
-        :rtype: ``dict``
-        """
-        return asdict(self)
-
     def to_response(self) -> Response:
         """Converts the dataclass data to a Flask jsonified format, building a
         consistent response format for the requests to the API.
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -10,11 +10,21 @@
 from tests.utils import RequestDataTest
 
 
+@pytest.fixture(scope="module")
+def test_tern_app():
+    # Configuration
+    tern_app.config["TERN_API_CACHE_DIR"] = "FakeCacheDir"
+    tern_app.config["TERN_DEFAULT_REGISTRY"] = "registry_fake_tests"
+
+    return tern_app
+
+
 @pytest.fixture
-def api_request():
+def api_request(test_tern_app):
     def _api_request(request_data: RequestDataTest) -> TestResponse:
-        with tern_app.test_client() as api_client:
-            with tern_app.app_context():
+        with test_tern_app.test_client() as api_client:
+            with test_tern_app.app_context():
+
                 if request_data.method.lower() == "get":
                     response = api_client.get(
                         request_data.endpoint, json=request_data.payload
@@ -41,3 +51,13 @@ def _api_request(request_data: RequestDataTest) -> TestResponse:
                 return response
 
     return _api_request
+
+
+@pytest.fixture
+def fake_id():
+    class FakeID:
+        @property
+        def hex(self):
+            return "fake-id"
+
+    return FakeID
diff --git a/tests/tern_api/test_reports.py b/tests/tern_api/test_reports.py
diff --git a/tests/tern_api/test_utils.py b/tests/tern_api/test_utils.py
diff --git a/tests/tern_api/v1/test_reports.py b/tests/tern_api/v1/test_reports.py
diff --git a/tests/tern_api/v1/test_version.py b/tests/tern_api/v1/test_version.py
diff --git a/tox.ini b/tox.ini

Original file line number	Diff line number	Diff line change
`@@ -13,3 +13,4 @@`
`13`	`13`	`tern_app.config["TERN_DEFAULT_REGISTRY"] = os.getenv("TERN_DEFAULT_REGISTRY")`
`14`	`14`
`15`	`15`	`tern_tasks = Executor(tern_app)`
	`16`	`+tern_app.config["TERN_TASKS"] = tern_tasks`