[ENG-18631] track test runs against staging (#48)

* update test deps

* adds ddtrace

* adds test_verified_translate

* fix action triggers

Author: tgallant

.github/workflows/pythonapp.yml

@@ -2,10 +2,15 @@ name: Pytest Tests
 env:
   API_HOST: ${{ vars.API_HOST }}
   API_KEY: ${{ secrets.API_KEY }}
+  DD_API_KEY: ${{ secrets.DD_API_KEY }}
+  DD_SITE: us5.datadoghq.com
+  DD_ENV: staging
+  DD_SERVICE: lilt-python
+  DD_CIVISIBILITY_AGENTLESS_ENABLED: true
 
 on:
   push:
-    branches: [ master, openapi-bindings ]
+    branches: [ master ]
   pull_request:
     branches: [ master, openapi-bindings ]
 
@@ -25,14 +30,15 @@ jobs:
         python -m pip install --upgrade pip
         pip install -r requirements.txt
         pip install -r test-requirements.txt
+    - name: Install datadog tracing library
+      run: |
+        pip install -U ddtrace
     - name: Lint with flake8
       run: |
-        pip install flake8
         # stop the build if there are Python syntax errors or undefined names
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Test with pytest
       run: |
-        pip install pytest
-        pytest workflow_tests
+        pytest workflow_tests --ddtrace

test-requirements.txt

@@ -1,5 +1,6 @@
-pytest~=7.1.2
+pytest~=8.3.3
 pytest-cov>=2.8.1
 pytest-randomly==1.2.3 # needed for python 2.7+3.4
 python-dotenv==1.0.1
 tenacity==8.5.0
+flake8

workflow_tests/test_verified_translate.py

@@ -0,0 +1,108 @@
+from dotenv import load_dotenv
+
+import os
+from io import BytesIO
+from zipfile import ZipFile
+
+import pytest
+import lilt
+
+from tenacity import (
+    retry,
+    stop_after_delay,
+    wait_exponential,
+    retry_if_result,
+)
+
+load_dotenv()
+
+
+@pytest.fixture(scope="module")
+def client():
+    configuration = lilt.Configuration(
+        host=os.environ["API_HOST"], api_key={"key": os.environ["API_KEY"]}
+    )
+    api_client = lilt.ApiClient(configuration)
+    commit = os.environ.get("GIT_COMMIT_SHA", "no_version_available")
+    api_client.user_agent = f"lilt-python-e2e-tests/{commit}"
+    return api_client
+
+
+@pytest.fixture(scope="module")
+def api(client):
+    return lilt.JobsApi(client)
+
+
+@pytest.fixture(scope="module")
+def uploaded_file(client):
+    api = lilt.FilesApi(client)
+    name = "test_file.txt"
+    body = b"hello world"
+    response = api.upload_file(name, body)
+    yield response
+    api.delete_file(response.id)
+
+
+@pytest.fixture(scope="module")
+def memory(client):
+    api = lilt.MemoriesApi(client)
+    body = lilt.MemoryCreateParameters(
+        name="test_memory",
+        srclang="en",
+        srclocale="US",
+        trglang="de",
+        trglocale="DE",
+    )
+    response = api.create_memory(body)
+    yield response
+    api.delete_memory(response.id)
+
+
+@pytest.fixture(scope="module")
+def job(api, memory, uploaded_file):
+    lang = lilt.LanguagePair(trg_lang=memory.trglang, memory_id=memory.id)
+    body = lilt.JobCreateParameters(
+        name="test_job",
+        src_lang="en",
+        src_locale="US",
+        language_pairs=[lang],
+        file_ids=[uploaded_file.id],
+    )
+    response = api.create_job(body)
+    yield response
+    api.delete_job(response.id)
+
+
+def test_verified_translate_monitor_job_status(api, job):
+    api.deliver_job(job.id)
+    jobs = api.retrieve_all_jobs(is_delivered="true", is_archived="false")
+    assert len(jobs) > 0
+
+
+@retry(
+    retry=retry_if_result(lambda is_processing: is_processing != 0),
+    stop=stop_after_delay(2 * 60),
+    wait=wait_exponential(),
+)
+def wait_for_job_export(api, job_id):
+    job = api.get_job(job_id)
+    return job.is_processing
+
+
+def test_verified_translate_download_job(api, job):
+    api.export_job(job.id, type="files")
+    wait_for_job_export(api, job.id)
+    response = api.download_job(job.id, _preload_content=False)
+    zip_file = BytesIO(response.data)
+    with ZipFile(zip_file) as zip_ref:
+        contents = zip_ref.infolist()
+        assert len(contents) == 2
+        translated_file = contents[1]
+        with zip_ref.open(translated_file) as tf:
+            assert tf.read() == b"hello world"
+
+
+def test_verified_translate_archive_job(api, job):
+    api.archive_job(job.id)
+    updated_job = api.get_job(job.id)
+    assert updated_job.status == "archived"