Merge pull request #152 from WikiMovimentoBrasil/2025-01-csv-report

feat: basic csv report of finished batches

Author: mgalves

src/core/models.py

@@ -1,4 +1,5 @@
 import copy
+import csv
 import logging
 import jsonpatch
 from typing import Optional
@@ -201,6 +202,10 @@ def is_initial_or_running(self):
     def is_preview_initial_or_running(self):
         return self.is_preview or self.is_initial or self.is_running
 
+    @property
+    def is_done(self):
+        return self.status == Batch.STATUS_DONE
+
     def add_preview_command(self, preview_command: "BatchCommand") -> bool:
         if not hasattr(self, "_preview_commands"):
             self._preview_commands = []
@@ -234,6 +239,43 @@ def wikibase_url(self):
         """
         return settings.BASE_REST_URL.replace("https://", "http://").split("/w/rest.php")[0]
 
+    # ------
+    # REPORT
+    # ------
+
+    def write_report(self, csvfile):
+        """
+        Uses `csvfile` as the csv writer's file to write the batch report.
+        """
+        writer = csv.writer(csvfile)
+        writer.writerow(
+            [
+                "batch_id",
+                "index",
+                "operation",
+                "status",
+                "error",
+                "message",
+                "entity_id",
+                "raw_input",
+                "api_response",
+            ]
+        )
+        for cmd in self.commands():
+            called_api = bool(cmd.response_json)
+            writer.writerow(
+                [
+                    self.pk,
+                    cmd.index,
+                    cmd.operation,
+                    cmd.get_status_display(),
+                    cmd.error,
+                    cmd.message,
+                    cmd.entity_id(),
+                    cmd.raw.replace("\t", "|"), # tabs are weird in csv
+                    cmd.response_json if called_api else None,
+                ]
+            )
 
 class BatchCommand(models.Model):
     """

src/core/tests/test_api.py

@@ -180,6 +180,14 @@ def create_item_failed_server(cls, mocker):
             status_code=500,
         )
 
+    @classmethod
+    def patch_item_successful(cls, mocker, item_id, json_result):
+        mocker.patch(
+            cls.wikibase_url(f"/entities/items/{item_id}"),
+            json=json_result,
+            status_code=200,
+        )
+
     @classmethod
     def add_statement_successful(cls, mocker, item_id):
         mocker.patch(

src/web/templates/batch.html

@@ -30,6 +30,10 @@ <h2> {% translate "Batch" %} #{{ batch.pk }} <img id="spinner" class="htmx-indic
       {% csrf_token %}
       <input type="submit" value="Restart">
     </form>
+    {% elif batch.is_done %}
+    <form method="GET" action="{% url 'batch_report' pk=batch.pk %}">
+      <input type="submit" value="Report">
+    </form>
     {% endif %}
   {% endif %}
 </div>

src/web/tests/test_views.py

@@ -18,6 +18,7 @@
 
 class ViewsTest(TestCase):
     URL_NAME = "profile"
+    maxDiff = None
 
     def assertInRes(self, substring, response):
         """Checks if a substring is contained in response content"""
@@ -560,3 +561,44 @@ def test_batch_preview_commands(self, mocker):
         self.assertInRes("Save and run batch", res)
         res = self.client.get("/batch/new/preview/commands/")
         self.assertEqual(res.status_code, 200)
+
+    @requests_mock.Mocker()
+    def test_batch_report(self, mocker):
+        ApiMocker.is_autoconfirmed(mocker)
+        ApiMocker.wikidata_property_data_types(mocker)
+        ApiMocker.property_data_type(mocker, "P2", "wikibase-item")
+        ApiMocker.item_empty(mocker, "Q1234")
+        ApiMocker.item_empty(mocker, "Q11")
+        ApiMocker.patch_item_successful(mocker, "Q1234", {"id": "Q1234$stuff"})
+        ApiMocker.patch_item_successful(mocker, "Q11", {"id": "Q11", "labels": {"en": "label"}})
+        user, api_client = self.login_user_and_get_token("wikiuser")
+        parser = V1CommandParser()
+        batch = parser.parse("Batch", "wikiuser", """Q1234\tP2\tQ1||Q11|Len|"label" """)
+        batch.save_batch_and_preview_commands()
+        pk = batch.pk
+
+        response = self.client.get(f"/batch/{pk}/")
+        self.assertEqual(response.status_code, 200)
+        self.assertTemplateUsed("batch.html")
+        self.assertNotInRes(f"""<form method="GET" action="/batch/{pk}/report/">""", response)
+
+        batch.run()
+
+        response = self.client.get(f"/batch/{pk}/")
+        self.assertEqual(response.status_code, 200)
+        self.assertTemplateUsed("batch.html")
+        self.assertInRes(f"""<form method="GET" action="/batch/{pk}/report/">""", response)
+
+        response = self.client.post(f"/batch/{pk}/report/")
+        self.assertEqual(response.status_code, 405)
+
+        response = self.client.get(f"/batch/{pk}/report/")
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.headers["Content-Disposition"], f'attachment; filename="batch-{pk}-report.csv"')
+        result = (
+            """b'batch_id,index,operation,status,error,message,entity_id,raw_input,api_response\\r\\n"""
+            """1,0,set_statement,Done,,,Q1234,Q1234|P2|Q1,{\\\'id\\\': \\\'Q1234$stuff\\\'}\\r\\n"""
+            """1,1,set_label,Done,,,Q11,"Q11|Len|""label"" ","""
+            """"{\\\'id\\\': \\\'Q11\\\', \\\'labels\\\': {\\\'en\\\': \\\'label\\\'}}"\\r\\n\'"""
+        )
+        self.assertEqual(result, str(response.content).strip())

src/web/urls.py

@@ -9,6 +9,7 @@
 from .views.batch import batch_commands
 from .views.batch import batch_stop
 from .views.batch import batch_restart
+from .views.batch import batch_report
 from .views.batch import batch_summary
 from .views.batches import home
 from .views.batches import last_batches
@@ -34,6 +35,7 @@
     path("batch/preview/", batch, name="batch_preview"),
     path("batch/<int:pk>/stop/", batch_stop, name="batch_stop"),
     path("batch/<int:pk>/restart/", batch_restart, name="batch_restart"),
+    path("batch/<int:pk>/report/", batch_report, name="batch_report"),
     path("batch/<int:pk>/summary/", batch_summary, name="batch_summary"),
     path("batch/<int:pk>/commands/", batch_commands, name="batch_commands"),
     path("batch/new/", new_batch, name="new_batch"),

src/web/views/batch.py

@@ -3,6 +3,8 @@
 from django.shortcuts import render
 from django.urls import reverse
 from django.views.decorators.http import require_http_methods
+from django.views.decorators.http import require_GET
+from django.http import HttpResponse
 
 from core.client import Client
 from core.models import Batch
@@ -81,6 +83,23 @@ def batch_restart(request, pk):
     except Batch.DoesNotExist:
         return render(request, "batch_not_found.html", {"pk": pk}, status=404)
 
+@require_GET
+def batch_report(request, pk):
+    try:
+        batch = Batch.objects.get(pk=pk)
+        current_owner = request.user.is_authenticated and request.user.username == batch.user
+        if current_owner and batch.is_done:
+            res = HttpResponse(
+                content_type="text/csv",
+                headers={"Content-Disposition": f'attachment; filename="batch-{pk}-report.csv"'},
+            )
+            batch.write_report(res)
+            return res
+        else:
+            return render(request, "batch_not_found.html", {"pk": pk}, status=404)
+    except Batch.DoesNotExist:
+        return render(request, "batch_not_found.html", {"pk": pk}, status=404)
+
 
 @require_http_methods(
     [