Merge pull request #399 from wikimediabrasil/create-property

feat: CREATE_PROPERTY && remove wikibase from docker-compose

Author: arcstur

docker-compose.yml

@@ -25,11 +25,6 @@ x-application-service: &application-service
     - ./translations:/root/www/python/translations
     - qs-static-data:/root/www/python/static
 
-x-wikibase-volumes: &wikibase-volumes
-  volumes:
-    - wikibase-config-data:/config
-    - wikibase-static-data:/var/www/html/images
-
 services:
   mariadb:
     image: mariadb:11.7.2
@@ -79,54 +74,6 @@ services:
       app:
         condition: service_healthy
 
-  wikibase:
-    <<: *wikibase-volumes
-    image: wikibase/wikibase:4.0
-    ports:
-      - 8888:80
-    environment:
-      MW_ADMIN_NAME: "${WIKIBASE_ADMIN_USERNAME:-admin}"
-      MW_ADMIN_PASS: "${WIKIBASE_ADMIN_PASSWORD:-wikibase_admin_pass}"
-      MW_ADMIN_EMAIL: "${WIKIBASE_ADMIN_EMAIL:-admin@wikibase.example}"
-      MW_WG_SERVER: http://localhost:8888
-      DB_SERVER: wikibase-db:3306
-      DB_NAME: wikibase
-      DB_USER: wikibase
-      DB_PASS: wikibase_pass
-    healthcheck:
-      test: curl --silent --fail localhost/wiki/Main_Page
-      interval: 10s
-      start_period: 5m
-    depends_on:
-      wikibase-db:
-        condition: service_healthy
-
-  wikibase-jobrunner:
-    <<: *wikibase-volumes
-    image: wikibase/wikibase:4.0
-    command: /jobrunner-entrypoint.sh
-    depends_on:
-      wikibase:
-        condition: service_healthy
-
-  wikibase-db:
-    image: mariadb:10.11
-    environment:
-      MARIADB_DATABASE: wikibase
-      MARIADB_USER: wikibase
-      MARIADB_PASSWORD: wikibase_pass
-      MARIADB_RANDOM_ROOT_PASSWORD: "true"
-    healthcheck:
-      test: healthcheck.sh --connect --innodb_initialized
-      start_period: 1m
-      interval: 20s
-      timeout: 5s
-    volumes:
-      - wikibase-db-data:/var/lib/mysql
-
 volumes:
   qs-mariadb-data:
   qs-static-data:
-  wikibase-config-data:
-  wikibase-static-data:
-  wikibase-db-data:

src/core/models.py

@@ -1,4 +1,5 @@
 import copy
+import traceback
 import csv
 import logging
 import math
@@ -214,13 +215,16 @@ def get_property_value_type(self, property_id):
         Uses a dictionary attribute for caching.
         """
         endpoint = f"/entities/properties/{property_id}"
-        url = self.wikibase_url(endpoint)
 
         try:
-            res = self.session.get(url).json()
+            res = self.wikibase_request_wrapper("get", endpoint, {})
             data_type = res["data_type"]
         except (KeyError, UserError):
             raise NonexistantPropertyOrNoDataType(property_id)
+        except Exception as e:
+            logger.error(f"Error while trying to get data type: {e}")
+            traceback.print_exc()
+            raise NonexistantPropertyOrNoDataType(property_id)
 
         try:
             value_type = self.data_type_to_value_type(data_type)
@@ -285,6 +289,8 @@ def wikibase_entity_endpoint(entity_id, entity_endpoint=""):
             base = "/entities/items"
         elif entity_id.startswith("P"):
             base = "/entities/properties"
+        elif entity_id.upper() == "LAST":
+            raise LastCouldNotBeEvaluated()
         else:
             raise EntityTypeNotImplemented(entity_id)
 
@@ -985,7 +991,7 @@ def start(self):
     def finish(self):
         logger.info(f"[{self}] finished")
         self.status = BatchCommand.STATUS_DONE
-        if self.is_id_last_or_create_item():
+        if self.is_entity_creation():
             self.set_entity_id(self.response_id)
         self.save()
         self.propagate_to_previous_commands()
@@ -999,6 +1005,7 @@ def error_with_value(self, value: Error, message: str = None):
 
     def error_with_exception(self, exception: Exception):
         message = getattr(exception, "message", str(exception))
+        traceback.print_exc()
         self.error_with_message(message)
 
     def error_with_message(self, message):
@@ -1015,7 +1022,7 @@ def propagate_to_previous_commands(self):
             if self.is_error_status():
                 cmd.error = self.Error.COMBINING_COMMAND_FAILED
                 cmd.message = cmd.error.label
-            elif cmd.is_id_last_or_create_item():
+            elif cmd.is_entity_creation():
                 cmd.set_entity_id(self.entity_id)
             cmd.save()
 
@@ -1084,6 +1091,10 @@ def sitelink(self):
     def what(self):
         return self.json.get("what", "").upper()
 
+    @property
+    def property_data_type(self):
+        return self.json.get("data", "").lower()
+
     @property
     def related_identifiers_set(self):
         """
@@ -1337,11 +1348,27 @@ def is_sitelink_command(self):
     def is_error_status(self):
         return self.status == BatchCommand.STATUS_ERROR
 
-    def is_not_create_item(self):
-        return self.operation != self.Operation.CREATE_ITEM
+    def is_not_create_entity(self):
+        return self.operation not in [self.Operation.CREATE_ITEM, self.Operation.CREATE_PROPERTY]
 
-    def is_id_last_or_create_item(self):
-        return self.entity_id == "LAST" or self.operation == self.Operation.CREATE_ITEM
+    def get_first_command_operation(self):
+        first_command = self
+        previous = getattr(self, "previous_commands", [])
+        if len(previous) > 0:
+            first_command = previous[-1]
+        return first_command.operation
+
+    def is_item_creation(self):
+        operation = self.get_first_command_operation()
+        return operation == self.Operation.CREATE_ITEM
+
+    def is_property_creation(self):
+        operation = self.get_first_command_operation()
+        return operation == self.Operation.CREATE_PROPERTY
+
+    def is_entity_creation(self):
+        operation = self.get_first_command_operation()
+        return operation in [self.Operation.CREATE_ITEM, self.Operation.CREATE_PROPERTY]
 
     # # -----------------
     # # LAST related methods
@@ -1447,6 +1474,7 @@ def operation_is_combinable(self):
             self.Operation.CREATE_ITEM,
             self.Operation.SET_STATEMENT,
             self.Operation.CREATE_STATEMENT,
+            self.Operation.CREATE_PROPERTY,
             self.Operation.REMOVE_STATEMENT_BY_VALUE,
             self.Operation.REMOVE_QUALIFIER,
             self.Operation.REMOVE_REFERENCE,
@@ -1480,18 +1508,19 @@ def check_combination(self, state: CombiningState, next: Optional["BatchCommand"
             and self.operation_is_combinable()
             and next is not None
             and next.operation_is_combinable()
-            and next.is_not_create_item()
+            and next.is_not_create_entity()
             and self.has_combinable_id_with(next)
         )
         self.previous_entity_json = state.entity
         self.previous_commands = state.commands
 
     def has_combinable_id_with(self, next: "BatchCommand"):
         """
-        Returns True if `self` is CREATE_ITEM and `next`
+        Returns True if `self` is CREATE_ITEM or CREATE_PROPERTY and `next`
         has LAST as entity id, or if both have the same entity id.
         """
-        return (self.operation == self.Operation.CREATE_ITEM and next.entity_id == "LAST") or (
+        create_operations = [self.Operation.CREATE_ITEM, self.Operation.CREATE_PROPERTY]
+        return (self.operation in create_operations and next.entity_id == "LAST") or (
             self.entity_id == next.entity_id
         )
 
@@ -1528,10 +1557,11 @@ def get_original_entity_json(self, client: Client):
             self.previous_entity_json = copy.deepcopy(entity)
         return entity
 
-    def get_entity_or_empty_item(self, client: Client):
+    def get_entity_or_empty_entity(self, client: Client):
         """
         Calls the API to get the entity json or returns
-        an empty item if this command is a CREATE_ITEM.
+        an empty entity. Items for CREATE_ITEM and
+        properties for CREATE_PROPERTY.
         """
         if self.operation == self.Operation.CREATE_ITEM:
             return {
@@ -1543,6 +1573,16 @@ def get_entity_or_empty_item(self, client: Client):
                 "sitelinks": {},
                 "id": None,
             }
+        elif self.operation == self.Operation.CREATE_PROPERTY:
+            return {
+                "type": "property",
+                "data_type": self.property_data_type,
+                "labels": {},
+                "descriptions": {},
+                "aliases": {},
+                "statements": {},
+                "id": None,
+            }
         else:
             if self.entity_id == "LAST":
                 raise LastCouldNotBeEvaluated()
@@ -1556,7 +1596,7 @@ def get_previous_entity_json(self, client: Client):
         to the next command.
         """
         cached = getattr(self, "previous_entity_json", None)
-        entity = cached if cached else self.get_entity_or_empty_item(client)
+        entity = cached if cached else self.get_entity_or_empty_entity(client)
         return entity
 
     def get_final_entity_json(self, client: Client) -> dict:
@@ -1712,9 +1752,13 @@ def api_payload(self, client: Client):
         """
         if self.operation == self.Operation.CREATE_ITEM:
             return {"item": {}}
+        if self.operation == self.Operation.CREATE_PROPERTY:
+            return {"property": {"data_type": self.property_data_type}}
         if self.operation_is_combinable():
-            if self.is_id_last_or_create_item():
+            if self.is_item_creation():
                 return {"item": self.get_final_entity_json(client)}
+            elif self.is_property_creation():
+                return {"property": self.get_final_entity_json(client)}
             else:
                 return {"patch": self.entity_patch(client)}
         return {}
@@ -1739,8 +1783,6 @@ def send_to_api(self, client: Client) -> dict:
         - `NotImplementedError` if the operation
         is not implemented.
         """
-        if self.operation == self.Operation.CREATE_PROPERTY:
-            raise NotImplementedError()
         method, endpoint = self.operation_method_and_endpoint(client)
         body = self.api_body(client)
         return client.wikibase_request_wrapper(method, endpoint, body)
@@ -1755,8 +1797,10 @@ def operation_method_and_endpoint(self, client: Client):
         necessary for the operation.
         """
         if self.operation_is_combinable():
-            if self.is_id_last_or_create_item():
+            if self.is_item_creation():
                 return ("POST", "/entities/items")
+            elif self.is_property_creation():
+                return ("POST", "/entities/properties")
             else:
                 return ("PATCH", Client.wikibase_entity_endpoint(self.entity_id))
         match self.operation:

src/core/tests/test_api.py

@@ -280,6 +280,13 @@ def create_item(self, mocker, item_id):
             status_code=200,
         )
 
+    def create_property(self, mocker, property_id):
+        mocker.post(
+            self.wikibase_url("/entities/properties"),
+            json={"id": property_id},
+            status_code=200,
+        )
+
     def property_data_types(self, mocker, mapper):
         mocker.get(
             self.wikibase_url("/property-data-types"),
@@ -666,15 +673,18 @@ def test_send_create_item(self, mocker):
     @requests_mock.Mocker()
     def test_send_create_property(self, mocker):
         self.api_mocker.is_autoconfirmed(mocker)
+        self.api_mocker.create_property(mocker, "P5")
         parser = V1CommandParser()
         batch = BatchFactory.load_from_parser(
             parser, "b", "u", "CREATE_PROPERTY|wikibase-item||LAST|P1|Q1"
         )
         cmd: BatchCommand = batch.commands()[0]
         cmd.run(self.api_client)
         self.assertEqual(cmd.operation, BatchCommand.Operation.CREATE_PROPERTY)
-        self.assertEqual(cmd.status, BatchCommand.STATUS_ERROR)
-        self.assertEqual(cmd.error, BatchCommand.Error.OP_NOT_IMPLEMENTED)
-        self.assertIsNone(cmd.response_id)
-        with self.assertRaises(NotImplementedError):
-            cmd.send_to_api(self.api_client)
+        self.assertEqual(cmd.status, BatchCommand.STATUS_DONE)
+        self.assertIsNone(cmd.error)
+        self.assertEqual(
+            cmd.api_payload(self.api_client),
+            {"property": {"data_type": "wikibase-item"}},
+        )
+        self.assertEqual(cmd.response_id, "P5")

src/core/tests/test_batch_processing.py

@@ -500,7 +500,6 @@ def test_all_errors(self, mocker):
         )
         batch = self.parse(
             """
-        CREATE_PROPERTY|url
         Q1234|P5|12
         Q1234|Sptwikix|"Cool article"
         Q5|P5|123
@@ -513,25 +512,23 @@ def test_all_errors(self, mocker):
         batch.run()
         self.assertEqual(batch.status, Batch.STATUS_DONE)
         commands = batch.commands()
-        self.assertEqual(commands[0].operation, BatchCommand.Operation.CREATE_PROPERTY)
-        self.assertEqual(commands[0].error, BatchCommand.Error.OP_NOT_IMPLEMENTED)
-        self.assertEqual(commands[1].operation, BatchCommand.Operation.SET_STATEMENT)
-        self.assertEqual(commands[1].error, BatchCommand.Error.COMBINING_COMMAND_FAILED)
-        self.assertEqual(commands[2].operation, BatchCommand.Operation.SET_SITELINK)
-        self.assertEqual(commands[2].error, BatchCommand.Error.SITELINK_INVALID)
+        self.assertEqual(commands[0].operation, BatchCommand.Operation.SET_STATEMENT)
+        self.assertEqual(commands[0].error, BatchCommand.Error.COMBINING_COMMAND_FAILED)
+        self.assertEqual(commands[1].operation, BatchCommand.Operation.SET_SITELINK)
+        self.assertEqual(commands[1].error, BatchCommand.Error.SITELINK_INVALID)
+        self.assertEqual(commands[2].operation, BatchCommand.Operation.SET_STATEMENT)
+        self.assertEqual(commands[2].error, BatchCommand.Error.API_USER_ERROR)
         self.assertEqual(commands[3].operation, BatchCommand.Operation.SET_STATEMENT)
-        self.assertEqual(commands[3].error, BatchCommand.Error.API_USER_ERROR)
-        self.assertEqual(commands[4].operation, BatchCommand.Operation.SET_STATEMENT)
-        self.assertEqual(commands[4].error, BatchCommand.Error.API_SERVER_ERROR)
+        self.assertEqual(commands[3].error, BatchCommand.Error.API_SERVER_ERROR)
         self.assertEqual(
-            commands[5].operation, BatchCommand.Operation.REMOVE_STATEMENT_BY_VALUE
+            commands[4].operation, BatchCommand.Operation.REMOVE_STATEMENT_BY_VALUE
         )
-        self.assertEqual(commands[5].error, BatchCommand.Error.NO_STATEMENTS_PROPERTY)
+        self.assertEqual(commands[4].error, BatchCommand.Error.NO_STATEMENTS_PROPERTY)
         self.assertEqual(
-            commands[6].operation, BatchCommand.Operation.REMOVE_STATEMENT_BY_VALUE
+            commands[5].operation, BatchCommand.Operation.REMOVE_STATEMENT_BY_VALUE
         )
-        self.assertEqual(commands[6].error, BatchCommand.Error.NO_STATEMENTS_VALUE)
-        self.assertEqual(len(commands), 7)
+        self.assertEqual(commands[5].error, BatchCommand.Error.NO_STATEMENTS_VALUE)
+        self.assertEqual(len(commands), 6)
         for command in commands:
             self.assertEqual(command.status, BatchCommand.STATUS_ERROR)