Merge pull request #115 from openzim/hint

Add support for Hints in python-libzim

Author: rgaudin

.github/workflows/test.yml

@@ -2,7 +2,7 @@ name: test
 on: [push]
 
 env:
-  LIBZIM_VERSION: 2021-07-08
+  LIBZIM_VERSION: 2021-08-26
   LIBZIM_INCLUDE_PATH: include/zim
   CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   MAX_LINE_LENGTH: 88

libzim/lib.cxx

@@ -129,6 +129,20 @@ std::unique_ptr<zim::writer::ContentProvider> WriterItemWrapper::callCythonRetur
   return ret_val;
 }
 
+zim::writer::Hints WriterItemWrapper::callCythonReturnHints(std::string methodName) const
+{
+  if (!this->m_obj)
+    throw std::runtime_error("Python object not set");
+
+  std::string error;
+
+  auto ret_val = hints_cy_call_fct(this->m_obj, methodName, &error);
+  if (!error.empty())
+    throw std::runtime_error(error);
+
+  return ret_val;
+}
+
 std::string
 WriterItemWrapper::getPath() const
 {
@@ -152,3 +166,8 @@ WriterItemWrapper::getContentProvider() const
 {
     return callCythonReturnContentProvider("get_contentprovider");
 }
+
+zim::writer::Hints WriterItemWrapper::getHints() const
+{
+	return callCythonReturnHints("get_hints");
+}

libzim/lib.h

@@ -115,9 +115,11 @@ class WriterItemWrapper : public zim::writer::Item, private ObjWrapper
     virtual std::string getTitle() const;
     virtual std::string getMimeType() const;
     virtual std::unique_ptr<zim::writer::ContentProvider> getContentProvider() const;
+    virtual zim::writer::Hints getHints() const;
 
   private:
     std::unique_ptr<zim::writer::ContentProvider> callCythonReturnContentProvider(std::string) const;
+    zim::writer::Hints callCythonReturnHints(std::string) const;
 };
 
 class ContentProviderWrapper : public zim::writer::ContentProvider, private ObjWrapper

libzim/wrapper.pxd

@@ -24,6 +24,7 @@ from libcpp cimport bool
 from libcpp.memory cimport shared_ptr, unique_ptr
 from libcpp.string cimport string
 from libcpp.vector cimport vector
+from libcpp.map cimport map
 
 
 cdef extern from "zim/zim.h" namespace "zim":
@@ -50,6 +51,9 @@ cdef extern from "zim/blob.h" namespace "zim":
 cdef extern from "zim/writer/item.h" namespace "zim::writer":
     cdef cppclass WriterItem "zim::writer::Item":
         pass
+    ctypedef enum HintKeys:
+        COMPRESS
+        FRONT_ARTICLE
 
 cdef extern from "zim/writer/contentProvider.h" namespace "zim::writer":
     cdef cppclass ContentProvider:
@@ -66,7 +70,7 @@ cdef extern from "zim/writer/creator.h" namespace "zim::writer":
         void startZimCreation(string filepath) nogil except +;
         void addItem(shared_ptr[WriterItem] item) nogil except +
         void addMetadata(string name, string content, string mimetype) nogil except +
-        void addRedirection(string path, string title, string targetpath) nogil except +
+        void addRedirection(string path, string title, string targetpath, map[HintKeys, uint64_t] hints) nogil except +
         void finishZimCreation() nogil except +
         void setMainPath(string mainPath)
         void addIllustration(unsigned int size, string content)

libzim/wrapper.pyx

@@ -33,12 +33,15 @@ from libcpp.string cimport string
 from libcpp cimport bool
 from libcpp.set cimport set
 from libcpp.memory cimport shared_ptr, make_shared, unique_ptr
+from libcpp.map cimport map
 
 import pathlib
 import datetime
 import traceback
 
 
+pybool = type(True)
+
 #########################
 #         Blob          #
 #########################
@@ -166,6 +169,23 @@ cdef public api:
 
         return 0
 
+    map[HintKeys, uint64_t] convertToCppHints(dict hintsDict):
+        cdef map[HintKeys, uint64_t] ret;
+        for key, value in hintsDict.items():
+            ret[key.value] = <uint64_t>value
+        return ret
+
+    map[HintKeys, uint64_t] hints_cy_call_fct(object obj, string method, string* error) with gil:
+        cdef map[HintKeys, uint64_t] ret;
+        try:
+            func = getattr(obj, method.decode('UTF-8'))
+            hintsDict = {k: pybool(v) for k, v in func().items() if isinstance(k, Hint)}
+            return convertToCppHints(hintsDict)
+        except Exception as e:
+            error[0] = traceback.format_exc().encode('UTF-8')
+
+        return ret
+
 
 class Compression(enum.Enum):
     """ Compression algorithms available to create ZIM files """
@@ -174,6 +194,12 @@ class Compression(enum.Enum):
     zstd = wrapper.CompressionType.zimcompZstd
 
 
+class Hint(enum.Enum):
+    COMPRESS = wrapper.HintKeys.COMPRESS
+    FRONT_ARTICLE = wrapper.HintKeys.FRONT_ARTICLE
+
+
+
 cdef class Creator:
     """ Zim Creator
 
@@ -278,15 +304,16 @@ cdef class Creator:
         with nogil:
             self.c_creator.addMetadata(_name, _content, _mimetype)
 
-    def add_redirection(self, str path, str title, str targetPath):
+    def add_redirection(self, str path, str title, str targetPath, dict hints):
         if not self._started:
             raise RuntimeError("ZimCreator not started")
 
         cdef string _path = path.encode('utf8')
         cdef string _title = title.encode('utf8')
         cdef string _targetPath = targetPath.encode('utf8')
+        cdef map[HintKeys, uint64_t] _hints = convertToCppHints(hints)
         with nogil:
-            self.c_creator.addRedirection(_path, _title, _targetPath)
+            self.c_creator.addRedirection(_path, _title, _targetPath, _hints)
 
     def __enter__(self):
         cdef string _path = str(self._filename).encode('utf8')

libzim/writer.py

@@ -33,10 +33,11 @@
 
 import os
 import datetime
-from typing import Union
+from typing import Union, Dict
 
 from .wrapper import Creator as _Creator, Compression
 from .wrapper import WritingBlob as Blob
+from .wrapper import Hint
 
 __all__ = [
     "Item",
@@ -132,6 +133,9 @@ def get_contentprovider(self) -> ContentProvider:
         """ContentProvider containing the complete content of the item"""
         raise NotImplementedError("get_contentprovider must be implemented.")
 
+    def get_hints(self) -> Dict[Hint, int]:
+        raise NotImplementedError("get_hints must be implemented.")
+
     def __repr__(self) -> str:
         return (
             f"{self.__class__.__name__}(path={self.get_path()}, "

tests/test_libzim_creator.py

@@ -6,7 +6,7 @@
 import datetime
 import itertools
 import subprocess
-
+from typing import Dict
 
 import pytest
 
@@ -18,6 +18,7 @@
     FileProvider,
     StringProvider,
     Blob,
+    Hint,
 )
 from libzim.reader import Archive
 
@@ -45,6 +46,9 @@ def get_contentprovider(self) -> libzim.writer.ContentProvider:
             return FileProvider(filepath=self.filepath)
         return StringProvider(content=getattr(self, "content", ""))
 
+    def get_hints(self) -> Dict[Hint, int]:
+        return getattr(self, "hints", {Hint.FRONT_ARTICLE: True})
+
 
 @pytest.fixture(scope="function")
 def fpath(tmpdir):
@@ -423,13 +427,13 @@ def test_creator_redirection(fpath, lipsum_item):
     # ensure we can't add if not started
     c = Creator(fpath)
     with pytest.raises(RuntimeError, match="not started"):
-        c.add_redirection("home", "hello", HOME_PATH)
+        c.add_redirection("home", "hello", HOME_PATH, {Hint.FRONT_ARTICLE: True})
     del c
 
     with Creator(fpath) as c:
         c.add_item(lipsum_item)
-        c.add_redirection("home", "hello", HOME_PATH)
-        c.add_redirection("accueil", "bonjour", HOME_PATH)
+        c.add_redirection("home", "hello", HOME_PATH, {Hint.FRONT_ARTICLE: True})
+        c.add_redirection("accueil", "bonjour", HOME_PATH, {Hint.FRONT_ARTICLE: True})
 
     zim = Archive(fpath)
     assert zim.entry_count == 3
@@ -524,6 +528,9 @@ def get_mimetype(self):
         def get_contentprovider(self):
             return ""
 
+        def get_hints(self):
+            return {}
+
     with Creator(fpath) as c:
         with pytest.raises(RuntimeError, match="ContentProvider is None"):
             c.add_item(AnItem())
@@ -540,11 +547,76 @@ def get_title(self):
         def get_mimetype(self):
             return ""
 
+        def get_hints(self):
+            return {}
+
     with Creator(fpath) as c:
         with pytest.raises(RuntimeError, match="has no attribute"):
             c.add_item(AnItem())
 
 
+def test_missing_hints(fpath):
+    class AnItem:
+        def get_path(self):
+            return ""
+
+        def get_title(self):
+            return ""
+
+        def get_mimetype(self):
+            return ""
+
+    with Creator(fpath) as c:
+        with pytest.raises(RuntimeError, match="has no attribute 'get_hints'"):
+            c.add_item(AnItem())
+
+        with pytest.raises(RuntimeError, match="must be implemented"):
+            c.add_item(libzim.writer.Item())
+
+
+def test_nondict_hints(fpath):
+    with Creator(fpath) as c:
+        with pytest.raises(RuntimeError, match="has no attribute 'items'"):
+            c.add_item(StaticItem(path="1", title="", hints=1))
+
+        with pytest.raises(TypeError, match="hints"):
+            c.add_redirection("a", "", "b", hints=1)
+
+
+def test_hints_values(fpath):
+    with Creator(fpath) as c:
+        # correct values
+        c.add_item(StaticItem(path="0", title="", hints={}))
+        c.add_item(
+            StaticItem(
+                path="1",
+                title="",
+                hints={Hint.FRONT_ARTICLE: True, Hint.COMPRESS: False},
+            )
+        )
+        # non-expected Hints are ignored
+        c.add_item(StaticItem(path="2", title="", hints={"hello": "world"}))
+        # Hint values are casted to bool
+        c.add_item(StaticItem(path="3", title="", hints={Hint.FRONT_ARTICLE: "world"}))
+        c.add_redirection(
+            path="4", title="", targetPath="0", hints={Hint.COMPRESS: True}
+        )
+        # filtered-out values
+        c.add_item(StaticItem(path="5", title="", hints={5: True}))
+        c.add_item(StaticItem(path="6", title="", hints={"yolo": True}))
+        c.add_item(StaticItem(path="7", title="", hints={"FRONT_ARTICLE": True}))
+        c.add_item(StaticItem(path="8", title="", hints={0: True}))
+
+        # non-existent Hint
+        with pytest.raises(AttributeError, match="YOLO"):
+            c.add_item(StaticItem(path="0", title="", hints={Hint.YOLO: True}))
+
+        with pytest.raises(AttributeError, match="YOLO"):
+            c.add_redirection(
+                path="5", title="", target_path="0", hints={Hint.YOLO: True}
+            )
+
+
 def test_reimpfeed(fpath):
     class AContentProvider:
         def __init__(self):
@@ -569,6 +641,9 @@ def get_title(self):
         def get_mimetype(self):
             return ""
 
+        def get_hints(self):
+            return {}
+
         def get_contentprovider(self):
             return AContentProvider()
 
@@ -599,6 +674,9 @@ def get_title(self):
         def get_mimetype(self):
             return ""
 
+        def get_hints(self):
+            return {}
+
         def get_contentprovider(self):
             return AContentProvider()