Merge pull request #124 from openzim/contrib

Contributors-friendly README

Author: rgaudin

.gitignore

@@ -32,3 +32,8 @@ libzim/libzim_api.h
 Pipfile
 .dev
 .env
+
+libzim.so
+libzim.so.*
+libzim.dylib
+libzim.*.dylib

MANIFEST.in

@@ -2,6 +2,9 @@ include LICENSE
 include README.md
 include tests/*.py
 include pyproject.toml
+include setup.cfg
+include requirements-dev.txt
+include tasks.py
 
 include libzim/libzim.7.dylib
 include libzim/libzim.so.7

README.md

@@ -1,16 +1,10 @@
 # python-libzim
 
-The Python-libzim package allows you to read/write [ZIM
-files](https://openzim.org) in Python. It provides a shallow Python
-interface on top of the [`libzim`](https://github.com/openzim/libzim)
-C++ library.
+`libzim` module allows you to read and write [ZIM
+files](https://openzim.org) in Python. It provides a shallow python
+interface on top of the [C++ `libzim` library](https://github.com/openzim/libzim).
 
-It is primarily used in openZIM scrapers like for example
-[`Sotoki`](https://github.com/openzim/sotoki) or
-[`Youtube2zim`](https://github.com/openzim/youtube).
-
-Read [CONTRIBUTING.md](./CONTRIBUTING.md) to know more about
-Python-libzim development.
+It is primarily used in [openZIM](https://github.com/openzim/) scrapers like [`sotoki`](https://github.com/openzim/sotoki) or [`youtube2zim`](https://github.com/openzim/youtube).
 
 [![Build Status](https://github.com/openzim/python-libzim/workflows/test/badge.svg?query=branch%3Amaster)](https://github.com/openzim/python-libzim/actions?query=branch%3Amaster)
 [![CodeFactor](https://www.codefactor.io/repository/github/openzim/python-libzim/badge)](https://www.codefactor.io/repository/github/openzim/python-libzim)
@@ -20,52 +14,114 @@ Python-libzim development.
 
 ## Installation
 
-The [PyPI package](https://pypi.org/project/libzim/) is bundled with a
-recent version of the libzim for macOS and GNU/Linux (x86_64
-architecture). For other OSes, the latest libzim has to be
-compiled manually, See [Setup hints](#setup-hints) to know more.
+```sh
+pip install libzim
+```
+
+The [PyPI package](https://pypi.org/project/libzim/) is available for x86_64 macOS and GNU/Linux only. It bundles a [recent release](http://download.openzim.org/release/libzim/) of the C++ libzim.
+
+On other platforms, you'd have to [compile C++ libzim from
+source](https://github.com/openzim/libzim) first then build this one, adjusting `LD_LIBRARY_PATH`.
 
-```bash
-pip3 install libzim
+## Contributions
+
+``` sh
+git clone [email protected]:openzim/python-libzim.git && cd python-libzim
+# python -m venv env && source env/bin/activate
+pip install -U setuptools invoke
+invoke download-libzim install-dev build-ext test
+# invoke --list for available development helpers
 ```
 
-## Quickstart
+See [CONTRIBUTING.md](./CONTRIBUTING.md) for additional details then [Open a ticket](https://github.com/openzim/python-libzim/issues/new) or submit a Pull Request on Github 🤗!
+
+## Usage
 
-### Read a ZIM
+### Read a ZIM file
 
 ```python
 from libzim.reader import Archive
+from libzim.search import Query, Searcher
+from libzim.suggestion import SuggestionSearcher
 
 zim = Archive("test.zim")
 print(f"Main entry is at {zim.main_entry.get_item().path}")
-entry = zim.get_entry_by_path("path/to/my-article")
-print(f"Entry {entry.title} at {entry.path} is {entry.get_item().size}b:")
+entry = zim.get_entry_by_path("home/fr")
+print(f"Entry {entry.title} at {entry.path} is {entry.get_item().size}b.")
 print(bytes(entry.get_item().content).decode("UTF-8"))
+
+# searching using full-text index
+search_string = "Welcome"
+query = Query().set_query(search_string)
+searcher = Searcher(zim)
+search = searcher.search(query)
+search_count = search.getEstimatedMatches()
+print(f"there are {search_count} matches for {search_string}")
+print(list(search.getResults(0, search_count)))
+
+# accessing suggestions
+search_string = "kiwix"
+suggestion_searcher = SuggestionSearcher(zim)
+suggestion = suggestion_searcher.suggest(search_string)
+suggestion_count = suggestion.getEstimatedMatches()
+print(f"there are {suggestion_count} matches for {search_string}")
+print(list(suggestion.getResults(0, suggestion_count)))
 ```
 
-### Write a ZIM
+### Write a ZIM file
 
-See [example](examples/basic_writer.py) for a basic usage of the
-writer API.
+```py
+from libzim.writer import Creator, Item, StringProvider, FileProvider, Hint
 
-## Setup hints
 
-### Installing the `libzim` dylib and headers manually
+class MyItem(Item):
+    def __init__(self, title, path, content = "", fpath = None):
+        super().__init__()
+        self.path = path
+        self.title = title
+        self.content = content
+        self.fpath = fpath
 
-If you have to install the libzim manually, you will have to [compile
-`libzim` from
-source](https://github.com/openzim/libzim). This binding has been designed
-to work with the latest version of the libzim, we only recommend to
-use it with latest released version.
+    def get_path(self):
+        return self.path
 
-If you have not installed libzim in standard directory, you will have
-to set `LD_LIBRARY_PATH` to allow python to find the library. Assuming
-you have extracted (or installed) the library if LIBZIM_DIR:
+    def get_title(self):
+        return self.title
 
-```bash
-export LD_LIBRARY_PATH="${LIBZIM_DIR}/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH"
+    def get_mimetype(self):
+        return "text/html"
+
+    def get_contentprovider(self):
+        if self.fpath is not None:
+            return FileProvider(self.fpath)
+        return StringProvider(self.content)
+       
+    def get_hints(self):
+        return {Hint.FRONT_ARTICLE: True}
+
+
+content = """<html><head><meta charset="UTF-8"><title>Web Page Title</title></head>
+<body><h1>Welcome to this ZIM</h1><p>Kiwix</p></body></html>"""
+
+item = MyItem("Hello Kiwix", "home", content)
+item2 = MyItem("Bonjour Kiwix", "home/fr", None, "home-fr.html")
+
+with Creator("test.zim").config_indexing(True, "eng") as creator:
+    creator.set_mainpath("home")
+    creator.add_item(item)
+    creator.add_item(item2)
+    for name, value in {
+        "creator": "python-libzim",
+        "description": "Created in python",
+        "name": "my-zim",
+        "publisher": "You",
+        "title": "Test ZIM",
+    }.items():
+
+        creator.add_metadata(name.title(), value)
 ```
 
+
 ## License
 
 [GPLv3](https://www.gnu.org/licenses/gpl-3.0) or later, see

examples/basic_writer.py

@@ -5,9 +5,58 @@
 A description file for invoke (https://www.pyinvoke.org/)
 """
 
+import pathlib
+import platform
+import re
+import urllib.request
+
 from invoke import task
 
 
+@task
+def download_libzim(c, version="7.0.0"):
+    """download C++ libzim binary"""
+
+    if platform.machine() != "x86_64" or platform.system() not in ("Linux", "Darwin"):
+        raise NotImplementedError(f"Platform {platform.platform()} not supported")
+
+    is_nightly = re.match(r"^\d{4}-\d{2}-\d{2}$", version)
+
+    if not is_nightly and not re.match(r"^\d\.\d\.\d$", version):
+        raise ValueError(
+            f"Unrecognised version {version}. "
+            "Must be either a x.x.x release or a Y-M-D date to use a nightly"
+        )
+
+    fname = pathlib.Path(
+        "libzim_{os}-x86_64-{version}.tar.gz".format(
+            os={"Linux": "linux", "Darwin": "macos"}.get(platform.system()),
+            version=version,
+        )
+    )
+    url = (
+        f"https://download.openzim.org/nightly/{version}/{fname.name}"
+        if is_nightly
+        else f"https://download.openzim.org/release/libzim/{fname.name}"
+    )
+    print("Downloading from", url)
+
+    with urllib.request.urlopen(url) as response, open(fname, "wb") as fh:  # nosec
+        fh.write(response.read())
+    c.run(f"tar -xvf {fname.name}")
+    c.run(f"rm -vf {fname.name}")
+
+    dname = fname.with_suffix("").stem
+    c.run(f"mv -v {dname}/include/* ./include/")
+    c.run(f"mv -v {dname}/lib/* ./lib/")
+    c.run(f"rmdir {dname}/lib {dname}/include/ {dname}")
+
+    if platform.system() == "Darwin":
+        c.run(f"ln -svf ./lib/libzim.{version[0]}.dylib ./")
+    else:
+        c.run(f"ln -svf ./lib/libzim.so.{version[0]} ./")
+
+
 @task
 def build_ext(c):
     c.run("PROFILE=1 python setup.py build_ext -i")