Merge pull request #1 from rhpvorderman/init

Start on python-isal project.

Author: rhpvorderman

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,3 @@
+
+### Checklist
+- [ ] Pull request details were added to CHANHELOG.rst

.github/release_checklist.md

@@ -0,0 +1,21 @@
+Release checklist
+- [ ] Check outstanding issues on JIRA and Github.
+- [ ] Check [latest documentation](https://python-isal.readthedocs.io/en/latest/) looks fine.
+- [ ] Create a release branch.
+  - [ ] Set version to a stable number.
+  - [ ] Change current development version in `CHANGELOG.rst` to stable version.
+- [ ] Merge the release branch into `main`.
+- [ ] Create a test pypi package from the main branch. ([Instructions.](
+https://packaging.python.org/tutorials/packaging-projects/#generating-distribution-archives
+))
+- [ ] Install the packages from the test pypi repository to see if they work.
+- [ ] Created an annotated tag with the stable version number. Include changes 
+from CHANGELOG.rst.
+- [ ] Push tag to remote.
+- [ ] Push tested packages to pypi.
+- [ ] merge `main` branch back into `develop`.
+- [ ] Add updated version number to develop.
+- [ ] Build the new tag on readthedocs. Only build the last patch version of
+each minor version. So `1.1.1` and `1.2.0` but not `1.1.0`, `1.1.1` and `1.2.0`.
+- [ ] Create a new release on github.
+- [ ] Update the package on conda-forge.

.gitignore

@@ -6,6 +6,10 @@ __pycache__/
 # C extensions
 *.so
 
+# Cython generated files
+*.c
+*.html
+
 # Distribution / packaging
 .Python
 build/

.travis.yml

@@ -0,0 +1,35 @@
+language: python
+dist: focal
+
+before_install:
+  # Install conda
+  - export MINICONDA=${HOME}/miniconda
+  - export PATH=${MINICONDA}/bin:${PATH}
+  - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+  - bash miniconda.sh -b -f -p ${MINICONDA}
+  - conda config --set always_yes yes
+  - conda config --add channels defaults
+  - conda config --add channels conda-forge
+install:
+  - conda create -n test python=$TRAVIS_PYTHON_VERSION isa-l tox
+  - source activate test
+
+python: 3.6  # Use the oldest supported version of python as default.
+script:
+    - tox -e $TOX_ENV
+matrix:
+  include:
+    # TEST DOCS AND LINTING
+    # Use default python3 version here.
+    - env: TOX_ENV=docs
+    - env: TOX_ENV=lint
+    # UNIT TESTS
+    # On most recent versions of python.
+    - env: TOX_ENV=py36
+      after_success:
+        - pip install codecov
+        - codecov -v  # -v to make sure coverage upload works.
+    - python: 3.7
+      env: TOX_ENV=py37
+    - python: 3.8
+      env: TOX_ENV=py38

CHANGELOG.rst

@@ -0,0 +1,15 @@
+==========
+Changelog
+==========
+
+.. Newest changes should be on top.
+
+.. This document is user facing. Please word the changes in such a way
+.. that users understand how the changes affect the new version.
+
+version 0.1.0-dev
+-----------------
++ Add test suite that tests all possible settings for functions on the
+  isal_zlib module.
++ Create igzip module which implements all gzip functions and methods.
++ Create isal_zlib module which implements all zlib functions and methods.

README.md renamed to README.rst

@@ -1,4 +1,6 @@
-# python-isal
+python-isal
+===========
+
 Python bindings for the [isa-l](https://github.com/intel/isa-l) library.
 
 The Intel Infrastructure Storage Acceleration Library (isa-l) implements several key 

benchmark.py

@@ -0,0 +1,114 @@
+import argparse
+import gzip
+import timeit
+import zlib
+from pathlib import Path
+from typing import Dict
+
+from isal import igzip, isal_zlib  # noqa: F401 used in timeit strings
+
+DATA_DIR = Path(__file__).parent / "tests" / "data"
+COMPRESSED_FILE = DATA_DIR / "test.fastq.gz"
+with gzip.open(str(COMPRESSED_FILE), mode="rb") as file_h:
+    data = file_h.read()
+
+sizes: Dict[str, bytes] = {
+    "0b": b"",
+    "8b": data[:8],
+    "128b": data[:128],
+    "1kb": data[:1024],
+    "8kb": data[:8 * 1024],
+    "16kb": data[:16 * 1024],
+    "32kb": data[:32 * 1024],
+    "64kb": data[:64 * 1024],
+    # "128kb": data[:128*1024],
+    # "512kb": data[:512*1024]
+}
+compressed_sizes = {name: zlib.compress(data_block)
+                    for name, data_block in sizes.items()}
+
+compressed_sizes_gzip = {name: gzip.compress(data_block)
+                         for name, data_block in sizes.items()}
+
+
+def show_sizes():
+    print("zlib sizes")
+    print("name\t" + "\t".join(str(level) for level in range(-1, 10)))
+    for name, data_block in sizes.items():
+        orig_size = max(len(data_block), 1)
+        rel_sizes = (
+            str(round(len(zlib.compress(data_block, level)) / orig_size, 3))
+            for level in range(-1, 10))
+        print(name + "\t" + "\t".join(rel_sizes))
+
+    print("isal sizes")
+    print("name\t" + "\t".join(str(level) for level in range(0, 4)))
+    for name, data_block in sizes.items():
+        orig_size = max(len(data_block), 1)
+        rel_sizes = (
+            str(round(len(isal_zlib.compress(data_block, level)) / orig_size,
+                      3))
+            for level in range(0, 4))
+        print(name + "\t" + "\t".join(rel_sizes))
+
+
+def benchmark(name: str,
+              names_and_data: Dict[str, bytes],
+              isal_string: str,
+              zlib_string: str,
+              number: int = 10_000,
+              **kwargs):
+    print(name)
+    print("name\tisal\tzlib\tratio")
+    for name, data_block in names_and_data.items():
+        timeit_kwargs = dict(globals=dict(**globals(), **locals()),
+                             number=number, **kwargs)
+        isal_time = timeit.timeit(isal_string, **timeit_kwargs)
+        zlib_time = timeit.timeit(zlib_string, **timeit_kwargs)
+        isal_nanosecs = round(isal_time * (1_000_000 / number), 2)
+        zlib_nanosecs = round(zlib_time * (1_000_000 / number), 2)
+        ratio = round(isal_time / zlib_time, 2)
+        print("{0}\t{1}\t{2}\t{3}".format(name,
+                                          isal_nanosecs,
+                                          zlib_nanosecs,
+                                          ratio))
+
+
+# show_sizes()
+
+def argument_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--all", action="store_true")
+    parser.add_argument("--checksums", action="store_true")
+    parser.add_argument("--functions", action="store_true")
+    parser.add_argument("--gzip", action="store_true")
+    return parser
+
+
+if __name__ == "__main__":
+    args = argument_parser().parse_args()
+    if args.checksums or args.all:
+        benchmark("CRC32", sizes,
+                  "isal_zlib.crc32(data_block)",
+                  "zlib.crc32(data_block)")
+
+        benchmark("Adler32", sizes,
+                  "isal_zlib.adler32(data_block)",
+                  "zlib.adler32(data_block)")
+    if args.functions or args.all:
+        benchmark("Compression", sizes,
+                  "isal_zlib.compress(data_block, 1)",
+                  "zlib.compress(data_block, 1)")
+
+        benchmark("Decompression", compressed_sizes,
+                  "isal_zlib.decompress(data_block)",
+                  "zlib.decompress(data_block)")
+
+    if args.gzip or args.all:
+        benchmark("Compression", sizes,
+                  "igzip.compress(data_block, 1)",
+                  "gzip.compress(data_block, 1)")
+
+        benchmark("Decompression", compressed_sizes_gzip,
+                  "igzip.decompress(data_block)",
+                  "gzip.decompress(data_block)")

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/conf.py

@@ -0,0 +1,56 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+from distutils.dist import DistributionMetadata
+from pathlib import Path
+
+import pkg_resources
+
+# -- Project information -----------------------------------------------------
+
+# Get package information from the installed package.
+package = pkg_resources.get_distribution("isal")
+metadata_file = Path(package.egg_info) / Path(package.PKG_INFO)
+metadata = DistributionMetadata(path=str(metadata_file))
+
+project = 'python-isal'
+copyright = '2020, Leiden University Medical Center'
+author = 'Leiden University Medical Center'
+
+# The short X.Y version
+version = package.parsed_version.base_version
+# The full version, including alpha/beta/rc tags
+release = package.version
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']

docs/index.rst

@@ -0,0 +1,20 @@
+.. python-isal documentation master file, created by
+   sphinx-quickstart on Fri Sep 11 15:42:56 2020.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to python-isal's documentation!
+=======================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`