update setup

Author: jararap

README.md

@@ -7,11 +7,12 @@ conda install tbb-devel
 ```
 2. If using python wrapper (Todo: automate pip installation)
 
-    a. Install pybind11, simply:
+    a. Using pip:
       ```
-      pip install pybind11
+      pip install -r requirements.txt
+      pip install .
       ```
-    b. Compile greedy_builder
+    b. Or compile manually e.g. (have to specify links)
       ```
       c++ -O3 -Wall -shared -std=c++20 \
       -fPIC $(python3 -m pybind11 --includes) \
@@ -26,7 +27,7 @@ conda install tbb-devel
     c. import and use! Examples in [eval_tokenizer_example.ipynb](https://github.com/PreferredAI/aoatt/blob/main/eval_tokenizer_example.ipynb)
 3. If using C++ files directly
 
-    a. Compile greedy_cache.py
+    a. Compile greedy_cache.py e.g.:
       ```
       c++ -O3 -std=c++20 \
       -I$CONDA_PREFIX/include/ \

requirements.txt

@@ -0,0 +1,2 @@
+pybind11
+tbb-devel

setup.py

@@ -0,0 +1,26 @@
+import pip
+from sysconfig import get_path
+from setuptools import setup, Extension
+
+PATH_PREFIX = get_path('data')
+module1 = Extension(f'greedy_builder',
+                    extra_compile_args = ["-O3", "-std=c++20"],
+                    define_macros = [('MAJOR_VERSION', '0'),
+                                     ('MINOR_VERSION', '1')],
+                    include_dirs = [f'{PATH_PREFIX}/include/',
+                                    f'{PATH_PREFIX}/include/tbb',
+                                    f'{PATH_PREFIX}/include/oneapi'],
+                    library_dirs = [f'{PATH_PREFIX}/lib/'],
+                    libraries = ['tbb'],
+                    sources = ['pcatt/greedy_builder.cpp'])
+
+setup(
+    name="greedtok",
+    version="0.1",
+    description="Partition Cover Approach to Tokenization",
+    author="JP Lim",
+    author_email="[email protected]",
+    license = "MIT",
+    setup_requires=['pybind11', 'tbb-devel'],
+    ext_modules = [module1]
+)