[chore] adding api docs

Author: yashbonde

.github/workflows/static.yaml

@@ -0,0 +1,65 @@
+# Simple workflow for deploying static content to GitHub Pages
+name: Deploy static content to Pages
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["main"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  # Single deploy job since we're just deploying
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.11
+
+      - name: Install dependencies
+        run: |
+          python -m pip install -U pip
+          python -m pip install .
+          python -m pip install Sphinx==7.4.7
+          python -m pip install sphinx-rtd-theme==2.0.0
+
+      - name: Build docs
+        run: |
+          cd docs
+          sphinx-apidoc -o source/ ../tuneapi
+          make html
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v1
+        with:
+          # Upload entire repository
+          path: './docs/_build/html'
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2

.gitignore

@@ -155,4 +155,8 @@ chunker/
 examples/
 cookbooks/
 datasets/
-
+docs/_static
+docs/_build
+docs/_templates
+docs/source
+test.py

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/conf.py

@@ -0,0 +1,31 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+import os, sys
+
+sys.path.insert(0, os.path.abspath("../"))
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = "tuneapi"
+copyright = "2024, Frello Technologies"
+author = "Frello Technologies"
+release = "0.4.11"
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = ["sphinx.ext.napoleon"]
+
+templates_path = ["_templates"]
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = "sphinx_rtd_theme"
+html_static_path = ["_static"]

docs/index.rst

@@ -0,0 +1,18 @@
+.. tuneapi documentation master file, created by
+   sphinx-quickstart on Tue Jul 30 17:14:39 2024.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+tuneapi documentation
+=====================
+
+TuneAPI 
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   source/tuneapi.apis
+   source/tuneapi.types
+   source/tuneapi.utils
+

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

tuneapi/apis/threads.py

@@ -18,6 +18,7 @@ def get_sub(
     sess.headers.update({"x-tune-key": tune_api_key})
     if tune_org_id:
         sess.headers.update({"X-Organization-Id": tune_org_id})
+    print("::::::", sess.headers)
     return tu.Subway(base_url, sess)
 
 
@@ -51,22 +52,20 @@ def put_thread(
         if not thread.title:
             thread.title = "thread_" + tu.get_snowflake()
 
+        body = {
+            "title": thread.title,
+            "messages": [m.to_dict(format="api", meta=False) for m in thread.chats],
+        }
         m = {}
         if thread.meta:
             m["thread_meta"] = thread.meta
         if thread.evals:
             m["evals"] = thread.evals
-        meta = {"meta": tu.to_json(m, tight=True)} if m else {}
+        if m:
+            body["metadata"] = {"meta": tu.to_json(m, tight=True)}
 
         tu.logger.info("Creating new thread")
-        out = self.sub.threads(
-            "post",
-            json={
-                "title": thread.title,
-                "metadata": meta,
-                "messages": [m.to_dict(format="api", meta=False) for m in thread.chats],
-            },
-        )
+        out = self.sub.threads("post", json=body)
         thread.id = out["id"]
         return thread
 

tuneapi/types/chats.py

@@ -15,8 +15,8 @@
 
 ########################################################################################################################
 #
-# The code in this section contains the primitive of this new chat API. The `Tool` class defines tools that the model
-# can predict. The `Message` class defines the container for storing the chat messages.
+# The code in this section contains the primitive of this new chat API. The ``Tool`` class defines tools that the model
+# can predict. The ``Message`` class defines the container for storing the chat messages.
 #
 ########################################################################################################################
 
@@ -133,7 +133,7 @@ def __init__(
         **kwargs,
     ):
         if role not in self.KNOWN_ROLES:
-            raise ValueError(f"Unknown role: {role}. Update dictionary `KNOWN_ROLES`")
+            raise ValueError(f"Unknown role: {role}. Update dictionary ``KNOWN_ROLES``")
         if value is None:
             raise ValueError("value cannot be None")
 
@@ -188,12 +188,15 @@ def to_dict(
         self,
         format: Optional[str] = None,
         meta: bool = False,
-    ):  #  ft: bool = False
+    ):
         """
-        if format == `ft` then export to following format: `{"from": "system/human/gpt", "value": "..."}`
-        elif format == `api` then `{"role": "system/user/assistant", "content": [{"type": "text", "text": {"value": "..."}]}`
-        elif format == `full` then `{"id": 1234421123, "role": "system/user/assistant", "content": [{"type": "text", "text": {"value": "..."}]}`
-        else export to following format: `{"role": "system/user/assistant", "content": "..."}`
+        if format == ``ft`` then export to following format: ``{"from": "system/human/gpt", "value": "..."}``
+
+        elif format == ``api`` then ``{"role": "system/user/assistant", "content": [{"type": "text", "text": {"value": "..."}]}``
+
+        elif format == ``full`` then ``{"id": 1234421123, "role": "system/user/assistant", "content": [{"type": "text", "text": {"value": "..."}]}``
+
+        else export to following format: ``{"role": "system/user/assistant", "content": "..."}``
         """
         role = self.role
 
@@ -699,7 +702,7 @@ def undo(self) -> "ThreadsTree":
 
     def pick(self, to: Message = None, from_: Message = None) -> Thread:
         """
-        Get a thread from the Tree srtucture by telling `to` and `from_` in the tree
+        Get a thread from the Tree srtucture by telling ``to`` and ``from_`` in the tree
         """
         if self.system:
             thread = Thread(system(self.system))
@@ -772,7 +775,7 @@ def regenerate_stream(
             # if we are regenerating for a human, then we need to add a prompt to the tree and then regenerate
             if not prompt:
                 raise ValueError(
-                    f"Regenerating for role 'human' but no `prompt` provided. pass `prompt`"
+                    f"Regenerating for role 'human' but no ``prompt`` provided. pass ``prompt``"
                 )
             if type(prompt) == str:
                 prompt = human(prompt)
@@ -786,7 +789,7 @@ def regenerate_stream(
             # to be a human
             if prompt:
                 raise ValueError(
-                    f"Regenerating for role 'gpt' but `prompt` provided. remove `prompt`"
+                    f"Regenerating for role 'gpt' but ``prompt`` provided. remove ``prompt``"
                 )
             thread = Thread()
             for x in self.tree.find(data_id=from_.id).get_parent_list():
@@ -939,7 +942,7 @@ def create_te_split(
         try:
             import numpy as np
         except ImportError:
-            raise ImportError("Install numpy to use `create_te_split` method")
+            raise ImportError("Install numpy to use ``create_te_split`` method")
 
         train_ds = ThreadsList()
         eval_ds = ThreadsList()

tuneapi/utils/serdeser.py

@@ -56,11 +56,12 @@ def from_json(x: str = "") -> Dict[str, Any]:
     """
     if os.path.exists(x):
         if x.endswith(".json"):
-            with open(x, "r") as f:
-                return json.load(f)
+            with open(x, "rb") as f:
+                return json.loads(f.read().decode("utf-8", "ignore"))
         elif x.endswith(".jsonl"):
-            with open(x, "r") as f:
-                return [json.loads(line) for line in f]
+            with open(x, "rb") as f:
+                for line in f.read().decode("utf-8", "ignore").splitlines():
+                    return json.loads(line)
     else:
         return json.loads(x)
 

tuneapi/utils/subway.py

@@ -152,6 +152,7 @@ def __call__(
 
         if SimplerTimes.get_now_i64() - self._last_update > 120:
             self._renew_session()
+        print(f"{url=}, {items=}")
         r = fn(url, **items, **kwargs)
         if 399 < r.status_code < 500:
             raise SubwayClientError(r.content.decode(), code=r.status_code)