docs: add refferences (#177)

jjmachan · web-flow · commit 33817ee003c5 · 2023-10-13T11:28:42.000+05:30
added autodoc and reference section
cards to most pages
diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -19,5 +19,8 @@ sphinx:
   # fail_on_warning: true
 
 python:
-   install:
-   - requirements: ./requirements/docs.txt
+  install:
+    - method: pip
+      path: .
+      extra_requirements:
+        - docs
diff --git a/Makefile b/Makefile
@@ -39,6 +39,5 @@ test-e2e: ## Run end2end tests
 docs-site: ## Build and serve documentation
 	@sphinx-build -nW --keep-going -j 4 -b html $(GIT_ROOT)/docs/ $(GIT_ROOT)/docs/_build/html
 	@python -m http.server --directory $(GIT_ROOT)/docs/_build/html
-
 watch-docs: ## Build and watch documentation
 	sphinx-autobuild docs docs/_build/html --watch $(GIT_ROOT)/src/ --ignore ".ipynb"
diff --git a/docs/concepts/index.md b/docs/concepts/index.md
@@ -1,14 +1,13 @@
+(core-concepts)=
 # Core Concepts
-Core concepts that drives Ragas
-
-
-```{toctree}
+:::{toctree}
 :caption: Concepts
 :hidden:
 
 metrics_driven
 metrics/index
 testset_generation
 feedback
+:::
 
-```
+Ragas is using 
diff --git a/docs/conf.py b/docs/conf.py
@@ -1,7 +1,7 @@
-# Configuration file for the Sphinx documentation builder.
-#
-# For the full list of built-in configuration values, see the documentation:
-# https://www.sphinx-doc.org/en/master/usage/configuration.html
+import os
+from dataclasses import asdict
+
+from sphinxawesome_theme import ThemeOptions
 
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
@@ -11,23 +11,36 @@
 author = "ExplodingGradients"
 release = "0.0.16"
 
-
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
 
+html_title = "Ragas"
+html_theme = "sphinxawesome_theme"
+html_static_path = ["_static"]
+html_css_files = ["css/ragas.css"]
+html_favicon = "./_static/favicon.ico"
+
 extensions = [
+    "sphinx.ext.doctest",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.autosummary",
+    "sphinxawesome_theme.highlighting",
+    "sphinxawesome_theme.docsearch",
     "myst_parser",
     "sphinx_design",
-    # "myst_parser",
-    "sphinxawesome_theme.highlighting",
-    # "sphinxawesome_theme.docsearch",
 ]
-source_suffix = [".rst", ".md"]
 
+source_suffix = [".rst", ".md"]
 templates_path = ["_templates"]
 exclude_patterns = []
 myst_number_code_blocks = ["typescript"]
 
+# algolia search
+docsearch_app_id = os.getenv("DOCSEARCH_APP_ID")
+docsearch_api_key = os.getenv("DOCSEARCH_API_KEY")
+docsearch_index_name = os.getenv("DOCSEARCH_INDEX_NAME")
+
 # -- Options for HTML output -------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
 
@@ -50,24 +63,13 @@
     # "attrs_block",
 ]
 
-html_title = "Ragas"
-html_theme = "sphinxawesome_theme"
-html_static_path = ["_static"]
-html_css_files = ["css/ragas.css"]
-html_favicon = "./_static/favicon.ico"
-
-
-from dataclasses import asdict
-
-from sphinxawesome_theme import ThemeOptions
-
 theme_options = ThemeOptions(
     logo_light="./_static/imgs/ragas-logo.png",
     logo_dark="./_static/imgs/ragas-logo.png",
     extra_header_link_icons={
         "discord": {
             "link": "https://discord.gg/5djav8GGNZ",
-            "icon": """<svg height="26px" style="margin-right: 3px; display: inline" viewBox="0 0 22 22" fill="currentColor" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenod" clip-rule="evenod" d="M18.59 5.88997C17.36 5.31997 16.05 4.89997 14.67 4.65997C14.5 4.95997 14.3 5.36997 14.17 5.69997C12.71 5.47997 11.26 5.47997 9.83001 5.69997C9.69001 5.36997 9.49001 4.95997 9.32001 4.65997C7.94001 4.89997 6.63001 5.31997 5.40001 5.88997C2.92001 9.62997 2.25001 13.28 2.58001 16.87C4.23001 18.1 5.82001 18.84 7.39001 19.33C7.78001 18.8 8.12001 18.23 8.42001 17.64C7.85001 17.43 7.31001 17.16 6.80001 16.85C6.94001 16.75 7.07001 16.64 7.20001 16.54C10.33 18 13.72 18 16.81 16.54C16.94 16.65 17.07 16.75 17.21 16.85C16.7 17.16 16.15 17.42 15.59 17.64C15.89 18.23 16.23 18.8 16.62 19.33C18.19 18.84 19.79 18.1 21.43 16.87C21.82 12.7 20.76 9.08997 18.61 5.88997H18.59ZM8.84001 14.67C7.90001 14.67 7.13001 13.8 7.13001 12.73C7.13001 11.66 7.88001 10.79 8.84001 10.79C9.80001 10.79 10.56 11.66 10.55 12.73C10.55 13.79 9.80001 14.67 8.84001 14.67ZM15.15 14.67C14.21 14.67 13.44 13.8 13.44 12.73C13.44 11.66 14.19 10.79 15.15 10.79C16.11 10.79 16.87 11.66 16.86 12.73C16.86 13.79 16.11 14.67 15.15 14.67Z" fill="currentColor"/></svg>""",
+            "icon": """<svg height="26px" style="margin-right: 3px; display: inline" viewBox="0 0 22 22" fill="currentColor" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenod" clip-rule="evenod" d="M18.59 5.88997C17.36 5.31997 16.05 4.89997 14.67 4.65997C14.5 4.95997 14.3 5.36997 14.17 5.69997C12.71 5.47997 11.26 5.47997 9.83001 5.69997C9.69001 5.36997 9.49001 4.95997 9.32001 4.65997C7.94001 4.89997 6.63001 5.31997 5.40001 5.88997C2.92001 9.62997 2.25001 13.28 2.58001 16.87C4.23001 18.1 5.82001 18.84 7.39001 19.33C7.78001 18.8 8.12001 18.23 8.42001 17.64C7.85001 17.43 7.31001 17.16 6.80001 16.85C6.94001 16.75 7.07001 16.64 7.20001 16.54C10.33 18 13.72 18 16.81 16.54C16.94 16.65 17.07 16.75 17.21 16.85C16.7 17.16 16.15 17.42 15.59 17.64C15.89 18.23 16.23 18.8 16.62 19.33C18.19 18.84 19.79 18.1 21.43 16.87C21.82 12.7 20.76 9.08997 18.61 5.88997H18.59ZM8.84001 14.67C7.90001 14.67 7.13001 13.8 7.13001 12.73C7.13001 11.66 7.88001 10.79 8.84001 10.79C9.80001 10.79 10.56 11.66 10.55 12.73C10.55 13.79 9.80001 14.67 8.84001 14.67ZM15.15 14.67C14.21 14.67 13.44 13.8 13.44 12.73C13.44 11.66 14.19 10.79 15.15 10.79C16.11 10.79 16.87 11.66 16.86 12.73C16.86 13.79 16.11 14.67 15.15 14.67Z" fill="currentColor"/></svg>""",  # noqa
         },
         "github": {
             "link": "https://github.com/explodinggradients/ragas",
@@ -102,4 +104,4 @@
 html_theme_options = asdict(theme_options)
 
 # -- Myst NB Config -------------------------------------------------
-nb_execution_mode = "auto"
+nb_execution_mode = "off"
diff --git a/docs/getstarted/evaluation.md b/docs/getstarted/evaluation.md
@@ -1,12 +1,4 @@
----
-file_format: mystnb
-kernelspec:
-  name: python3
-execution:
-  timeout: 300
----
-
-(heading-target)=
+(get-started-evaluation)=
 # Evaluation
 
 welcome to the ragas quickstart. We're going to get you up and running with ragas as quickly as you can so that you can go back to improving your Retrieval Augmented Generation pipelines while this library makes sure your changes are improving your entire pipeline.
diff --git a/docs/getstarted/index.md b/docs/getstarted/index.md
@@ -1,9 +1,49 @@
+(get-started)=
 # Get Started
 
-```{toctree}
+:::{toctree}
 :maxdepth: 1
+:hidden:
 install.md
 evaluation.md
 testset_generation.md
 monitoring.md
-```
+:::
+
+Welcome to the Ragas tutorials! These begineer-friendly tutorials will guide you
+through the fundamentals of working with Ragas. These tutorials do assume basic
+knowledge of Python and Retrieval Augmented Generation (RAG) pipelines. 
+
+Before you go further make sure you have [Ragas installed](./install.md)!
+
+:::{note}
+The tutorials only give you on overview of what you can do with ragas and the
+basic skill you need to use it. If you want an in-depth explaination of the
+core-concepts behind Ragas, check out the [Core Concepts](../concepts/index.md) page. You can also checkout the [How-to Guides](../howtos/index.md) if you want to specific applications of Ragas.
+:::
+
+
+If you have any questions about Ragas, feel free to join and ask in the
+`#questions` channel in our discord community ❤ .
+
+Let’s get started! 🏁
+
+:::{card} Ragas Metrics and Evaluation
+:link: get-started-evaluation
+:link-type: ref
+
+How to use the Ragas Metrics to evaluate your RAG pipelines.
+:::
+
+:::{card} Synthetic Testset Generation
+:link: get-started-testset-generation
+:link-type: ref
+
+How to generate testset to assess your RAG pipelines
+:::
+:::{card} Monitoring
+:link: get-started-monitoring
+:link-type: ref
+
+How to monitor your RAG systems in production.
+:::
diff --git a/docs/getstarted/monitoring.md b/docs/getstarted/monitoring.md
@@ -1,3 +1,4 @@
+(get-started-monitoring)=
 # Monitoring
 
 Maintaining the quality and performance of an LLM application in a production environment can be challenging. Ragas provides a solution through production quality monitoring, offering valuable insights into your application's performance. This is achieved by constructing custom, smaller, more cost-effective, and faster models.
@@ -31,4 +32,4 @@ This feature helps in detecting and quantifying responses with incorrect formatt
 ```{admonition} **Custom use-case**
 :class: hint
 
-For monitoring other critical aspects that are specific to your use case. [Talk to founders](https://calendly.com/shahules/30min)
+For monitoring other critical aspects that are specific to your use case. [Talk to founders](https://calendly.com/shahules/30min)
diff --git a/docs/getstarted/testset_generation.md b/docs/getstarted/testset_generation.md
@@ -1,3 +1,4 @@
+(get-started-testset-generation)=
 # Synthetic test data generation
 
 This tutorial is designed to help you create a synthetic evaluation dataset for assessing your RAG pipeline. To achieve this, we will utilize open-ai models, so please ensure you have your OpenAI API key ready and accessible within your environment.
@@ -45,4 +46,4 @@ testset.to_pandas()
 ```
 <p align="left">
 <img src="../_static/imgs/testset_output.png" alt="test-outputs" width="800" height="600" />
-</p>
+</p>
diff --git a/docs/getstarted/testsetgeneration.ipynb b/docs/getstarted/testsetgeneration.ipynb
@@ -51,6 +51,7 @@
    "outputs": [],
    "source": [
     "import os\n",
+    "\n",
     "os.environ[\"OPENAI_API_KEY\"] = \"your-openai-key\""
    ]
   },
@@ -72,7 +73,7 @@
    "source": [
     "import json\n",
     "from llama_index import download_loader\n",
-    "from ragas.testset import TestsetGenerator\n"
+    "from ragas.testset import TestsetGenerator"
    ]
   },
   {
diff --git a/docs/howtos/index.md b/docs/howtos/index.md
@@ -1,10 +1,29 @@
+(how-to-guides)=
 # How-to Guides
 
 
-```{toctree}
-:caption: How-to Guides
-:hidden:
+The how-to guides offer a more comprehensive overview of all the tools Ragas
+offers and how to use them. This will help you tackle messier real-world
+usecases when you're using the framework to help build your own RAG pipelines.
+
+The guides assume you are familiar and confortable with the Ragas basics. If
+your not feel free to checkout the [Get Started](../getstarted/index.md)
+sections first.
+
+These guides are organized into 3 sections
+
+- [Customisations](): How to customise Ragas to use with your system. Things
+  like changing the LLMs used underneath are covered here.
+- [Applications](): Shows how to leverage Ragas framework to solve various
+  real-world problems that you might be facing too.
+- [Integrations](): Integrations with other tools like langchain and llamaindex,
+  making it super easy to add ragas into your setup.
+
+You can check the entire list bellow 👇
+
+**Index**
+:::{toctree}
 
 applications/index
 
-```
+:::
diff --git a/docs/index.md b/docs/index.md
@@ -1,13 +1,45 @@
-# Welcome
-
-Ragas is a framework that helps you evaluate your Retrieval Augmented Generation (RAG) pipelines. RAG denotes a class of LLM applications that use external data to augment the LLM’s context. There are existing tools and frameworks that help you build these pipelines but evaluating it and quantifying your pipeline performance can be hard. This is where Ragas (RAG Assessment) comes in.
-
-Ragas provides you with the tools based on the latest research for evaluating LLM-generated text to give you insights about your RAG pipeline. Ragas can be integrated with your CI/CD to provide continuous checks to ensure performance.
-
-```{toctree}
+# Introduction
+:::{toctree}
 :hidden:
 getstarted/index.md
 concepts/index.md
 howtos/index.md
-references/index.md
-```
+references/index.rst
+:::
+
+Ragas is a framework that helps you evaluate your Retrieval Augmented Generation (RAG) pipelines. RAG denotes a class of LLM applications that use external data to augment the LLM’s context. There are existing tools and frameworks that help you build these pipelines but evaluating it and quantifying your pipeline performance can be hard. This is where Ragas (RAG Assessment) comes in.
+
+
+::::{grid} 2
+
+:::{grid-item-card} Get Started
+:link: get-started
+:link-type: ref
+
+Learn the basics and become familiar with the Ragas metrics and how to evaluate
+RAG pipelines with it, how to generate synthetic testsets to run evaluations
+against and setting up online monitoring for your RAG apps.
+:::
+
+:::{grid-item-card} Core Concepts
+:link: core-concepts
+:link-type: ref
+
+The high-level explainations for building a better understand about the
+important topics such how to think about metrics-driven development, how the Ragas metrics work under the hood and synthetic dataset generation.
+:::
+:::{grid-item-card} How-to Guides
+:link: how-to-guides
+:link-type: ref
+
+Practical guides to help you achieve a specific goals. Take a look at these
+guides to learn how to use Ragas to solve real-world problems.
+:::
+
+:::{grid-item-card} References
+:link: references
+:link-type: ref
+
+Technical descriptions of how Ragas classes and methods work.
+:::
+::::
diff --git a/docs/references/evaluation.rst b/docs/references/evaluation.rst
@@ -0,0 +1,6 @@
+Evaluation
+==========
+
+.. autofunction:: ragas.evaluation.evaluate
+
+.. autoclass:: ragas.evaluation.Result
diff --git a/docs/references/index.md b/docs/references/index.md
diff --git a/docs/references/index.rst b/docs/references/index.rst
@@ -0,0 +1,10 @@
+.. _references:
+References
+==========
+
+Reference documents for the ``ragas`` package.
+
+.. toctree::
+   evaluation
+   metrics
+   integrations/index
diff --git a/docs/references/integrations/index.rst b/docs/references/integrations/index.rst
@@ -0,0 +1,6 @@
+Integrations
+============
+
+.. toctree::
+   langchain.rst
+   llamaindex.rst
diff --git a/docs/references/integrations/langchain.rst b/docs/references/integrations/langchain.rst
@@ -0,0 +1,5 @@
+Langchain
+=========
+
+.. automodule:: ragas.langchain
+   :members:
diff --git a/docs/references/integrations/llamaindex.rst b/docs/references/integrations/llamaindex.rst
@@ -0,0 +1,5 @@
+LlamaIndex
+==========
+
+.. automodule:: ragas.llama_index
+   :members:
diff --git a/docs/references/metrics.rst b/docs/references/metrics.rst
@@ -0,0 +1,13 @@
+Metrics
+=======
+
+.. autosummary::
+   ragas.metrics.faithfulness.faithfulness
+   ragas.metrics.answer_relevancy  
+   ragas.metrics.answer_similarity
+   ragas.metrics.answer_correctness
+   ragas.metrics.context_precision
+   ragas.metrics.context_recall
+
+.. automodule:: ragas.metrics
+   :members:
diff --git a/pyproject.toml b/pyproject.toml
@@ -13,6 +13,15 @@ dependencies = [
 ]
 dynamic = ["version", "readme"]
 
+[project.optional-dependencies]
+docs = [
+    "sphinx",
+    "sphinxawesome-theme@git+https://github.com/jjmachan/sphinxawesome-theme.git@fix/docsearch",
+    "myst-parser[linkify]",
+    "sphinx_design",
+    "astroid<3",
+]
+
 [tool.setuptools]
 package-dir = {"" = "src"}
 
diff --git a/requirements/docs.txt b/requirements/docs.txt
@@ -1,5 +1,5 @@
 sphinx
-git+https://github.com/kai687/sphinxawesome-theme.git
+git+https://github.com/jjmachan/sphinxawesome-theme.git@fix/docsearch
 myst-parser[linkify]
-myst-nb
 sphinx_design
+astroid<3
diff --git a/src/ragas/llama_index/evaluation.py b/src/ragas/llama_index/evaluation.py
diff --git a/src/ragas/metrics/llms.py b/src/ragas/metrics/llms.py

Original file line number	Diff line number	Diff line change
`@@ -51,6 +51,7 @@`
`51`	`51`	`"outputs": [],`
`52`	`52`	`"source": [`
`53`	`53`	`"import os\n",`
	`54`	`+ "\n",`
`54`	`55`	`"os.environ[\"OPENAI_API_KEY\"] = \"your-openai-key\""`
`55`	`56`	`]`
`56`	`57`	`},`
`@@ -72,7 +73,7 @@`
`72`	`73`	`"source": [`
`73`	`74`	`"import json\n",`
`74`	`75`	`"from llama_index import download_loader\n",`
`75`		`- "from ragas.testset import TestsetGenerator\n"`
	`76`	`+ "from ragas.testset import TestsetGenerator"`
`76`	`77`	`]`
`77`	`78`	`},`
`78`	`79`	`{`