FIREFLY-1862: Cleanup the firefly-client docs and example notebooks

.github/workflows/publish-docs.yml

@@ -4,6 +4,10 @@ on:
     branches:
       - master
 
+  pull_request:
+    branches:
+      - master
+
   workflow_dispatch: # Manual trigger
 
 jobs:
@@ -13,6 +17,11 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Install system dependencies (pandoc)
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y pandoc
+
       - uses: actions/setup-python@v5
         with:
           python-version: '3.10'
@@ -33,9 +42,25 @@ jobs:
           make clean
           make html
 
-      - name: Deploy docs
+      # Deploy stable docs (root of gh-pages) only from master pushes
+      - name: Deploy docs (stable)
+        if: github.event_name == 'push' && github.ref == 'refs/heads/master'
         uses: peaceiris/actions-gh-pages@v3
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./docs/_build/html
 
+      # Deploy PR preview docs to gh-pages under pr/<number>/
+      - name: Deploy docs (PR preview)
+        if: github.event_name == 'pull_request'
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./docs/_build/html
+          destination_dir: pr/${{ github.event.pull_request.number }}
+          keep_files: true
+
+      - name: Print PR preview URL
+        if: github.event_name == 'pull_request'
+        run: |
+          echo "Deployed PR docs preview at: https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/pr/${{ github.event.pull_request.number }}/"

README.md

@@ -7,6 +7,5 @@ Python API for [Firefly](http://github.com/Caltech-IPAC/firefly), IPAC's Advance
 
 You can find the documentation at https://caltech-ipac.github.io/firefly_client.
 
-NOTE: Many parts of this documentation are a work in progress because the firefly_client API and its use cases evolved since it was written.
 Please report any issues or suggestions on the [GitHub issues](https://github.com/Caltech-IPAC/firefly_client/issues).
 

docs/_ext/script_pages.py

@@ -0,0 +1,223 @@
+"""Sphinx extension: generate per-script RST pages, copy .py into built _sources, and rewrite source links."""
+import os
+import glob
+import io
+from sphinx.util import logging
+
+logger = logging.getLogger(__name__)
+
+MARKER_TEMPLATE = '.. AUTO-GENERATED from {}'
+
+
+def _title_from_script(path):
+    """Extract a title from the first commented heading line; fallback to filename."""
+    title = os.path.splitext(os.path.basename(path))[0]
+    try:
+        with io.open(path, 'r', encoding='utf-8') as fh:
+            first_line = fh.readline()
+            # Skip shebang (#!) if present
+            if first_line and first_line.lstrip().startswith('#!'):
+                first_line = fh.readline()
+            # Look for first comment line
+            title_comment = None
+            if first_line and first_line.lstrip().startswith('#'):
+                title_comment = first_line.strip().lstrip('#').strip()
+            if title_comment:
+                title = title_comment
+    except Exception:
+        pass
+    return title
+
+
+def _generate_rst_for_scripts(app):
+    """Create a <name>.rst next to each <name>.py under docs/usage/examples/.
+
+    - If `<name>.rst` already exists and does NOT start with our auto-generated marker,
+      we leave it alone (so user-written RST isn't overwritten).
+    - Otherwise we create or overwrite the RST to include the script via literalinclude.
+    """
+    logger = logging.getLogger(__name__)
+    srcdir = app.srcdir
+
+    patterns = [
+        os.path.join(srcdir, 'usage', 'examples', '*.py'),
+    ]
+    py_files = []
+    for pat in patterns:
+        py_files.extend(glob.glob(pat))
+    py_files = sorted(set(py_files))
+
+    generated = 0
+
+    for script_fpath in py_files:
+        script_basename = os.path.basename(script_fpath)
+        # Skip conf.py and other build files
+        if script_basename in ('conf.py',):
+            continue
+
+        rst_path = os.path.splitext(script_fpath)[0] + '.rst'
+        marker = MARKER_TEMPLATE.format(script_basename)
+
+        # If rst exists and not generated by us, skip it
+        if os.path.exists(rst_path):
+            try:
+                with io.open(rst_path, 'r', encoding='utf-8') as rh:
+                    first = rh.readline().strip()
+                if first != marker:
+                    logger.info(f'Skipping existing RST: {rst_path}')
+                    continue
+            except Exception:
+                # If we can't read, skip to be safe
+                logger.warning(f'Could not read existing RST {rst_path}; skipping')
+                continue
+
+        # Build content
+        title = _title_from_script(script_fpath)
+        underline = '=' * len(title)
+        # Make the literalinclude path relative to the generated RST location
+        rst_dir = os.path.dirname(rst_path)
+        rel = os.path.relpath(script_fpath, rst_dir)
+        lines = [
+            marker,
+            '',
+            title,
+            underline,
+            '',
+            f'.. literalinclude:: {rel}',
+            '   :language: python',
+            '   :linenos:',
+            '',
+        ]
+
+        try:
+            with io.open(rst_path, 'w', encoding='utf-8') as oh:
+                oh.write('\n'.join(lines))
+            generated += 1
+            logger.info(f'Generated {rst_path} from {script_basename}')
+        except Exception as e:
+            logger.warning(f'Failed to write {rst_path}: {e}')
+
+    logger.info(f'Generated {generated} script RST files')
+
+
+def _remove_generated_rst(app, exception):
+    """Remove RST files that were auto-generated for scripts and copy .py into built _sources.
+
+    Safety rules:
+    - Only remove files whose first line starts with the auto-generated marker
+      ('.. AUTO-GENERATED from ...'). This avoids deleting user-authored RST.
+    - Only remove files after a successful build (exception is None).
+    """
+    logger = logging.getLogger(__name__)
+
+    # If build failed, do not remove files so devs can inspect outputs
+    if exception is not None:
+        logger.info('Build failed; leaving auto-generated RST files in place')
+        return
+
+    srcdir = app.srcdir
+    patterns = [
+        os.path.join(srcdir, 'usage', 'examples', '*.rst'),
+    ]
+
+    rst_files = []
+    for pat in patterns:
+        rst_files.extend(glob.glob(pat))
+
+    removed = 0
+    for rst in sorted(set(rst_files)):
+        try:
+            with io.open(rst, 'r', encoding='utf-8') as fh:
+                first = fh.readline().strip()
+            if first.startswith(MARKER_TEMPLATE.format('')):
+                # If building HTML, copy the original .py into the built _sources
+                # directory so the "View source" link can serve the real Python file.
+                try:
+                    py_full = os.path.splitext(rst)[0] + '.py'
+                    if os.path.exists(py_full) and getattr(app, 'builder', None) is not None:
+                        try:
+                            builder_name = getattr(app.builder, 'name', None)
+                            outdir = getattr(app.builder, 'outdir', None)
+                        except Exception:
+                            builder_name = None
+                            outdir = None
+
+                        if builder_name == 'html' and outdir:
+                            dest_dir = os.path.join(outdir, '_sources')
+                            rel_py = os.path.relpath(py_full, app.srcdir)
+                            dest_path = os.path.join(dest_dir, rel_py)
+                            os.makedirs(os.path.dirname(dest_path), exist_ok=True)
+                            # Copy file contents
+                            try:
+                                with io.open(py_full, 'r', encoding='utf-8') as rf, io.open(dest_path, 'w', encoding='utf-8') as wf:
+                                    wf.write(rf.read())
+                                logger.info(f'Copied Python source to built _sources: {dest_path}')
+                            except Exception as e:
+                                logger.warning(f'Failed to write built source {dest_path}: {e}')
+
+                except Exception:
+                    # non-fatal; continue to removal attempt
+                    pass
+
+                try:
+                    os.remove(rst)
+                    removed += 1
+                    logger.info(f'Removed generated RST: {rst}')
+                except Exception as e:
+                    logger.warning(f'Failed to remove generated RST {rst}: {e}')
+        except Exception as e:
+            logger.warning(f'Could not read {rst}; skipping removal: {e}')
+
+    logger.info(f'Removed {removed} auto-generated script RST files')
+
+
+def _rewrite_sourcelink_to_py(app, pagename, templatename, context, doctree):
+    """If the current page was generated from a script RST, point "View source" to the .py.
+
+    This mirrors nbsphinx behaviour for notebooks where the "Show source" link
+    points to the original notebook instead of the generated RST.
+    """
+    try:
+        # doc2path with base=None returns a path relative to srcdir
+        rst_rel = app.env.doc2path(pagename, base=None)
+    except Exception:
+        return
+
+    rst_full = os.path.join(app.srcdir, rst_rel)
+    if not os.path.exists(rst_full):
+        return
+
+    try:
+        with io.open(rst_full, 'r', encoding='utf-8') as fh:
+            first = fh.readline().strip()
+    except Exception:
+        return
+
+    # Only rewrite for our auto-generated script RST files
+    if not first.startswith(MARKER_TEMPLATE.format('')):
+        return
+
+    py_full = os.path.splitext(rst_full)[0] + '.py'
+    if not os.path.exists(py_full):
+        return
+
+    # Make the path relative to the source directory (what Sphinx expects)
+    rel_py = os.path.relpath(py_full, app.srcdir)
+
+    # Replace the template context variable so the HTML theme will link to .py
+    context['sourcename'] = rel_py
+
+
+def setup(app):
+    # Generate per-script RST files before reading sources
+    app.connect('builder-inited', _generate_rst_for_scripts)
+    # Remove the generated per-script RST files after a successful build
+    app.connect('build-finished', _remove_generated_rst)
+    # Make "View source" point to the original .py for auto-generated script pages
+    app.connect('html-page-context', _rewrite_sourcelink_to_py)
+
+    return {
+        'version': '1.0',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }

docs/conf.py

@@ -6,20 +6,32 @@
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
+# Make local extension modules under _ext/ importable
+import os
+import sys
+sys.path.insert(0, os.path.abspath('_ext'))
+
 project = 'firefly_client'
-copyright = '2024, Caltech/IPAC Firefly Developers'
+copyright = '2026, Caltech/IPAC Firefly Developers'
 author = 'Caltech/IPAC Firefly Developers'
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
 
 extensions = [
     'sphinx_automodapi.automodapi',
-    'myst_parser'
+    'myst_parser',
+    'nbsphinx',
+    'script_pages', # custom extension for auto-generating RST files for examples/*.py scripts
 ]
 
 templates_path = ['_templates']
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = [
+    '_build', 'Thumbs.db', '.DS_Store',
+    # exclude any docs in subdirectories under usage/examples/ since they are
+    # not included in any toctree and are for internal use only
+    'usage/examples/*/**'
+]
 
 
 # -- Options for HTML output -------------------------------------------------
@@ -52,3 +64,16 @@
 
 # -- Options for extensions -------------------------------------------------
 myst_heading_anchors = 3
+
+# nbsphinx configuration: render notebooks and Python scripts
+# Do not execute notebooks during docs build; use stored outputs if present.
+nbsphinx_execute = 'never'
+
+# Allow build to continue even if some notebooks error (useful for demos).
+nbsphinx_allow_errors = True
+
+# Ensure Pygments syntax highlighting for Jupyter code cells.
+nbsphinx_codecell_lexer = 'ipython'
+
+# Remove the .txt suffix that gets added to source files
+html_sourcelink_suffix = ''

docs/development/guide.rst

@@ -4,18 +4,37 @@ Local development
 
 Get latest source from ``master`` branch at https://github.com/Caltech-IPAC/firefly_client.
 
+.. code-block:: shell
+
+    git clone https://github.com/Caltech-IPAC/firefly_client.git
+    cd firefly_client
+
 
 Environment Setup
 -----------------
 
-TBD
+Create a Python virtual environment and install required dependencies.
+The folllowing commands demonstrate how to do this using ``conda`` (assuming you have `miniconda <https://www.anaconda.com/docs/getting-started/miniconda/>`_ installed on your system):
+
+.. code-block:: shell
+
+    conda create -n ffpy -c conda-forge python jupyter astropy # jupyter and astropy are needed for running examples
+    conda activate ffpy
+    pip install -e ".[docs]" # editable installation with docs dependencies
 
 
+Now you can run the examples notebooks/scripts in the ``examples/`` directory.
+This can be done by starting a Jupyter Notebook or JupyterLab session from the terminal (``jupyter notebook`` or ``jupyter lab``), or by using an IDE that supports running Python notebooks or scripts (like VSCode, IntelliJ, etc.).
+
+.. note::
+    The changes you make to the source code will be reflected when you run the examples since the package is installed in editable mode.
+    But make sure to restart the active Python kernel/session to pick up the changes.
+
 Building documentation
 ----------------------
 
-Make sure you have the virtual/conda environment activated and documentation
-dependencies installed in that environment.
+Make sure you have the virtual environment activated and documentation
+dependencies installed in that environment (``[docs]``).
 
 Then do:
 
@@ -28,4 +47,13 @@ Open ``docs/_build/html/index.html`` in your browser to see the documentation
 website. 
 
 Each time you make a change in documentation source, build it using 
-above command and reload the above file in browser.
+above command and reload the above html file in browser.
+
+.. note::
+    The Sphinx docs include rendered Jupyter notebooks (via ``nbsphinx``), which can require **pandoc**.
+    If you see an error like ``PandocMissing``, install pandoc first (e.g., ``brew install pandoc`` on macOS).
+
+Development Tests/Examples
+--------------------------
+
+Refer to the `examples/development_tests directory <https://github.com/Caltech-IPAC/firefly_client/tree/master/examples/development_tests>`_ of firefly-client GitHub repository.