Implementing database views (#56)

* working prototype of database view

* adding tests for views and SampleView to schema

* expanding view testing

* updating documentation with notes on using views

* adding missing spectra documentation

* fixing indentation error in documentation

Author: dr-rodriguez

astrodbkit2/schema_example.py

@@ -1,8 +1,10 @@
 # Example schema for part of the SIMPLE database
 
+import sqlalchemy as sa
 from sqlalchemy import Boolean, Column, Float, ForeignKey, Integer, String, BigInteger, Enum, Date, DateTime
 import enum
 from astrodbkit2.astrodb import Base
+from astrodbkit2.views import view
 
 
 # -------------------------------------------------------------------------------------------------------------------
@@ -82,3 +84,17 @@ class SpectralTypes(Base):
     best = Column(Boolean)  # flag for indicating if this is the best measurement or not
     comments = Column(String(1000))
     reference = Column(String(30), ForeignKey('Publications.name', ondelete='cascade'), primary_key=True)
+
+
+# -------------------------------------------------------------------------------------------------------------------
+# Views
+SampleView = view(
+        "SampleView",
+        Base.metadata,
+        sa.select(
+            Sources.source.label("source"),
+            Sources.ra.label("s_ra"),
+            Sources.dec.label("s_dec"),
+            SpectralTypes.spectral_type.label("spectral_type"),
+        ).select_from(Sources).join(SpectralTypes, Sources.source == SpectralTypes.source)
+        )

astrodbkit2/tests/test_astrodb.py

@@ -5,13 +5,14 @@
 import pytest
 import io
 import pandas as pd
-from sqlalchemy import select, func
+import sqlalchemy as sa
 from sqlalchemy.exc import IntegrityError
 from astropy.table import Table
 from astropy.coordinates import SkyCoord
 from astropy.units.quantity import Quantity
 from astropy.io import ascii
 from astrodbkit2.astrodb import Database, create_database, Base, copy_database_schema
+from astrodbkit2.views import view
 from astrodbkit2.schema_example import *
 try:
     import mock
@@ -338,6 +339,50 @@ def test_inventory(db):
     assert db.inventory('2MASS J13571237+1428398') == test_dict
 
 
+def test_views(db):
+    # Test database views
+
+    # Create one manually
+    PhotView = view(
+        "PhotView",
+        db.metadata,
+        sa.select(
+            db.Sources.c.source.label("source"),
+            db.Sources.c.ra.label("s_ra"),
+            db.Sources.c.dec.label("s_dec"),
+            db.Photometry.c.band.label("band"),
+            db.Photometry.c.magnitude.label("value"),
+        ).select_from(db.Sources).join(db.Photometry, db.Sources.c.source == db.Photometry.c.source)
+        )
+    # Explicitly create
+    with db.engine.begin() as conn:
+        db.metadata.create_all(conn)
+
+    # Query the view
+    t = db.query(PhotView).table()
+    print(t)
+    assert len(t) == 3 # 3 Photometry values for the single source
+
+    # Query one created in schema file
+    t = db.query(SampleView).table()
+    print(t)
+    assert len(t) == 1
+
+    # Test views are listed when inspected
+    insp = sa.inspect(db.engine)
+    view_list = insp.get_view_names()
+    assert 'PhotView' in view_list and 'SampleView' in view_list
+
+    # Reflect a view and query it
+    ViewCopy = sa.Table('SampleView', sa.MetaData())  
+    insp.reflect_table(ViewCopy, include_columns=None)
+    assert db.query(ViewCopy).count() == 1
+
+    # Confirm that views are not used in inventory
+    assert 'PhotView' not in db.inventory('2MASS J13571237+1428398').keys()
+    assert 'SampleView' not in db.inventory('2MASS J13571237+1428398').keys()
+
+
 def test_save_reference_table(db, db_dir):
     # Test saving a reference table
     if os.path.exists(os.path.join(db_dir, 'Publications.json')):

astrodbkit2/tests/test_views.py

@@ -0,0 +1,83 @@
+# Tests for views
+# Adapted from https://github.com/sqlalchemy/sqlalchemy/wiki/Views
+
+import sqlalchemy as sa
+from sqlalchemy.orm import declarative_base
+from sqlalchemy.orm import Session
+from astrodbkit2.views import *
+
+
+def test_views():
+    engine = sa.create_engine("sqlite://", echo=True)
+    metadata = sa.MetaData()
+    stuff = sa.Table(
+        "stuff",
+        metadata,
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column("data", sa.String(50)),
+    )
+
+    more_stuff = sa.Table(
+        "more_stuff",
+        metadata,
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column("stuff_id", sa.Integer, sa.ForeignKey("stuff.id")),
+        sa.Column("data", sa.String(50)),
+    )
+
+    # the .label() is to suit SQLite which needs explicit label names
+    # to be given when creating the view
+    # See http://www.sqlite.org/c3ref/column_name.html
+    stuff_view = view(
+        "stuff_view",
+        metadata,
+        sa.select(
+            stuff.c.id.label("id"),
+            stuff.c.data.label("data"),
+            more_stuff.c.data.label("moredata"),
+        )
+        .select_from(stuff.join(more_stuff))
+        .where(stuff.c.data.like(("%orange%"))),
+    )
+
+    assert stuff_view.primary_key == [stuff_view.c.id]
+
+    with engine.begin() as conn:
+        metadata.create_all(conn)
+
+    with engine.begin() as conn:
+        conn.execute(
+            stuff.insert(),
+            [
+                {"data": "apples"},
+                {"data": "pears"},
+                {"data": "oranges"},
+                {"data": "orange julius"},
+                {"data": "apple jacks"},
+            ],
+        )
+
+        conn.execute(
+            more_stuff.insert(),
+            [
+                {"stuff_id": 3, "data": "foobar"},
+                {"stuff_id": 4, "data": "foobar"},
+            ],
+        )
+
+    with engine.connect() as conn:
+        assert conn.execute(
+            sa.select(stuff_view.c.data, stuff_view.c.moredata)
+        ).all() == [("oranges", "foobar"), ("orange julius", "foobar")]
+
+    # illustrate ORM usage
+    Base = declarative_base(metadata=metadata)
+
+    class MyStuff(Base):
+        __table__ = stuff_view
+
+        def __repr__(self):
+            return f"MyStuff({self.id!r}, {self.data!r}, {self.moredata!r})"
+
+    with Session(engine) as s:
+        assert s.query(MyStuff).count() == 2

astrodbkit2/views.py

@@ -0,0 +1,57 @@
+# Logic to implement and set up views in SQLAlchemy
+# Adapted from https://github.com/sqlalchemy/sqlalchemy/wiki/Views
+
+import sqlalchemy as sa
+from sqlalchemy.ext import compiler
+from sqlalchemy.schema import DDLElement
+from sqlalchemy.sql import table
+
+
+class CreateView(DDLElement):
+    def __init__(self, name, selectable):
+        self.name = name
+        self.selectable = selectable
+
+
+class DropView(DDLElement):
+    def __init__(self, name):
+        self.name = name
+
+
+@compiler.compiles(CreateView)
+def _create_view(element, compiler, **kw):
+    return "CREATE VIEW %s AS %s" % (
+        element.name,
+        compiler.sql_compiler.process(element.selectable, literal_binds=True),
+    )
+
+
+@compiler.compiles(DropView)
+def _drop_view(element, compiler, **kw):
+    return "DROP VIEW %s" % (element.name)
+
+
+def view_exists(ddl, target, connection, **kw):
+    return ddl.name in sa.inspect(connection).get_view_names()
+
+
+def view_doesnt_exist(ddl, target, connection, **kw):
+    return not view_exists(ddl, target, connection, **kw)
+
+
+def view(name, metadata, selectable):
+    t = table(name)
+
+    t._columns._populate_separate_keys(
+        col._make_proxy(t) for col in selectable.selected_columns
+    )
+
+    sa.event.listen(
+        metadata,
+        "after_create",
+        CreateView(name, selectable).execute_if(callable_=view_doesnt_exist),
+    )
+    sa.event.listen(
+        metadata, "before_drop", DropView(name).execute_if(callable_=view_exists)
+    )
+    return t

docs/index.rst

@@ -310,6 +310,42 @@ for example `$ASTRODB_SPECTRA/infrared/myfile.fits`.
 **AstrodbKit2** would examine the environment variable `$ASTRODB_SPECTRA` and use that as
 part of the absolute path to the file.
 
+Working With Views
+------------------
+
+If your database contains views, they will not be included in the inventory methods or in the output JSON files. 
+However, you can still work with and query data in them. 
+The best way to define them is in the schema file, for example, with something like::
+
+    SampleView = view(
+        "SampleView",
+        Base.metadata,
+        sa.select(
+            Sources.source.label("source"),
+            Sources.ra.label("s_ra"),
+            Sources.dec.label("s_dec"),
+            SpectralTypes.spectral_type.label("spectral_type"),
+        ).select_from(Sources).join(SpectralTypes, Sources.source == SpectralTypes.source)
+        )
+
+When created, the database will contain these views and users can reflect them to access them::
+    
+    import sqlalchemy as sa
+
+    # Inspect the database to get a list of available views
+    insp = sa.inspect(db.engine)
+    print(insp.get_view_names())
+
+    # Reflect a view and query it
+    SampleView = sa.Table('SampleView', sa.MetaData())  
+    insp.reflect_table(SampleView, include_columns=None)
+    db.query(SampleView).table()
+
+It is important that `sa.MetaData()` be used in the `sa.Table()` call as you don't want to modify 
+the metadata of the actual database. 
+If you don't do this and instead use `db.metadata`, you may end up with errors in other parts of AstrodbKit2 
+functionality as the view will be treated as a physical table.
+
 Modifying Data
 ==============