Associate latent variables with generators.

Add bayesdb_add_latent to add a latent variable from a metamodel's
create_generator operation.

Two generators may not currently share an overlapping latent variable
name.  (XXX It is not clear that this is the right design choice.  We
could require the variable to be qualified by the generator's name if
ambiguous.  But we'll defer that for now.)

Author: riastradh-probcomp

src/ast.py

@@ -78,7 +78,6 @@
     'name',
 ])
 PopVar = namedtuple('PopVar', [
-    'latent',
     'name',
     'stattype',
 ])

src/bql.py

@@ -532,7 +532,6 @@ def _create_population(bdb, phrase):
     duplicates = set()
     missing = set()
     invalid = set()
-    nlatent = 0
     colno_sql = '''
         SELECT colno FROM bayesdb_column
             WHERE tabname = :table AND name = :column_name
@@ -541,16 +540,11 @@ def _create_population(bdb, phrase):
         SELECT COUNT(*) FROM bayesdb_stattype WHERE name = :stattype
     '''
     for variable in phrase.schema:
-        latent = variable.latent
         name = casefold(variable.name)
         stattype = casefold(variable.stattype)
         if name in variable_map:
             duplicates.add(name)
             continue
-        if latent:
-            nlatent += 1
-            variable_map[name] = -nlatent
-            continue
         cursor = bdb.sql_execute(colno_sql, {
             'table': phrase.table,
             'column_name': name,

src/core.py

@@ -278,6 +278,23 @@ def bayesdb_variable_stattype(bdb, population_id, colno):
         assert len(row) == 1
         return row[0]
 
+def bayesdb_add_latent(bdb, population_id, generator_id, var, stattype):
+    """Add a generator's latent variable to a population.
+
+    NOTE: To be used ONLY by a metamodel's create_generator method
+    when establishing any latent variables of that generator.
+    """
+    with bdb.savepoint():
+        cursor = bdb.sql_execute('''
+            SELECT MIN(colno) FROM bayesdb_variable WHERE population_id = ?
+        ''', (population_id,))
+        colno = min(-1, cursor_value(cursor) - 1)
+        bdb.sql_execute('''
+            INSERT INTO bayesdb_variable
+                (population_id, colno, name, stattype)
+                VALUES (?, ?, ?, ?)
+        ''', (population_id, colno, var, stattype))
+
 def bayesdb_population_cell_value(bdb, population_id, rowid, colno):
     if colno < 0:
         # Latent variables do not appear in the table.

src/grammar.y

@@ -72,7 +72,6 @@ command(drop_pop)	::= K_DROP K_POPULATION ifexists(ifexists)
 pop_schema(one)		::= pop_var(var).
 pop_schema(many)	::= pop_schema(schema) T_COMMA pop_var(var).
 pop_var(manifest)	::= column_name(name) stattype(st).
-pop_var(latent)		::= K_LATENT column_name(name) stattype(st).
 stattype(st)		::= L_NAME(name).
 
 /* XXX Temporary generators?  */

src/parse.py

@@ -182,10 +182,7 @@ def p_command_drop_pop(self, ifexists, name):
 
     def p_pop_schema_one(self, var):            return [var]
     def p_pop_schema_many(self, schema, var): schema.append(var); return schema
-    def p_pop_var_manifest(self, name, st):
-        return ast.PopVar(False, name, st)
-    def p_pop_var_latent(self, name, st):
-        return ast.PopVar(True, name, st)
+    def p_pop_var_manifest(self, name, st):     return ast.PopVar(name, st)
     def p_stattype_st(self, name):              return name
 
     def p_command_creategen(self, ifnotexists0, name, ifnotexists1, pop,

src/schema.py

@@ -131,12 +131,15 @@
 
 CREATE TABLE bayesdb_variable (
 	population_id	INTEGER NOT NULL REFERENCES bayesdb_population(id),
+	generator_id	INTEGER REFERENCES bayesdb_generator(id),
 	colno		INTEGER NOT NULL,
-        name            TEXT COLLATE NOCASE NOT NULL,
+	name		TEXT COLLATE NOCASE NOT NULL,
 	stattype	TEXT COLLATE NOCASE NOT NULL
 				REFERENCES bayesdb_stattype(name),
 	PRIMARY KEY(population_id, colno),
-	UNIQUE(population_id, name)
+	UNIQUE(population_id, name),
+	UNIQUE(generator_id, colno),
+	UNIQUE(generator_id, name)
 );
 
 INSERT INTO bayesdb_variable (population_id, name, colno, stattype)

tests/test_bql.py

@@ -101,14 +101,6 @@ def test_trivial_population():
         # XXX if (not) exists
         bdb.execute('create population p for t(age numerical)')
         bdb.execute('drop population p')
-        bdb.execute('''
-            create population p for t (
-                age numerical,
-                latent cluster categorical,
-                latent error numerical
-            )
-        ''')
-        bdb.execute('drop population p')
 
 @stochastic(max_runs=2, min_passes=1)
 def test_conditional_probability(seed):

tests/test_cgpm.py

@@ -139,8 +139,7 @@ def test_cgpm():
                 country_of_operator CATEGORICAL,
                 launch_mass NUMERICAL,
                 perigee NUMERICAL,
-                period NUMERICAL,
-                latent kepler_error NUMERICAL
+                period NUMERICAL
             )
         ''')
         bdb.execute('''

tests/test_guess.py

@@ -121,8 +121,8 @@ def test_guess_population():
         # Population already exists.
         bayesdb_guess_population(bdb, 'p', 't')
     assert bdb.sql_execute('SELECT * FROM bayesdb_variable').fetchall() == [
-        (1, 1, 'y', 'categorical'),
-        (1, 2, 'z', 'numerical'),
+        (1, None, 1, 'y', 'categorical'),
+        (1, None, 2, 'z', 'numerical'),
     ]
 
 def isqrt(n):

tests/test_parse.py

@@ -492,18 +492,16 @@ def test_trivial_commands():
             launch_mass numerical,
             perigee numerical,
             apogee numerical,
-            period numerical,
-            latent kepler_error numerical
+            period numerical
         )
     ''') == \
         [ast.CreatePop(False, 'satellites', 'satellites_ucs', [
-            ast.PopVar(False, 'country_of_operator', 'categorical'),
-            ast.PopVar(False, 'orbit_type', 'categorical'),
-            ast.PopVar(False, 'launch_mass', 'numerical'),
-            ast.PopVar(False, 'perigee', 'numerical'),
-            ast.PopVar(False, 'apogee', 'numerical'),
-            ast.PopVar(False, 'period', 'numerical'),
-            ast.PopVar(True, 'kepler_error', 'numerical'),
+            ast.PopVar('country_of_operator', 'categorical'),
+            ast.PopVar('orbit_type', 'categorical'),
+            ast.PopVar('launch_mass', 'numerical'),
+            ast.PopVar('perigee', 'numerical'),
+            ast.PopVar('apogee', 'numerical'),
+            ast.PopVar('period', 'numerical'),
         ])]
     assert parse_bql_string('drop population satellites') == \
         [ast.DropPop(False, 'satellites')]