Tie up various loose ends around latent variables.

- Make bayesdb_add_latent actually associate the generator with it.
- Require caller to specify generator id or None in
  . core.bayesdb_has_variable
  . core.bayesdb_variable_number
- Teach compiler to reject attempts to use latent variables when no
  generator is specified, or when they are not latent variables of the
  specified generator.
- Fix various MODELLED BY mistakes in the compiler.
- Fix bayesdb_variable table's generator_id column to cascade on
  delete so that DROP GENERATOR works again with latent variables.

Author: riastradh-probcomp

src/bql.py

@@ -148,12 +148,15 @@ def execute_phrase(bdb, phrase, bindings=()):
             nsamples = cursor[0][0]
             assert isinstance(nsamples, int)
             constraints = [
-                (core.bayesdb_variable_number(bdb, population_id, var), value)
+                (core.bayesdb_variable_number(bdb, population_id, generator_id,
+                        var),
+                    value)
                 for (var, _expression), value in
                     zip(phrase.simulation.constraints, cursor[0][1:])
             ]
             colnos = [
-                core.bayesdb_variable_number(bdb, population_id, var)
+                core.bayesdb_variable_number(bdb, population_id, generator_id,
+                    var)
                 for var in column_names
             ]
             bdb.sql_execute('CREATE %sTABLE %s%s (%s)' %

src/compiler.py

@@ -208,20 +208,40 @@ def bayesdb_population_generators(bdb, population_id):
     ''', (population_id,))
     return [generator_id for (generator_id,) in cursor]
 
-def bayesdb_has_variable(bdb, population_id, name):
-    """True if the population has a given variable."""
-    cursor = bdb.sql_execute('''
-        SELECT COUNT(*) FROM bayesdb_variable
-            WHERE population_id = ? AND name = ?
-    ''', (population_id, name))
+def bayesdb_has_variable(bdb, population_id, generator_id, name):
+    """True if the population has a given variable.
+
+    generator_id is None for manifest variables and the id of a
+    generator for variables that may be latent.
+    """
+    if generator_id is None:
+        cursor = bdb.sql_execute('''
+            SELECT COUNT(*) FROM bayesdb_variable
+                WHERE population_id = ? AND generator_id IS NULL AND name = ?
+        ''', (population_id, name))
+    else:
+        cursor = bdb.sql_execute('''
+            SELECT COUNT(*) FROM bayesdb_variable
+                WHERE population_id = ?
+                    AND (generator_id IS NULL OR generator_id = ?)
+                    AND name = ?
+        ''', (population_id, generator_id, name))
     return cursor_value(cursor) != 0
 
-def bayesdb_variable_number(bdb, population_id, name):
+def bayesdb_variable_number(bdb, population_id, generator_id, name):
     """Return the column number of a population variable."""
-    cursor = bdb.sql_execute('''
-        SELECT colno FROM bayesdb_variable
-            WHERE population_id = ? AND name = ?
-    ''', (population_id, name))
+    if generator_id is None:
+        cursor = bdb.sql_execute('''
+            SELECT colno FROM bayesdb_variable
+                WHERE population_id = ? AND generator_id IS NULL AND name = ?
+        ''', (population_id, name))
+    else:
+        cursor = bdb.sql_execute('''
+            SELECT colno FROM bayesdb_variable
+                WHERE population_id = ?
+                    AND (generator_id IS NULL OR generator_id = ?)
+                    AND name = ?
+        ''', (population_id, generator_id, name))
     return cursor_value(cursor)
 
 def bayesdb_variable_names(bdb, population_id):
@@ -234,7 +254,7 @@ def bayesdb_variable_numbers(bdb, population_id):
     """Return a list of the numbers of columns modelled in `population_id`."""
     sql = '''
         SELECT colno FROM bayesdb_variable
-            WHERE population_id = ?
+            WHERE population_id = ? AND generator_id IS NULL
             ORDER BY colno ASC
     '''
     return [row[0] for row in bdb.sql_execute(sql, (population_id,))]
@@ -291,9 +311,17 @@ def bayesdb_add_latent(bdb, population_id, generator_id, var, stattype):
         colno = min(-1, cursor_value(cursor) - 1)
         bdb.sql_execute('''
             INSERT INTO bayesdb_variable
-                (population_id, colno, name, stattype)
-                VALUES (?, ?, ?, ?)
-        ''', (population_id, colno, var, stattype))
+                (population_id, generator_id, colno, name, stattype)
+                VALUES (?, ?, ?, ?, ?)
+        ''', (population_id, generator_id, colno, var, stattype))
+
+def bayesdb_has_latent(bdb, population_id, var):
+    """True if the population has a latent variable by the given name."""
+    cursor = bdb.sql_execute('''
+        SELECT COUNT(*) FROM bayesdb_variable
+            WHERE population_id = ? AND name = ? AND generator_id IS NOT NULL
+    ''', (population_id, var))
+    return cursor_value(cursor)
 
 def bayesdb_population_cell_value(bdb, population_id, rowid, colno):
     if colno < 0:

src/core.py

@@ -260,7 +260,8 @@ def retrieve_analyze_variables(ast):
         # Retrieve target variables.
         analyze_ast = cgpm_analyze.parse.parse(program)
         variables = retrieve_analyze_variables(analyze_ast)
-        varnos = [core.bayesdb_variable_number(bdb, population_id, v)
+        varnos = [core.bayesdb_variable_number(bdb, population_id,
+                generator_id, v)
             for v in variables]
 
         # Run transition.
@@ -399,7 +400,8 @@ def _unique_rowid(self, rowids):
     def _data(self, bdb, generator_id, vars):
         # Get the column numbers and statistical types.
         population_id = core.bayesdb_generator_population(bdb, generator_id)
-        colnos = [core.bayesdb_variable_number(bdb, population_id, var)
+        colnos = [core.bayesdb_variable_number(bdb, population_id,
+                generator_id, var)
             for var in vars]
         stattypes = [core.bayesdb_variable_stattype(bdb, population_id, colno)
             for colno in colnos]
@@ -436,7 +438,8 @@ def map_value(colno, value):
     def _initialize_engine(self, bdb, generator_id, n, variables):
         population_id = core.bayesdb_generator_population(bdb, generator_id)
         def map_var(var):
-            return core.bayesdb_variable_number(bdb, population_id, var)
+            return core.bayesdb_variable_number(bdb, population_id,
+                generator_id, var)
         outputs = [map_var(var) for var, _st, _cct, _da in variables]
         cctypes = [cctype for _n, _st, cctype, _da in variables]
         distargs = [distargs for _n, _st, _cct, distargs in variables]
@@ -449,7 +452,8 @@ def map_var(var):
     def _initialize_cgpm(self, bdb, generator_id, cgpm_ext):
         population_id = core.bayesdb_generator_population(bdb, generator_id)
         def map_var(var):
-            return core.bayesdb_variable_number(bdb, population_id, var)
+            return core.bayesdb_variable_number(bdb, population_id,
+                generator_id, var)
         name = cgpm_ext['name']
         outputs = map(map_var, cgpm_ext['outputs'])
         inputs = map(map_var, cgpm_ext['inputs'])
@@ -632,11 +636,11 @@ def _create_schema(bdb, generator_id, schema_ast):
     duplicate = set()
     unknown = set()
     needed = set()
-    invalid_latent = set()
+    existing_latent = set()
     must_exist = []
 
     def _retrieve_stattype_dist_params(var):
-        colno = core.bayesdb_variable_number(bdb, population_id, var)
+        colno = core.bayesdb_variable_number(bdb, population_id, None, var)
         stattype = core.bayesdb_variable_stattype(bdb, population_id, colno)
         dist, params = _DEFAULT_DIST[stattype](bdb, generator_id, var)
         return stattype, dist, params
@@ -652,7 +656,7 @@ def _retrieve_stattype_dist_params(var):
             params = dict(clause.params) # XXX error checking
 
             # Reject if the variable does not exist.
-            if not core.bayesdb_has_variable(bdb, population_id, var):
+            if not core.bayesdb_has_variable(bdb, population_id, None, var):
                 unknown.add(var)
                 continue
 
@@ -661,12 +665,15 @@ def _retrieve_stattype_dist_params(var):
                 duplicate.add(var)
                 continue
 
-            # Reject if it is a latent variable.
-            colno = core.bayesdb_variable_number(bdb, population_id, var)
-            if colno < 0:
-                invalid_latent.add(var)
+            # Reject if the variable is latent.
+            if core.bayesdb_has_latent(bdb, population_id, var):
+                existing_latent.add(var)
                 continue
 
+            # Get the column number.
+            colno = core.bayesdb_variable_number(bdb, population_id, None, var)
+            assert not colno < 0
+
             # Add it to the list and mark it modelled by default.
             stattype = core.bayesdb_variable_stattype(
                 bdb, population_id, colno)
@@ -686,10 +693,16 @@ def _retrieve_stattype_dist_params(var):
 
             # Reject if the variable even *exists* in the population
             # at all yet.
-            if core.bayesdb_has_variable(bdb, population_id, var):
+            if core.bayesdb_has_variable(bdb, population_id, None, var):
                 duplicate.add(var)
                 continue
 
+            # Reject if the variable is already latent, from another
+            # generator.
+            if core.bayesdb_has_latent(bdb, population_id, var):
+                existing_latent.add(var)
+                continue
+
             # Add it to the set of latent variables.
             latents[var] = stattype
 
@@ -751,7 +764,7 @@ def _retrieve_stattype_dist_params(var):
     # Make sure all the outputs and inputs exist, either in the
     # population or as latents in this generator.
     for var in must_exist:
-        if core.bayesdb_has_variable(bdb, population_id, var):
+        if core.bayesdb_has_variable(bdb, population_id, None, var):
             continue
         if var in latents:
             continue
@@ -765,17 +778,17 @@ def _retrieve_stattype_dist_params(var):
     if unknown:
         raise BQLError(bdb, 'Unknown model variables: %r' %
             (sorted(unknown),))
-    if invalid_latent:
-        raise BQLError(bdb, 'Invalid latent variables: %r' %
-            (sorted(invalid_latent),))
+    if existing_latent:
+        raise BQLError(bdb, 'Latent variables already defined: %r' %
+            (sorted(existing_latent),))
 
     # Use the default distribution for any variables that remain to be
     # modelled, excluding any that are latent or that have statistical
     # types we don't know about.
     for var in core.bayesdb_variable_names(bdb, population_id):
         if var in modelled:
             continue
-        colno = core.bayesdb_variable_number(bdb, population_id, var)
+        colno = core.bayesdb_variable_number(bdb, population_id, None, var)
         if colno < 0:
             continue
         stattype, dist, params = _retrieve_stattype_dist_params(var)

src/metamodels/cgpm_metamodel.py

@@ -131,7 +131,8 @@
 
 CREATE TABLE bayesdb_variable (
 	population_id	INTEGER NOT NULL REFERENCES bayesdb_population(id),
-	generator_id	INTEGER REFERENCES bayesdb_generator(id),
+	generator_id	INTEGER REFERENCES bayesdb_generator(id)
+				ON DELETE CASCADE,
 	colno		INTEGER NOT NULL,
 	name		TEXT COLLATE NOCASE NOT NULL,
 	stattype	TEXT COLLATE NOCASE NOT NULL

src/schema.py

@@ -454,7 +454,8 @@ def test_estimate_columns_trivial():
     prefix1 = ' FROM bayesdb_population AS p,' \
         ' bayesdb_variable AS v, bayesdb_column AS c' \
         ' WHERE p.id = 1 AND v.population_id = p.id' \
-        ' AND c.tabname = p.tabname AND c.colno = v.colno'
+        ' AND c.tabname = p.tabname AND c.colno = v.colno' \
+        ' AND v.generator_id IS NULL'
     prefix = prefix0 + prefix1
     assert bql2sql('estimate * from columns of p1;') == \
         prefix + ';'
@@ -557,6 +558,8 @@ def test_estimate_pairwise_trivial():
     infix0 += ' AND v0.population_id = p.id AND v1.population_id = p.id'
     infix0 += ' AND c0.tabname = p.tabname AND c0.colno = v0.colno'
     infix0 += ' AND c1.tabname = p.tabname AND c1.colno = v1.colno'
+    infix0 += ' AND v0.generator_id IS NULL'
+    infix0 += ' AND v1.generator_id IS NULL'
     infix += infix0
     assert bql2sql('estimate dependence probability'
             ' from pairwise columns of p1;') == \
@@ -695,6 +698,7 @@ def test_estimate_pairwise_selected_columns():
         ' AND v0.population_id = p.id AND v1.population_id = p.id' \
         ' AND c0.tabname = p.tabname AND c0.colno = v0.colno' \
         ' AND c1.tabname = p.tabname AND c1.colno = v1.colno' \
+        ' AND v0.generator_id IS NULL AND v1.generator_id IS NULL' \
         ' AND c0.colno IN (1, 2) AND c1.colno IN (1, 2);'
     assert bql2sql('estimate dependence probability'
             ' from pairwise columns of p1'
@@ -710,6 +714,7 @@ def test_estimate_pairwise_selected_columns():
         ' AND v0.population_id = p.id AND v1.population_id = p.id' \
         ' AND c0.tabname = p.tabname AND c0.colno = v0.colno' \
         ' AND c1.tabname = p.tabname AND c1.colno = v1.colno' \
+        ' AND v0.generator_id IS NULL AND v1.generator_id IS NULL' \
         ' AND c0.colno IN (3, 1) AND c1.colno IN (3, 1);'
 
 def test_select_columns_subquery():
@@ -980,9 +985,11 @@ def trace(string, _bindings):
                     ' bayesdb_column AS c'
                 ' WHERE p.id = 1 AND v.population_id = p.id'
                     ' AND c.tabname = p.tabname AND c.colno = v.colno'
+                    ' AND v.generator_id IS NULL'
                 ' LIMIT 1',
             'SELECT colno FROM bayesdb_variable'
-                ' WHERE population_id = ? AND name = ?',
+                ' WHERE population_id = ? AND generator_id IS NULL'
+                    ' AND name = ?',
             # ESTIMATE SIMILARITY TO (rowid=1):
             'SELECT tabname FROM bayesdb_population WHERE id = ?',
             'SELECT bql_row_similarity(1, NULL, _rowid_,'
@@ -1022,9 +1029,11 @@ def trace(string, _bindings):
                     ' bayesdb_column AS c'
                 ' WHERE p.id = 1 AND v.population_id = p.id'
                     ' AND c.tabname = p.tabname AND c.colno = v.colno'
+                    ' AND v.generator_id IS NULL'
                 ' LIMIT ?1',
             'SELECT colno FROM bayesdb_variable'
-                ' WHERE population_id = ? AND name = ?',
+                ' WHERE population_id = ? AND generator_id IS NULL'
+                    ' AND name = ?',
             'SELECT tabname FROM bayesdb_population WHERE id = ?',
             # ESTIMATE SIMILARITY TO (rowid=1):
             'SELECT bql_row_similarity(1, NULL, _rowid_,'
@@ -1054,13 +1063,17 @@ def trace(string, _bindings):
             'PRAGMA table_info("t")',
             "SELECT CAST(4 AS INTEGER), 'F'",
             'SELECT colno FROM bayesdb_variable'
-                ' WHERE population_id = ? AND name = ?',
+                ' WHERE population_id = ? AND generator_id IS NULL'
+                    ' AND name = ?',
             'SELECT colno FROM bayesdb_variable'
-                ' WHERE population_id = ? AND name = ?',
+                ' WHERE population_id = ? AND generator_id IS NULL'
+                    ' AND name = ?',
             'SELECT colno FROM bayesdb_variable'
-                ' WHERE population_id = ? AND name = ?',
+                ' WHERE population_id = ? AND generator_id IS NULL'
+                    ' AND name = ?',
             'SELECT colno FROM bayesdb_variable'
-                ' WHERE population_id = ? AND name = ?',
+                ' WHERE population_id = ? AND generator_id IS NULL'
+                    ' AND name = ?',
             'CREATE TEMP TABLE IF NOT EXISTS "sim"'
                 ' ("age" NUMERIC,"RANK" NUMERIC,"division" NUMERIC)',
             'SELECT tabname FROM bayesdb_population WHERE id = ?',
@@ -1194,13 +1207,13 @@ def trace(string, _bindings):
             'SELECT id FROM bayesdb_population WHERE name = ?',
             'SELECT tabname FROM bayesdb_population WHERE id = ?',
             'PRAGMA table_info("t")',
-            'SELECT name, stattype FROM bayesdb_variable'
-                ' WHERE population_id = ? AND colno < 0',
             "SELECT CAST(4 AS INTEGER), 'F'",
             'SELECT colno FROM bayesdb_variable'
-                ' WHERE population_id = ? AND name = ?',
+                ' WHERE population_id = ? AND generator_id IS NULL'
+                    ' AND name = ?',
             'SELECT colno FROM bayesdb_variable'
-                ' WHERE population_id = ? AND name = ?',
+                ' WHERE population_id = ? AND generator_id IS NULL'
+                    ' AND name = ?',
             'SELECT tabname FROM bayesdb_population WHERE id = ?',
             'SELECT MAX(_rowid_) FROM "t"',
             'SELECT id FROM bayesdb_generator WHERE population_id = ?',
@@ -1440,7 +1453,7 @@ def test_createtab():
             overrides=[('age', 'ignore')])
         bdb.execute('drop population p0')
         population_id = core.bayesdb_get_population(bdb, 'p')
-        colno = core.bayesdb_variable_number(bdb, population_id, 'age')
+        colno = core.bayesdb_variable_number(bdb, population_id, None, 'age')
         assert core.bayesdb_variable_stattype(bdb, population_id, colno) == \
             'numerical'
         bdb.execute('initialize 1 model for p_cc')

tests/test_bql.py

@@ -568,9 +568,9 @@ def impute_and_confidence(self, seed, M_c, X_L, X_D, Y, Q, n):
             )
         ''')
         pid = core.bayesdb_get_population(bdb, 'p1')
-        assert core.bayesdb_variable_number(bdb, pid, 'label') == 1
-        assert core.bayesdb_variable_number(bdb, pid, 'age') == 2
-        assert core.bayesdb_variable_number(bdb, pid, 'weight') == 3
+        assert core.bayesdb_variable_number(bdb, pid, None, 'label') == 1
+        assert core.bayesdb_variable_number(bdb, pid, None, 'age') == 2
+        assert core.bayesdb_variable_number(bdb, pid, None, 'weight') == 3
         gid = core.bayesdb_get_generator(bdb, 'p1_cc')
         from bayeslite.metamodels.crosscat import crosscat_cc_colno
         assert crosscat_cc_colno(bdb, gid, 1) == 0