Merge remote-tracking branch 'origin/20160628-fsaad-cgpm' into 20160624-riastradh-cgpm

Author: riastradh-probcomp

src/metamodels/cgpm_metamodel.py

@@ -35,6 +35,7 @@
 from bayeslite.util import json_dumps
 
 import cgpm_schema.parse
+import cgpm_analyze.parse
 
 CGPM_SCHEMA_1 = '''
 INSERT INTO bayesdb_metamodel (name, version) VALUES ('cgpm', 1);
@@ -190,7 +191,7 @@ def initialize_models(self, bdb, generator_id, modelnos):
         for cgpm_ext in schema['cgpm_composition']:
             cgpms = [self._initialize_cgpm(bdb, generator_id, cgpm_ext)
                 for _ in xrange(n)]
-            engine.compose_cgpm(cgpms, N=1, multithread=self._ncpu)
+            engine.compose_cgpm(cgpms, multithread=self._ncpu)
 
         # Store the newly initialized engine.
         engine_json = json_dumps(engine.to_metadata())
@@ -221,16 +222,46 @@ def analyze_models(self, bdb, generator_id, modelnos=None, iterations=1,
 
         if ckpt_iterations is not None or ckpt_seconds is not None:
             # XXX
-            raise NotImplementedError('cgpm analysis checkpoint')
-        if program is not None:
-            # XXX
-            raise NotImplementedError('cgpm analysis programs')
+            raise NotImplementedError('CGpm analysis checkpoint not supported.')
 
-        # Get the engine.
-        engine = self._engine(bdb, generator_id)
+        if program is None:
+            program = []
+
+        population_id = core.bayesdb_generator_population(bdb, generator_id)
+
+        def retrieve_analyze_variables(ast):
+            # Transition all variables by default.
+            if len(ast) == 0:
+                variables = core.bayesdb_variable_names(bdb, population_id)
+            # Exactly 1 clause supported.
+            elif len(ast) == 1:
+                clause = ast[0]
+                # Transition user specified variables only.
+                if isinstance(clause, cgpm_analyze.parse.Variables):
+                    variables = clause.vars
+                # Transition all variables except user specified skip.
+                elif isinstance(clause, cgpm_analyze.parse.Skip):
+                    variables = filter(
+                        lambda v: v not in clause.vars,
+                        core.bayesdb_variable_names(bdb, population_id))
+                # Unknown/impossible clause.
+                else:
+                    raise ValueError('Unknown clause in ANALYZE: %s.' % ast)
+            # Crash if more than 1 clause.
+            else:
+                raise ValueError('1 clause permitted in ANALYZE: %s.' % ast)
+            return variables
 
-        # Do the transition.
-        engine.transition(N=iterations, S=max_seconds, multithread=self._ncpu)
+        # Retrieve target variables.
+        analyze_ast = cgpm_analyze.parse.parse(program)
+        variables = retrieve_analyze_variables(analyze_ast)
+        varnos = [core.bayesdb_variable_number(bdb, population_id, v)
+            for v in variables]
+
+        # Run transition.
+        engine = self._engine(bdb, generator_id)
+        engine.transition(
+            N=iterations, S=max_seconds, cols=varnos, multithread=self._ncpu)
 
         # Serialize the engine.
         engine_json = json_dumps(engine.to_metadata())
@@ -323,7 +354,7 @@ def simulate_joint(self, bdb, generator_id, targets, constraints, modelno,
         cgpm_query = [colno for _r, colno in targets]
         cgpm_evidence = {
             colno: self._to_numeric(bdb, generator_id, colno, value)
-            for colno, value in constraints
+            for _r, colno, value in constraints
         }
         engine = self._engine(bdb, generator_id)
         samples = engine.simulate(
@@ -586,6 +617,7 @@ def _create_schema(bdb, generator_id, schema_ast):
     # State.
     variables = []
     categoricals = {}
+    declared_latents = []
     cgpm_composition = []
     modelled = set()
     subsample = None
@@ -634,6 +666,18 @@ def _retrieve_stattype_dist_params(var):
             variables.append([var, stattype, dist, params])
             modelled.add(var)
 
+        elif isinstance(clause, cgpm_schema.parse.Latent):
+            # Reject if the latent variable has already been declared.
+            if any(l[0] == clause.name for l in declared_latents):
+                duplicate.add(clause.name)
+
+            # XXX FILL ME XXX
+
+            # Register the latent variable name and stattype into bayesdb.
+            # Do something related to the error checking data structures.
+
+            declared_latents.append((clause.name, clause.stattype))
+
         elif isinstance(clause, cgpm_schema.parse.Foreign):
             # Foreign model: some set of output variables is to be
             # modelled by foreign logic, possibly conditional on some
@@ -677,22 +721,21 @@ def _retrieve_stattype_dist_params(var):
                     _, dist, params = _retrieve_stattype_dist_params(var)
                     cctypes.append(dist)
                     ccargs.append(params)
-                else:
-                    # Finally, add a cgpm_composition record.
-                    cgpm_composition.append({
-                        'name': name,
-                        'inputs': inputs,
-                        'outputs': outputs,
-                        'kwds': kwds,
-                    })
+                # Finally, add a cgpm_composition record.
+                cgpm_composition.append({
+                    'name': name,
+                    'inputs': inputs,
+                    'outputs': outputs,
+                    'kwds': kwds,
+                })
 
         elif isinstance(clause, cgpm_schema.parse.Subsample):
             if subsample is not None:
                 raise BQLError(bdb, 'Duplicate subsample: %r' % (clause.n,))
             subsample = clause.n
 
         else:
-            assert False
+            raise BQLError(bdb, 'Unknown clause: %r' % (clause,))
 
     # Raise an exception if there were duplicates or unknown
     # variables.

src/metamodels/cgpm_schema/grammar.y

@@ -21,34 +21,38 @@
  * - L_ means a lexeme, which has useful associated text, e.g. an integer.
  */
 
-cgpm(empty)	::= .
-cgpm(schema)	::= schema(s).
+cgpm(empty)         ::= .
+cgpm(schema)        ::= schema(s).
 
-schema(one)	::= clause(c).
-schema(some)	::= schema(s) T_COMMA clause_opt(c).
+schema(one)         ::= clause(c).
+schema(some)        ::= schema(s) T_COMMA clause_opt(c).
 
-clause_opt(none)::= .
-clause_opt(some)::= clause(c).
+clause_opt(none)    ::= .
+clause_opt(some)    ::= clause(c).
 
-clause(basic)	::= var(var) dist(dist) param_opt(params).
-clause(foreign)	::= K_MODEL vars(outputs) given_opt(inputs)
-			K_USING foreign(name) param_opt(params).
-clause(subsamp)	::= K_SUBSAMPLE L_NUMBER(n).
+clause(basic)       ::= var(var) dist(dist) param_opt(params).
+clause(foreign)     ::= K_MODEL vars(outputs) given_opt(inputs)
+            K_USING foreign(name) param_opt(params).
+clause(subsamp)     ::= K_SUBSAMPLE L_NUMBER(n).
+clause(latent)      ::= K_LATENT var(var) stattype(st).
 
-dist(name)	::= L_NAME(dist).
-foreign(name)	::= L_NAME(foreign).
+dist(name)          ::= L_NAME(dist).
+foreign(name)       ::= L_NAME(foreign).
 
-given_opt(none)	::= .
-given_opt(some)	::= K_GIVEN vars(vars).
+given_opt(none)     ::= .
+given_opt(some)     ::= K_GIVEN vars(vars).
 
-vars(one)	::= var(var).
-vars(many)	::= vars(vars) T_COMMA var(var).
-var(name)	::= L_NAME(var).
+vars(one)           ::= var(var).
+vars(many)          ::= vars(vars) T_COMMA var(var).
 
-param_opt(none)	::= .
-param_opt(some)	::= T_LROUND params(ps) T_RROUND.
-params(one)	::= param(param).
-params(many)	::= params(params) T_COMMA param(param).
+var(name)           ::= L_NAME(var).
 
-param(num)	::= L_NAME(p) T_EQ L_NUMBER(num).
-param(nam)	::= L_NAME(p) T_EQ L_NAME(nam).
+stattype(s)         ::= L_NAME(st).
+
+param_opt(none)     ::= .
+param_opt(some)     ::= T_LROUND params(ps) T_RROUND.
+params(one)         ::= param(param).
+params(many)        ::= params(params) T_COMMA param(param).
+
+param(num)          ::= L_NAME(p) T_EQ L_NUMBER(num).
+param(nam)          ::= L_NAME(p) T_EQ L_NAME(nam).

src/metamodels/cgpm_schema/parse.py

@@ -31,6 +31,7 @@
 
 KEYWORDS = {
     'given': grammar.K_GIVEN,
+    'latent': grammar.K_LATENT,
     'model': grammar.K_MODEL,
     'subsample': grammar.K_SUBSAMPLE,
     'using': grammar.K_USING,
@@ -132,6 +133,8 @@ def p_clause_foreign(self, outputs, inputs, name, params):
         return Foreign(outputs, inputs, name, params)
     def p_clause_subsamp(self, n):
         return Subsample(n)
+    def p_clause_latent(self, var, st):
+        return Latent(var, st)
 
     def p_dist_name(self, dist):                return casefold(dist)
     def p_foreign_name(self, foreign):          return casefold(foreign)
@@ -141,7 +144,9 @@ def p_given_opt_some(self, vars):           return vars
 
     def p_vars_one(self, var):                  return [var]
     def p_vars_many(self, vars, var):           vars.append(var); return vars
-    def p_var_name(self, var):                  return var
+    def p_var_name(self, var):                  return casefold(var)
+
+    def p_stattype_s(self, st):                 return st
 
     def p_param_opt_none(self):                 return []
     def p_param_opt_some(self, ps):             return ps
@@ -166,3 +171,8 @@ def p_param_nam(self, p, nam):              return (p, nam)
 Subsample = namedtuple('Subsample', [
     'n',
 ])
+
+Latent = namedtuple('Latent', [
+    'name',
+    'stattype',
+])

tests/test_vscgpm.py

@@ -16,6 +16,7 @@
 
 import math
 import numpy as np
+import pytest
 import random                   # XXX
 
 #from cgpm.regressions.forest import RandomForest
@@ -165,6 +166,8 @@ def test_cgpm_extravaganza__ci_slow():
         bdb.execute('''
             CREATE GENERATOR g0 FOR satellites USING cgpm (
                 apogee NORMAL,
+                LATENT kepler_cluster_id NUMERICAL,
+                LATENT kepler_noise NUMERICAL,
                 MODEL kepler_cluster_id, kepler_noise, period
                     GIVEN apogee, perigee
                     USING venturescript (source = "{}"),
@@ -177,6 +180,30 @@ def test_cgpm_extravaganza__ci_slow():
 
         # -- MODEL country_of_operator GIVEN class_of_orbit USING forest;
         bdb.execute('INITIALIZE 1 MODELS FOR g0')
+        bdb.execute('ANALYZE g0 FOR 1 iteration WAIT (;)')
+        bdb.execute('''
+            ANALYZE g0 FOR 1 iteration WAIT (VARIABLES kepler_cluster_id)
+        ''')
+        bdb.execute('''
+            ANALYZE g0 FOR 1 iteration WAIT (
+                SKIP kepler_cluster_id, kepler_noise, period;
+            )
+        ''')
+        with pytest.raises(Exception):
+            # Disallow both SKIP and VARIABLES clauses.
+            #
+            # XXX Catch a more specific exception.
+            bdb.execute('''
+                ANALYZE g0 FOR 1 ITERATION WAIT (
+                    SKIP kepler_cluster_id;
+                    VARIABLES apogee, perigee;
+                )
+            ''')
+        bdb.execute('''
+            ANALYZE g0 FOR 1 iteration WAIT (
+                SKIP kepler_cluster_id, kepler_noise, period;
+            )
+        ''')
         bdb.execute('ANALYZE g0 FOR 1 ITERATION WAIT')
 
         bdb.execute('''