Merge branch '20160624-riastradh-cgpm' of github.com:probcomp/bayeslite into 20160628-fsaad-cgpm

Author: F Saad

MANIFEST.in

@@ -110,11 +110,9 @@ include shell/scripts/bayeslite
 include shell/tests/test_pretty.py
 include shell/tests/test_shell.py
 include shell/tests/thooks.py
-include src/grammar.sha256
 include src/grammar.y
 include tests/dha.csv
 include tests/dha_codebook.csv
-include tests/dha_models.pkl.gz
 include tests/test_bql.py
 include tests/test_codebook.py
 include tests/test_column_dep.py
@@ -123,7 +121,6 @@ include tests/test_correlation.py
 include tests/test_csv.py
 include tests/test_geweke.py
 include tests/test_guess.py
-include tests/test_legacy.py
 include tests/test_math_util.py
 include tests/test_metamodels.py
 include tests/test_parse.py

shell/src/core.py

@@ -69,7 +69,6 @@ def __init__(self, bdb, metamodel, stdin=None, stdout=None, stderr=None):
         self._installcmd('guess', self.dot_guess)
         self._installcmd('help', self.dot_help)
         self._installcmd('hook', self.dot_hook)
-        self._installcmd('legacymodels', self.dot_legacymodels)
         self._installcmd('open', self.dot_open)
         self._installcmd('pythexec', self.dot_pythexec)
         self._installcmd('python', self.dot_python)
@@ -480,49 +479,22 @@ def dot_codebook(self, line):
             self.stdout.write(traceback.format_exc())
 
     def dot_guess(self, line):
-        '''guess data generator
-        <generator> <table>
+        '''guess population schema
+        <population> <table>
 
-        Create a generator named <generator> for the table <table>,
-        guessing the statistical types of the columns in <table>.
+        Create a population named <population> with variables
+        corresponding to columns in table <table>, heuristically
+        guessing their statistical types.
         '''
         # XXX Lousy, lousy tokenizer.
         tokens = line.split()
         if len(tokens) != 2:
-            self.stdout.write('Usage: .guess <generator> <table>\n')
+            self.stdout.write('Usage: .guess <population> <table>\n')
             return
-        generator = tokens[0]
+        population = tokens[0]
         table = tokens[1]
         try:
-            guess.bayesdb_guess_generator(self._bdb, generator, table,
-                                          self._metamodel)
-        except Exception:
-            self.stdout.write(traceback.format_exc())
-
-    def dot_legacymodels(self, line):
-        '''load legacy models
-        <generator> <table> </path/to/models.pkl.gz>
-
-        Create a Crosscat generator named <generator> for the table
-        <table> from the legacy models stored in
-        </path/to/models.pkl.gz>.
-        '''
-        # XXX Lousy, lousy tokenizer.
-        tokens = line.split()
-        if len(tokens) != 3:
-            self.stdout.write('Usage:'
-                              ' .legacymodels <generator> <table>'
-                              ' </path/to/models.pkl.gz>\n')
-            return
-        generator = tokens[0]
-        table = tokens[1]
-        pathname = tokens[2]
-        try:
-            bayeslite.bayesdb_load_legacy_models(self._bdb, generator, table,
-                                                 self._metamodel, pathname,
-                                                 create=True)
-        except IOError as e:
-            self.stdout.write('%s\n' % (e,))
+            guess.bayesdb_guess_population(self._bdb, population, table)
         except Exception:
             self.stdout.write(traceback.format_exc())
 
@@ -537,8 +509,9 @@ def dot_describe(self, line):
         tokens = line.split()
         if len(tokens) == 0:
             self.stdout.write('Usage: .describe table(s) [<table>...]\n')
+            self.stdout.write('       .describe population(s) [<pop>...]\n')
+            self.stdout.write('       .describe variables <pop>\n')
             self.stdout.write('       .describe generator(s) [<gen>...]\n')
-            self.stdout.write('       .describe columns <gen>\n')
             self.stdout.write('       .describe model(s) <gen> [<model>...]\n')
             return
         if casefold(tokens[0]) == 'table' or \
@@ -570,6 +543,31 @@ def dot_describe(self, line):
             ''' % (qualifier,)
             with self._bdb.savepoint():
                 pretty.pp_cursor(self.stdout, self._bdb.execute(sql, params))
+        elif casefold(tokens[0]) in ('population', 'populations'):
+            params = None
+            qualifier = None
+            if len(tokens) == 1:
+                params = ()
+                qualifier = '1'
+            else:
+                params = tokens[1:]
+                names = ','.join('?%d' % (i + 1,) for i in xrange(len(params)))
+                qualifier = '(name IN (%s))' % (names,)
+                ok = True
+                for population in params:
+                    if not core.bayesdb_has_population(self._bdb, population):
+                        self.stdout.write('No such population: %s\n' %
+                            (repr(population),))
+                        ok = False
+                if not ok:
+                    return
+            with self._bdb.savepoint():
+                cursor = self._bdb.sql_execute('''
+                    SELECT id, name, tabname
+                        FROM bayesdb_population
+                        WHERE %s
+                ''' % (qualifier,), params)
+                pretty.pp_cursor(self.stdout, cursor)
         elif casefold(tokens[0]) == 'generator' or \
                 casefold(tokens[0]) == 'generators':
             params = None
@@ -599,29 +597,30 @@ def dot_describe(self, line):
             with self._bdb.savepoint():
                 pretty.pp_cursor(self.stdout,
                     self._bdb.sql_execute(sql, params))
-        elif casefold(tokens[0]) == 'columns':
+        elif casefold(tokens[0]) == 'variables':
             if len(tokens) != 2:
-                self.stdout.write('Describe columns of what generator?\n')
+                self.stdout.write('Usage: .describe variables <population>\n')
                 return
-            generator = tokens[1]
+            population = tokens[1]
             with self._bdb.savepoint():
-                if not core.bayesdb_has_generator(self._bdb, generator):
-                    self.stdout.write('No such generator: %s\n' %
-                        (repr(generator),))
+                if not core.bayesdb_has_population(self._bdb, population):
+                    self.stdout.write('No such population: %r\n' %
+                        (population,))
                     return
-                generator_id = core.bayesdb_get_generator(self._bdb, generator)
+                population_id = core.bayesdb_get_population(
+                    self._bdb, population)
                 sql = '''
                     SELECT c.colno AS colno, c.name AS name,
-                            gc.stattype AS stattype, c.shortname AS shortname
-                        FROM bayesdb_generator AS g,
+                            v.stattype AS stattype, c.shortname AS shortname
+                        FROM bayesdb_population AS p,
                             (bayesdb_column AS c LEFT OUTER JOIN
-                                bayesdb_generator_column AS gc
+                                bayesdb_variable AS v
                                 USING (colno))
-                        WHERE g.id = ? AND g.id = gc.generator_id
-                            AND g.tabname = c.tabname
+                        WHERE p.id = ? AND p.id = v.population_id
+                            AND p.tabname = c.tabname
                         ORDER BY colno ASC;
                 '''
-                cursor = self._bdb.sql_execute(sql, (generator_id,))
+                cursor = self._bdb.sql_execute(sql, (population_id,))
                 pretty.pp_cursor(self.stdout, cursor)
         elif casefold(tokens[0]) == 'model' or \
                 casefold(tokens[0]) == 'models':
@@ -665,5 +664,5 @@ def dot_describe(self, line):
         else:
             self.stdout.write('Usage: .describe table(s) [<table>...]\n')
             self.stdout.write('       .describe generator(s) [<gen>...]\n')
-            self.stdout.write('       .describe columns <gen>\n')
+            self.stdout.write('       .describe variables <pop>\n')
             self.stdout.write('       .describe model(s) <gen> [<model>...]\n')

shell/tests/test_shell.py

@@ -104,17 +104,26 @@ def spawntable():
 
 
 @pytest.fixture
-def spawntablegen(spawntable):
+def spawntablepop(spawntable):
     table, c = spawntable
-    c.sendexpectcmd('.guess dha_cc %s' % (table,))
+    c.sendexpectcmd('.guess hospitals %s' % (table,))
     c.expect_prompt()
-    return table, 'dha_cc', c
+    return table, 'hospitals', c
 
 
 @pytest.fixture
-def spawngen(spawntablegen):
-    table, gen, c = spawntablegen
-    return gen, c
+def spawntablepopgen(spawntablepop):
+    table, pop, c = spawntablepop
+    c.sendexpectcmd(
+        'create generator hospitals_cc for %s using crosscat;' % (pop,))
+    c.expect_prompt()
+    return table, pop, 'hospitals_cc', c
+
+
+@pytest.fixture
+def spawnpop(spawntablepop):
+    table, pop, c = spawntablepop
+    return pop, c
 
 
 # Tests begin
@@ -153,20 +162,19 @@ def test_help_returns_list_of_commands(spawnbdb):
     c = spawnbdb
     c.sendexpectcmd('.help')
     c.expect_lines([
-        '     .codebook    load codebook for table',
-        '          .csv    create table from CSV file',
-        '     .describe    describe BayesDB entities',
-        '        .guess    guess data generator',
-        '         .help    show help for commands',
-        '         .hook    add custom commands from a python source file',
-        ' .legacymodels    load legacy models',
-        '         .open    close existing database and open new one',
-        '     .pythexec    execute a Python statement',
-        '       .python    evaluate a Python expression',
-        '         .read    read a file of shell commands',
-        '          .sql    execute a SQL query',
-        '        .trace    trace queries',
-        '      .untrace    untrace queries',
+        ' .codebook    load codebook for table',
+        '      .csv    create table from CSV file',
+        ' .describe    describe BayesDB entities',
+        '    .guess    guess population schema',
+        '     .help    show help for commands',
+        '     .hook    add custom commands from a python source file',
+        '     .open    close existing database and open new one',
+        ' .pythexec    execute a Python statement',
+        '   .python    evaluate a Python expression',
+        '     .read    read a file of shell commands',
+        '      .sql    execute a SQL query',
+        '    .trace    trace queries',
+        '  .untrace    untrace queries',
         "Type `.help <cmd>' for help on the command <cmd>.",
     ])
     c.expect_prompt()
@@ -204,10 +212,10 @@ def test_dot_csv_dup(spawnbdb):
         c.expect_prompt()
 
 
-def test_describe_columns_without_generator(spawntable):
+def test_describe_columns_without_population(spawntable):
     table, c = spawntable
-    c.sendexpectcmd('.describe columns %s' % (table,))
-    c.expect_lines(['No such generator: %s' % (repr(table),)])
+    c.sendexpectcmd('.describe variables %s' % (table,))
+    c.expect_lines(['No such population: %r' % (table,)])
     c.expect_prompt()
 
 
@@ -229,7 +237,7 @@ def test_bql_select(spawntable):
 
 def test_guess(spawntable):
     table, c = spawntable
-    c.sendexpectcmd('.guess dha_cc %s' % (table,))
+    c.sendexpectcmd('.guess hospitals %s' % (table,))
     c.expect_prompt()
 
 
@@ -253,23 +261,23 @@ def test_sql(spawntable):
     c.expect_prompt()
 
 
-def test_describe_generator(spawntablegen):
-    table, gen, c = spawntablegen
-    generator_output = [
-        'id |   name | tabname | metamodel',
-        '---+--------+---------+----------',
-        ' 1 | dha_cc |     dha |  crosscat',
+def test_describe_population(spawntablepop):
+    table, pop, c = spawntablepop
+    population_output = [
+        'id |      name | tabname',
+        '---+-----------+--------',
+        ' 1 | hospitals |     dha',
     ]
-    c.sendexpectcmd('.describe generator %s' % (table,))
-    c.expect_lines(['No such generator: %s' % (repr(table),),])
+    c.sendexpectcmd('.describe population %s' % (table,))
+    c.expect_lines(['No such population: %r' % (table,),])
     c.expect_prompt()
-    c.sendexpectcmd('.describe generator %s' % (gen,))
-    c.expect_lines(generator_output)
+    c.sendexpectcmd('.describe population %s' % (pop,))
+    c.expect_lines(population_output)
     c.expect_prompt()
 
 
-def test_describe_models(spawntablegen):
-    table, gen, c = spawntablegen
+def test_describe_models(spawntablepopgen):
+    table, pop, gen, c = spawntablepopgen
     # XXX apsw bug: There is no way to discover the description of a
     # cursor that yields no rows.  The best we can do without failing
     # is to assume that such a cursor has no columns either.  This is
@@ -291,9 +299,9 @@ def test_describe_models(spawntablegen):
     c.expect_prompt()
 
 
-def test_describe_column_with_generator(spawntablegen):
-    table, gen, c = spawntablegen
-    columns_output = [
+def test_describe_column_with_generator(spawntablepopgen):
+    table, pop, gen, c = spawntablepopgen
+    variables_output = [
         'colno |                name |  stattype | shortname',
         '------+---------------------+-----------+----------',
         '    1 |         N_DEATH_ILL | numerical |      None',
@@ -360,11 +368,11 @@ def test_describe_column_with_generator(spawntablegen):
         '   62 |           CHF_SCORE | numerical |      None',
         '   63 |         PNEUM_SCORE | numerical |      None',
     ]
-    c.sendexpectcmd('.describe columns %s' % (table,))
-    c.expect_lines(['No such generator: %s' % (repr(table),),])
+    c.sendexpectcmd('.describe variables %s' % (table,))
+    c.expect_lines(['No such population: %s' % (repr(table),),])
     c.expect_prompt()
-    c.sendexpectcmd('.describe columns %s' % (gen,))
-    c.expect_lines(columns_output)
+    c.sendexpectcmd('.describe variables %s' % (pop,))
+    c.expect_lines(variables_output)
     c.expect_prompt()
 
 
@@ -378,21 +386,20 @@ def test_hook(spawnbdb):
     c.expect_prompt()
     c.sendexpectcmd('.help')
     c.expect_lines([
-        '     .codebook    load codebook for table',
-        '          .csv    create table from CSV file',
-        '     .describe    describe BayesDB entities',
-        '        .guess    guess data generator',
-        '         .help    show help for commands',
-        '         .hook    add custom commands from a python source file',
-        ' .legacymodels    load legacy models',
-        '       .myhook    myhook help string',
-        '         .open    close existing database and open new one',
-        '     .pythexec    execute a Python statement',
-        '       .python    evaluate a Python expression',
-        '         .read    read a file of shell commands',
-        '          .sql    execute a SQL query',
-        '        .trace    trace queries',
-        '      .untrace    untrace queries',
+        ' .codebook    load codebook for table',
+        '      .csv    create table from CSV file',
+        ' .describe    describe BayesDB entities',
+        '    .guess    guess population schema',
+        '     .help    show help for commands',
+        '     .hook    add custom commands from a python source file',
+        '   .myhook    myhook help string',
+        '     .open    close existing database and open new one',
+        ' .pythexec    execute a Python statement',
+        '   .python    evaluate a Python expression',
+        '     .read    read a file of shell commands',
+        '      .sql    execute a SQL query',
+        '    .trace    trace queries',
+        '  .untrace    untrace queries',
         "Type `.help <cmd>' for help on the command <cmd>."
     ])
     c.expect_prompt()

src/__init__.py

@@ -72,7 +72,6 @@
 from bayeslite.codebook import bayesdb_load_codebook_csv_file
 from bayeslite.exception import BayesDBException
 from bayeslite.exception import BQLError
-from bayeslite.legacy_models import bayesdb_load_legacy_models
 from bayeslite.metamodel import IBayesDBMetamodel
 from bayeslite.metamodel import bayesdb_builtin_metamodel
 from bayeslite.metamodel import bayesdb_deregister_metamodel
@@ -102,7 +101,6 @@ def bql_quote_name(name):
     'BayesDBTxnError',
     'bayesdb_deregister_metamodel',
     'bayesdb_load_codebook_csv_file',
-    'bayesdb_load_legacy_models',
     'bayesdb_open',
     'bayesdb_read_csv',
     'bayesdb_read_csv_file',

src/ast.py

@@ -117,6 +117,7 @@
     'ckpt_iterations',
     'ckpt_seconds',
     'wait',
+    'program',
 ])
 DropModels = namedtuple('DropModels', [
     'generator',