Merge pull request #32 from codinuum/develop

Develop

Author: codinuum

Dockerfile

@@ -16,7 +16,7 @@ RUN set -x && \
     cd /root && \
     apt-get update && \
     env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-            sudo \
+            gosu \
             vim \
             opam \
             net-tools \
@@ -36,7 +36,7 @@ RUN set -x && \
             git rsync && \
     wget https://bootstrap.pypa.io/get-pip.py && \
     python3 get-pip.py && \
-    pip3 install pyodbc setuptools build[virtualenv] javalang && \
+    pip3 install pyodbc setuptools build[virtualenv] javalang python-rapidjson && \
     rm get-pip.py
 
 RUN set -x && \
@@ -67,10 +67,16 @@ RUN set -x && \
     cd /root && \
     opam init -y --disable-sandboxing && \
     eval $(opam env) && \
-    opam install -y camlp-streams camlzip cryptokit csv git-unix git-cohttp-unix menhir ocamlnet pxp ulex uuidm pcre volt && \
+    opam switch create 4.14.0 && \
+    eval $(opam env) && \
+    opam install -y camlp-streams camlzip cryptokit csv git-unix menhir ocamlnet pxp ulex uuidm pcre cohttp volt && \
+    eval $(opam env) && \
     cd src && \
-    make && \
+    make -C mldiff && \
+    make -C util && \
+    make -C otreediff && \
     cd ast/analyzing && \
+    make production && \
     cp -r bin etc /opt/cca/ && \
     cp modules/Mverilog*.cmxs /opt/cca/modules/ && \
     cp modules/Mpython*.cmxs /opt/cca/modules/ && \

Dockerfile.20.04

@@ -16,7 +16,7 @@ RUN set -x && \
     cd /root && \
     apt-get update && \
     env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-            sudo \
+            gosu \
             vim \
             opam \
             net-tools \
@@ -36,7 +36,7 @@ RUN set -x && \
             git rsync && \
     wget https://bootstrap.pypa.io/get-pip.py && \
     python3 get-pip.py && \
-    pip3 install pyodbc setuptools build[virtualenv] javalang && \
+    pip3 install pyodbc setuptools build[virtualenv] javalang python-rapidjson && \
     rm get-pip.py
 
 RUN set -x && \
@@ -67,10 +67,16 @@ RUN set -x && \
     cd /root && \
     opam init -y --disable-sandboxing && \
     eval $(opam env) && \
-    opam install -y camlp-streams camlzip cryptokit csv git-unix git-cohttp-unix menhir ocamlnet pxp ulex uuidm pcre volt && \
+    opam switch create 4.14.0 && \
+    eval $(opam env) && \
+    opam install -y camlp-streams camlzip cryptokit csv git-unix menhir ocamlnet pxp ulex uuidm pcre cohttp volt && \
+    eval $(opam env) && \
     cd src && \
-    make && \
+    make -C mldiff && \
+    make -C util && \
+    make -C otreediff && \
     cd ast/analyzing && \
+    make production && \
     cp -r bin etc /opt/cca/ && \
     cp modules/Mverilog*.cmxs /opt/cca/modules/ && \
     cp modules/Mpython*.cmxs /opt/cca/modules/ && \

python/src/cca/ccautil/cca_config.py

@@ -229,10 +229,15 @@ def finalize(self, proj_dir=PROJECTS_DIR):
             self.proj_path = get_proj_dir(pid, proj_dir=proj_dir)
 
         # for phylogeny
+        if self.abbrev_tbl == {}:
+            self.abbrev_tbl = dict(self.abbrev_list)
+
+        elif self.abbrev_list == []:
+            self.abbrev_list = list(self.abbrev_tbl.items())
+
         if self.vers == []:
             self.vers = [n for (n, a) in self.abbrev_list]
 
-        self.abbrev_tbl = dict(self.abbrev_list)
         for (x, a) in self.abbrev_tbl.items():
             self.inv_abbrev_tbl[a.rstrip()] = x
 

python/src/cca/ccautil/common.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-import os.path
+import os
 import logging
 
 LOGGING_FORMAT = '[%(asctime)s][%(levelname)s][%(module)s][%(funcName)s] %(message)s'

python/src/cca/ccautil/diffts.py

@@ -20,6 +20,9 @@
 
 import os
 import re
+import gzip
+# import json
+import rapidjson as json
 import time
 import logging
 
@@ -40,6 +43,10 @@
 cfgfact_file_name = 'cfg.nt.gz'
 
 
+GMAP_JSON_FILE_NAME = 'gmap.json.gz'
+GDIFF_JSON_FILE_NAME = 'gdiff.json.gz'
+
+
 RETRY_COUNT = 3
 
 DEFAULT_FACT_SIZE_THRESH = 10000
@@ -217,7 +224,7 @@ def get_cache_dir1_(diff_cmd, a,
 
     opts = cache_opt + hash_opt
 
-    cmd = f'{diff_cmd} -parseonly{opts} -getcache {a}'
+    cmd = f'{diff_cmd} -parseonly{opts} -getcache "{a}"'
 
     if not quiet:
         logger.info(f'cmd: "{cmd}"')
@@ -249,7 +256,7 @@ def get_cache_dir(diff_cmd, a1, a2,
 
     opts = cache_opt + hash_opt
 
-    cmd = f'{diff_cmd}{opts} -getcache {a1} {a2}'
+    cmd = f'{diff_cmd}{opts} -getcache "{a1}" "{a2}"'
 
     if not quiet:
         logger.info(f'cmd: "{cmd}"')
@@ -280,6 +287,20 @@ def get_fact_proj_roots_opt(fact_proj_roots):
     return ' '.join(li)
 
 
+def read_json(data_path):
+    d = None
+    _open = open
+    if data_path.endswith('.gz'):
+        _open = gzip.open
+    try:
+        with _open(data_path, 'r') as f:
+            d = json.load(f)
+    except Exception as e:
+        logger.warning(f'{data_path}: {e}')
+
+    return d
+
+
 def read_file(r, name_pat_list, stat_paths, retry_count=RETRY_COUNT):
     count = 0
     stat_paths = stat_paths * (int(retry_count / len(stat_paths)) + 1)
@@ -444,9 +465,7 @@ def diffts(diff_cmd, file1, file2,
         if load_fact:
             logger.info('loading fact')
             if fact_versions:
-                fact_opt = \
-                    ' -fact -fact:add-versions {}'\
-                    .format(get_fact_versions_opt(fact_versions))
+                fact_opt = f' -fact -fact:add-versions {get_fact_versions_opt(fact_versions)}'
 
                 fact_opt += ' -fact:encoding:' + fact_encoding
                 fact_opt += ' -fact:hash:' + fact_hash_algo
@@ -467,16 +486,13 @@ def diffts(diff_cmd, file1, file2,
                     fact_opt += f' -fact:project {fact_proj}'
 
                 if fact_proj_roots:
-                    fact_opt += \
-                        ' {}'.format(get_fact_proj_roots_opt(fact_proj_roots))
+                    fact_opt += f' {get_fact_proj_roots_opt(fact_proj_roots)}'
 
                 if fact_into_virtuoso:
-                    fact_opt += \
-                        f' -fact:into-virtuoso {fact_into_virtuoso}'
+                    fact_opt += f' -fact:into-virtuoso {fact_into_virtuoso}'
 
                 if fact_into_directory:
-                    fact_opt += \
-                        f' -fact:into-directory {fact_into_directory}'
+                    fact_opt += f' -fact:into-directory {fact_into_directory}'
 
                 if fact_for_delta:
                     fact_opt += ' -fact:delta'
@@ -523,7 +539,7 @@ def diffts(diff_cmd, file1, file2,
         cmd = ''.join((diff_cmd,
                        cache_opt, cachedir_opt, prep_opt, prem_opt, fact_opt,
                        dumpccs_opt, check_opt, other_opts))
-        cmd += f' {file1} {file2}'
+        cmd += f' "{file1}" "{file2}"'
 
         logger.info(f'cmd="{cmd}"')
 
@@ -535,6 +551,10 @@ def diffts(diff_cmd, file1, file2,
         'ninserts': 0,
         'ndeletes': 0,
         'nrelabels': 0,
+        'nmoves': 0,
+        'nmovrels': 0,
+        'nnodes1': 0,
+        'nnodes2': 0,
         # 'exitcode': 0,
         }
 
@@ -553,23 +573,47 @@ def diffast(file1, file2, **options):
 
 
 def diffast_get_cache_dir1(file, **options):
-    return get_cache_dir1_(diffast_cmd, file, **options)
+    opts = options.copy()
+    if 'usecache' in opts:
+        del opts['usecache']
+    if 'quiet' not in opts:
+        opts['quiet'] = True
+    return get_cache_dir1_(diffast_cmd, file, **opts)
 
 
 def diffast_get_cache_dir(file1, file2, **options):
-    return get_cache_dir(diffast_cmd, file1, file2, **options)
+    opts = options.copy()
+    if 'usecache' in opts:
+        del opts['usecache']
+    if 'quiet' not in opts:
+        opts['quiet'] = True
+    return get_cache_dir(diffast_cmd, file1, file2, **opts)
+
+
+def diffast_get_gmap(file1, file2, **options):
+    cache_dir = diffast_get_cache_dir(file1, file2, **options)
+    gmap_json = os.path.join(cache_dir, GMAP_JSON_FILE_NAME)
+    d = read_json(gmap_json)
+    return d
+
+
+def diffast_get_gdiff(file1, file2, **options):
+    cache_dir = diffast_get_cache_dir(file1, file2, **options)
+    gdiff_json = os.path.join(cache_dir, GDIFF_JSON_FILE_NAME)
+    d = read_json(gdiff_json)
+    return d
 
 
 def dump_unparsed(path, to_path, quiet=False):
-    cmd = f'{diffast_cmd} -clearcache -parseonly -dump:src:out {to_path} {path}'
+    cmd = f'{diffast_cmd} -clearcache -parseonly -dump:src:out "{to_path}" "{path}"'
     if not quiet:
         logger.info('cmd="{}"'.format(cmd))
 
     return proc.system(cmd, quiet=quiet)
 
 
 def patchast(path, delta_path, out_path, quiet=False):
-    cmd = f'{patchast_cmd} -o {out_path} {path} {delta_path}'
+    cmd = f'{patchast_cmd} -o "{out_path}" "{path}" "{delta_path}"'
     if not quiet:
         logger.info(f'cmd="{cmd}"')
 

python/src/cca/ccautil/find_change_patterns.py

@@ -31,8 +31,7 @@
 from xml.sax.saxutils import escape
 import logging
 
-from .siteconf import CCA_HOME
-from . import cca_config as config
+from .siteconf import CCA_HOME, PROJECTS_DIR_NAME
 from . import project, sparql
 from .factextractor import compo_join, P_TYPE, make_literal
 from .ns import FB_NS, CHG_NS, NS_TBL, PREFIX_TBL
@@ -518,7 +517,7 @@ def get_ver_dir(self, verURI):
     def get_ver_dir_r(self, verURI):
         v = self._conf.versions[self.get_vindex(verURI)]
         vdn = self._conf.get_ver_dir_name(v)
-        return os.path.join(config.PROJECTS_DIR_NAME, self._proj_id, vdn)
+        return os.path.join(PROJECTS_DIR_NAME, self._proj_id, vdn)
 
     def get_abbrev_ver(self, lver):
         av = lver