Maintenance.

Author: mkskeller

CHANGELOG.md

@@ -1,5 +1,19 @@
 The changelog explains changes pulled through from the private development repository. Bug fixes and small enhancements are committed between releases and not documented here.
 
+## 0.3.7 (August 14, 2023)
+
+- Path Oblivious Heap (@tskovlund)
+- Adjust batch and bucket size to program
+- Direct communication available in more protocols
+- Option for seed in fake preprocessing (@strieflin)
+- Lower memory usage due to improved register allocation
+- New instructions to speed up CISC compilation
+- Protocol implementation example
+- Fixed security bug: missing MAC checks in multi-threaded programs
+- Fixed security bug: race condition in MAC check
+- Fixed security bug: missing shuffling check in PS mod 2^k and Brain
+- Fixed security bug: insufficient drowning in pairwise protocols
+
 ## 0.3.6 (May 9, 2023)
 
 - More extensive benchmarking outputs

Compiler/GC/types.py

@@ -781,8 +781,6 @@ def store_in_mem(self, address):
                     for i in range(n):
                         for j, x in enumerate(v[i].bit_decompose()):
                             x.store_in_mem(address + i + j * n)
-            def reveal(self):
-                return util.untuplify([x.reveal() for x in self.elements()])
             @classmethod
             def two_power(cls, nn, size=1):
                 return cls.from_vec(
@@ -919,8 +917,7 @@ def bit_decompose(self, n_bits=None, security=None, maybe_mixed=None):
         return self.v[:n_bits]
     bit_compose = from_vec
     def reveal(self):
-        assert len(self) == 1
-        return self.v[0].reveal()
+        return util.untuplify([x.reveal() for x in self.elements()])
     def long_one(self):
         return [x.long_one() for x in self.v]
     def __rsub__(self, other):
@@ -1279,7 +1276,8 @@ def pow2(self, k):
 
 class sbitintvec(sbitvec, _bitint, _number, _sbitintbase):
     """
-    Vector of signed integers for parallel binary computation::
+    Vector of signed integers for parallel binary computation.
+    The following example uses vectors of size two::
 
         sb32 = sbits.get_type(32)
         siv32 = sbitintvec.get_type(32)
@@ -1291,7 +1289,7 @@ class sbitintvec(sbitvec, _bitint, _number, _sbitintbase):
         print_ln('mul: %s, %s', c[0].reveal(), c[1].reveal())
         c = (a - b).elements()
         print_ln('sub: %s, %s', c[0].reveal(), c[1].reveal())
-        c = (a < b).bit_decompose()
+        c = (a < b).elements()
         print_ln('lt: %s, %s', c[0].reveal(), c[1].reveal())
 
     This should output::
@@ -1467,7 +1465,7 @@ class sbitfixvec(_fix, _vec):
         print_ln('mul: %s, %s', c[0].reveal(), c[1].reveal())
         c = (a - b).elements()
         print_ln('sub: %s, %s', c[0].reveal(), c[1].reveal())
-        c = (a < b).bit_decompose()
+        c = (a < b).elements()
         print_ln('lt: %s, %s', c[0].reveal(), c[1].reveal())
 
     This should output roughly::

Compiler/allocator.py

@@ -43,7 +43,7 @@ def pop(self, size):
             else:
                 done = False
                 for x in self.by_logsize[logsize + 1:]:
-                    for block_size, addresses in x.items():
+                    for block_size, addresses in sorted(x.items()):
                         if len(addresses) > 0:
                             done = True
                             break
@@ -60,16 +60,92 @@ def pop(self, size):
                 self.by_address[addr + size] = diff
             return addr
 
+class AllocRange:
+    def __init__(self, base=0):
+        self.base = base
+        self.top = base
+        self.limit = base
+        self.grow = True
+        self.pool = defaultdict(set)
+
+    def alloc(self, size):
+        if self.pool[size]:
+            return self.pool[size].pop()
+        elif self.grow or self.top + size <= self.limit:
+            res = self.top
+            self.top += size
+            self.limit = max(self.limit, self.top)
+            if res >= REG_MAX:
+                raise RegisterOverflowError()
+            return res
+
+    def free(self, base, size):
+        assert self.base <= base < self.top
+        self.pool[size].add(base)
+
+    def stop_growing(self):
+        self.grow = False
+
+    def consolidate(self):
+        regs = []
+        for size, pool in self.pool.items():
+            for base in pool:
+                regs.append((base, size))
+        for base, size in reversed(sorted(regs)):
+            if base + size == self.top:
+                self.top -= size
+                self.pool[size].remove(base)
+                regs.pop()
+            else:
+                if program.Program.prog.verbose:
+                    print('cannot free %d register blocks '
+                          'by a gap of %d at %d' %
+                          (len(regs), self.top - size - base, base))
+                break
+
+class AllocPool:
+    def __init__(self):
+        self.ranges = defaultdict(lambda: [AllocRange()])
+        self.by_base = {}
+
+    def alloc(self, reg_type, size):
+        for r in self.ranges[reg_type]:
+            res = r.alloc(size)
+            if res is not None:
+                self.by_base[reg_type, res] = r
+                return res
+
+    def free(self, reg):
+        r = self.by_base.pop((reg.reg_type, reg.i))
+        r.free(reg.i, reg.size)
+
+    def new_ranges(self, min_usage):
+        for t, n in min_usage.items():
+            r = self.ranges[t][-1]
+            assert (n >= r.limit)
+            if r.limit < n:
+                r.stop_growing()
+                self.ranges[t].append(AllocRange(n))
+
+    def consolidate(self):
+        for r in self.ranges.values():
+            for rr in r:
+                rr.consolidate()
+
+    def n_fragments(self):
+        return max(len(r) for r in self.ranges)
+
 class StraightlineAllocator:
     """Allocate variables in a straightline program using n registers.
     It is based on the precondition that every register is only defined once."""
     def __init__(self, n, program):
         self.alloc = dict_by_id()
-        self.usage = Compiler.program.RegType.create_dict(lambda: 0)
+        self.max_usage = defaultdict(lambda: 0)
         self.defined = dict_by_id()
         self.dealloc = set_by_id()
-        self.n = n
+        assert(n == REG_MAX)
         self.program = program
+        self.old_pool = None
 
     def alloc_reg(self, reg, free):
         base = reg.vectorbase
@@ -79,14 +155,7 @@ def alloc_reg(self, reg, free):
 
         reg_type = reg.reg_type
         size = base.size
-        if free[reg_type, size]:
-            res = free[reg_type, size].pop()
-        else:
-            if self.usage[reg_type] < self.n:
-                res = self.usage[reg_type]
-                self.usage[reg_type] += size
-            else:
-                raise RegisterOverflowError()
+        res = free.alloc(reg_type, size)
         self.alloc[base] = res
 
         base.i = self.alloc[base]
@@ -126,7 +195,7 @@ def dealloc_reg(self, reg, inst, free):
                 for x in itertools.chain(dup.duplicates, base.duplicates):
                     to_check.add(x)
 
-        free[reg.reg_type, base.size].append(self.alloc[base])
+        free.free(base)
         if inst.is_vec() and base.vector:
             self.defined[base] = inst
             for i in base.vector:
@@ -135,6 +204,7 @@ def dealloc_reg(self, reg, inst, free):
             self.defined[reg] = inst
 
     def process(self, program, alloc_pool):
+        self.update_usage(alloc_pool)
         for k,i in enumerate(reversed(program)):
             unused_regs = []
             for j in i.get_def():
@@ -161,12 +231,26 @@ def process(self, program, alloc_pool):
             if k % 1000000 == 0 and k > 0:
                 print("Allocated registers for %d instructions at" % k, time.asctime())
 
+        self.update_max_usage(alloc_pool)
+        alloc_pool.consolidate()
+
         # print "Successfully allocated registers"
         # print "modp usage: %d clear, %d secret" % \
         #     (self.usage[Compiler.program.RegType.ClearModp], self.usage[Compiler.program.RegType.SecretModp])
         # print "GF2N usage: %d clear, %d secret" % \
         #     (self.usage[Compiler.program.RegType.ClearGF2N], self.usage[Compiler.program.RegType.SecretGF2N])
-        return self.usage
+        return self.max_usage
+
+    def update_max_usage(self, alloc_pool):
+        for t, r in alloc_pool.ranges.items():
+            self.max_usage[t] = max(self.max_usage[t], r[-1].limit)
+
+    def update_usage(self, alloc_pool):
+        if self.old_pool:
+            self.update_max_usage(self.old_pool)
+        if id(self.old_pool) != id(alloc_pool):
+            alloc_pool.new_ranges(self.max_usage)
+            self.old_pool = alloc_pool
 
     def finalize(self, options):
         for reg in self.alloc:
@@ -178,6 +262,21 @@ def finalize(self, options):
                                                                 '\t\t'))
                     if options.stop:
                         sys.exit(1)
+        if self.program.verbose:
+            def p(sizes):
+                total = defaultdict(lambda: 0)
+                for (t, size) in sorted(sizes):
+                    n = sizes[t, size]
+                    total[t] += size * n
+                    print('%s:%d*%d' % (t, size, n), end=' ')
+                print()
+                print('Total:', dict(total))
+
+            sizes = defaultdict(lambda: 0)
+            for reg in self.alloc:
+                x = reg.reg_type, reg.size
+            print('Used registers: ', end='')
+            p(sizes)
 
 def determine_scope(block, options):
     last_def = defaultdict_by_id(lambda: -1)

Compiler/circuit.py

@@ -10,7 +10,7 @@
 """
 
 from Compiler.GC.types import *
-from Compiler.library import function_block
+from Compiler.library import function_block, get_tape
 from Compiler import util
 import itertools
 import struct
@@ -54,7 +54,7 @@ def __call__(self, *inputs):
         return self.run(*inputs)
 
     def run(self, *inputs):
-        n = inputs[0][0].n
+        n = inputs[0][0].n, get_tape()
         if n not in self.functions:
             self.functions[n] = function_block(lambda *args:
                                                self.compile(*args))

Compiler/compilerLib.py

@@ -270,9 +270,9 @@ def build_program(self, name=None):
         self.prog = Program(self.args, self.options, name=name)
         if self.execute:
             if self.options.execute in \
-               ("emulate", "ring", "rep-field"):
+               ("emulate", "ring", "rep-field", "rep4-ring"):
                 self.prog.use_trunc_pr = True
-            if self.options.execute in ("ring",):
+            if self.options.execute in ("ring", "ps-rep-ring", "sy-rep-ring"):
                 self.prog.use_split(3)
             if self.options.execute in ("semi2k",):
                 self.prog.use_split(2)
@@ -487,6 +487,7 @@ def local_execution(self, args=[]):
                     "Cannot produce %s. " % executable + \
                     "Note that compilation requires a few GB of RAM.")
         vm = "%s/Scripts/%s.sh" % (self.root, self.options.execute)
+        sys.stdout.flush()
         os.execl(vm, vm, self.prog.name, *args)
 
     def remote_execution(self, args=[]):

Compiler/decision_tree.py

@@ -633,7 +633,8 @@ def preprocess_pandas(data):
             res.append(data.iloc[:,i].to_numpy())
             types.append('c')
         elif pandas.api.types.is_object_dtype(t):
-            values = data.iloc[:,i].unique()
+            values = list(filter(lambda x: isinstance(x, str),
+                                 list(data.iloc[:,i].unique())))
             print('converting the following to unary:', values)
             if len(values) == 2:
                 res.append(data.iloc[:,i].to_numpy() == values[1])

Compiler/instructions.py

@@ -638,6 +638,44 @@ class prefixsums(base.Instruction):
     code = base.opcodes['PREFIXSUMS']
     arg_format = ['sw','s']
 
+class picks(base.VectorInstruction):
+    """ Extract part of vector.
+
+    :param: result (sint)
+    :param: input (sint)
+    :param: start offset (int)
+    :param: step
+
+    """
+    __slots__ = []
+    code = base.opcodes['PICKS']
+    arg_format = ['sw','s','int','int']
+
+    def __init__(self, *args):
+        super(picks, self).__init__(*args)
+        assert 0 <= args[2] < len(args[1])
+        assert 0 <= args[2] + args[3] * len(args[0]) <= len(args[1])
+
+class concats(base.VectorInstruction):
+    """ Concatenate vectors.
+
+    :param: result (sint)
+    :param: start offset (int)
+    :param: input (sint)
+    :param: (repeat from offset)...
+
+    """
+    __slots__ = []
+    code = base.opcodes['CONCATS']
+    arg_format = tools.chain(['sw'], tools.cycle(['int','s']))
+
+    def __init__(self, *args):
+        super(concats, self).__init__(*args)
+        assert len(args) % 2 == 1
+        assert len(args[0]) == sum(args[1::2])
+        for i in range(1, len(args), 2):
+            assert args[i] == len(args[i + 1])
+
 @base.gf2n
 @base.vectorize
 class mulc(base.MulBase):