Merge pull request #1879 from antmicro/fix-race-condition

Fix race condition

Author: acomodi

.github/workflows/scripts/db.sh

@@ -73,7 +73,7 @@ echo "----------------------------------------"
 		# Looking for the failing directories and packing them
 		# example of line from which the failing fuzzer directory gets extracted:
 		#   - Makefile:87: recipe for target '000-db-init/000-init-db/run.xc7a100tfgg676-1.ok' failed --> fuzzers/000-db-init
-		grep -Po "recipe for target '\K(.*)(?=\/run\..*\.ok')" $tmp | sed -e 's/^/fuzzers\//' | xargs tar -zcf fuzzers/fails.tgz
+		grep -Po "recipe for target '\K(.*)(?=\/run.*\.ok')" $tmp | sed -e 's/^/fuzzers\//' | xargs tar -zcf fuzzers/fails.tgz
 		echo "----------------------------------------"
 		echo "A failure occurred during Database build."
 		echo "----------------------------------------"

prjxray/bitfilter.py

@@ -8,6 +8,9 @@
 # https://opensource.org/licenses/ISC
 #
 # SPDX-License-Identifier: ISC
+
+from prjxray.util import OpenSafeFile
+
 class Bitfilter(object):
     def __init__(
             self, frames_to_include=None, frames_to_exclude=[],

prjxray/bitstream.py

@@ -10,7 +10,7 @@
 # SPDX-License-Identifier: ISC
 import json
 import os
-from prjxray import util
+from prjxray.util import block_type_s2i
 
 # Break frames into WORD_SIZE bit words.
 WORD_SIZE_BITS = 32
@@ -119,7 +119,7 @@ def addr_bits2word(block_type, top_bottom, cfg_row, cfg_col, minor_addr):
     """Convert a deconstructed address to a 32 bit word"""
     # https://www.xilinx.com/support/documentation/user_guides/ug470_7Series_Config.pdf
     ret = 0
-    ret |= util.block_type_s2i[block_type] << 23
+    ret |= block_type_s2i[block_type] << 23
     ret |= {"top": 0, "bottom": 1}[top_bottom] << 22
     ret |= cfg_row << 17
     ret |= cfg_col << 7

prjxray/lib.py

@@ -14,6 +14,8 @@
 import re
 from collections import namedtuple
 
+from prjxray.util import OpenSafeFile
+
 
 def read_root_csv(root_dir):
     """ Reads root.csv from raw db directory.
@@ -24,7 +26,7 @@ def read_root_csv(root_dir):
     tiles = {}
     nodes = []
 
-    with open(os.path.join(root_dir, 'root.csv')) as f:
+    with OpenSafeFile(os.path.join(root_dir, 'root.csv')) as f:
         for d in csv.DictReader(f):
             if d['filetype'] == 'tile':
                 if d['subtype'] not in tiles:
@@ -123,17 +125,17 @@ def load_from_root_csv(self, nodes):
         import pyjson5 as json5
         import progressbar
         for node in progressbar.progressbar(nodes):
-            with open(node) as f:
+            with OpenSafeFile(node) as f:
                 node_wires = json5.load(f)
                 assert node_wires['node'] not in self.nodes
                 self.nodes[node_wires['node']] = node_wires['wires']
 
     def load_from_file(self, fname):
-        with open(fname, 'rb') as f:
+        with OpenSafeFile(fname, 'rb') as f:
             self.nodes = pickle.load(f)
 
     def save_to_file(self, fname):
-        with open(fname, 'wb') as f:
+        with OpenSafeFile(fname, 'wb') as f:
             pickle.dump(self.nodes, f)
 
     def site_pin_node_to_wires(self, tile, node):

prjxray/lms_solver.py

@@ -54,6 +54,8 @@
 import numpy as np
 import numpy.linalg as linalg
 
+from prjxray.util import OpenSafeFile
+
 # =============================================================================
 
 
@@ -83,7 +85,7 @@ def load_data(file_name, tagfilter=lambda tag: True, address_map=None):
     segdata = None
     all_segdata = []
 
-    with open(file_name, "r") as fp:
+    with OpenSafeFile(file_name, "r") as fp:
         for line in fp.readlines():
             line = line.strip()
 
@@ -174,7 +176,7 @@ def write_segbits(file_name, all_tags, all_bits, W):
 
         lines.append(all_tags[r] + " " + " ".join(bits) + "\n")
 
-    with open(file_name, "w") as fp:
+    with OpenSafeFile(file_name, "w") as fp:
         for line in lines:
             fp.write(line)
 
@@ -702,7 +704,7 @@ def build_address_map(tilegrid_file):
     address_map = {}
 
     # Load tilegrid
-    with open(tilegrid_file, "r") as fp:
+    with OpenSafeFile(tilegrid_file, "r") as fp:
         tilegrid = json.load(fp)
 
     # Loop over tiles
@@ -982,7 +984,7 @@ def tagfilter(tag):
 
     # Dump to CSV
     if args.x is not None:
-        with open(args.x, "w") as fp:
+        with OpenSafeFile(args.x, "w") as fp:
             dump_solution_to_csv(fp, tags_to_solve, bits_to_solve, X)
 
     # Dump results

prjxray/node_lookup.py

@@ -13,6 +13,7 @@
 import pyjson5 as json5
 import os.path
 
+from prjxray.util import OpenSafeFile
 
 def create_tables(conn):
     c = conn.cursor()
@@ -63,7 +64,7 @@ def build_database(self, nodes, tiles):
 
         nodes_processed = set()
         for node in progressbar.progressbar(nodes):
-            with open(node) as f:
+            with OpenSafeFile(node) as f:
                 node_wires = json5.load(f)
                 assert node_wires['node'] not in nodes_processed
                 nodes_processed.add(node_wires['node'])

prjxray/segmaker.py

@@ -24,7 +24,7 @@
 '''
 
 import os, json, re
-from prjxray import util
+from prjxray.util import OpenSafeFile, get_db_root, get_fabric
 
 BLOCK_TYPES = set(('CLB_IO_CLK', 'BLOCK_RAM', 'CFG_CLB'))
 
@@ -85,12 +85,12 @@ class Segmaker:
     def __init__(self, bitsfile, verbose=None, db_root=None, fabric=None):
         self.db_root = db_root
         if self.db_root is None:
-            self.db_root = util.get_db_root()
+            self.db_root = get_db_root()
             assert self.db_root, "No db root specified."
 
         self.fabric = fabric
         if self.fabric is None:
-            self.fabric = util.get_fabric()
+            self.fabric = get_fabric()
             assert self.fabric, "No fabric specified."
 
         self.verbose = verbose if verbose is not None else os.getenv(
@@ -129,7 +129,7 @@ def set_def_bt(self, block_type):
 
     def load_grid(self):
         '''Load self.grid holding tile addresses'''
-        with open(os.path.join(self.db_root, self.fabric, "tilegrid.json"),
+        with OpenSafeFile(os.path.join(self.db_root, self.fabric, "tilegrid.json"),
                   "r") as f:
             self.grid = json.load(f)
         assert "segments" not in self.grid, "Old format tilegrid.json"
@@ -152,7 +152,7 @@ def load_bits(self, bitsfile):
         '''
         self.bits = dict()
         print("Loading bits from %s." % bitsfile)
-        with open(bitsfile, "r") as f:
+        with OpenSafeFile(bitsfile, "r") as f:
             for line in f:
                 # ex: bit_00020500_000_17
                 line = line.split("_")
@@ -446,7 +446,7 @@ def write(self, suffix=None, roi=False, allow_empty=False):
             segments = self.segments_by_type[segtype]
             if segments:
                 print("Writing %s." % filename)
-                with open(filename, "w") as f:
+                with OpenSafeFile(filename, "w") as f:
                     for segname, segdata in sorted(segments.items()):
                         # seg 00020300_010
                         print("seg %s" % segname, file=f)

prjxray/tile.py

@@ -13,6 +13,7 @@
 import json
 from prjxray import lib
 from prjxray.timing import fast_slow_tuple_to_corners, RcElement
+from prjxray.util import OpenSafeFile
 
 TileDbs = namedtuple(
     'TileDbs', 'segbits block_ram_segbits ppips mask tile_type')
@@ -313,7 +314,7 @@ def yield_pips(pips):
                     backward_timing=get_pip_timing(pip.get('dst_to_src')),
                 )
 
-        with open(self.tile_dbs.tile_type) as f:
+        with OpenSafeFile(self.tile_dbs.tile_type) as f:
             tile_type = json.load(f)
             assert self.tilename_upper == tile_type['tile_type']
             self.wires = get_wires(tile_type['wires'])

prjxray/tile_segbits.py

@@ -10,8 +10,8 @@
 # SPDX-License-Identifier: ISC
 from collections import namedtuple
 from prjxray import bitstream
-from prjxray import util
 from prjxray.grid_types import BlockType
+from prjxray.util import OpenSafeFile
 import enum
 
 
@@ -84,22 +84,16 @@ def __init__(self, tile_db):
         self.feature_addresses = {}
 
         if tile_db.ppips is not None:
-            with open(tile_db.ppips) as f:
-                util.lock_file(f, 10)
+            with OpenSafeFile(tile_db.ppips) as f:
                 self.ppips = read_ppips(f)
-                util.unlock_file(f)
 
         if tile_db.segbits is not None:
-            with open(tile_db.segbits) as f:
-                util.lock_file(f, 10)
+            with OpenSafeFile(tile_db.segbits) as f:
                 self.segbits[BlockType.CLB_IO_CLK] = read_segbits(f)
-                util.unlock_file(f)
 
         if tile_db.block_ram_segbits is not None:
-            with open(tile_db.block_ram_segbits) as f:
-                util.lock_file(f, 10)
+            with OpenSafeFile(tile_db.block_ram_segbits) as f:
                 self.segbits[BlockType.BLOCK_RAM] = read_segbits(f)
-                util.unlock_file(f)
 
         for block_type in self.segbits:
             for feature in self.segbits[block_type]:

prjxray/tile_segbits_alias.py

@@ -21,6 +21,7 @@
 from prjxray import bitstream
 from prjxray.grid_types import Bits
 from prjxray.tile_segbits import read_ppips
+from prjxray.util import OpenSafeFile
 
 
 class TileSegbitsAlias(object):
@@ -67,7 +68,7 @@ def __init__(self, db, tile_type, bits_map):
         self.ppips = {}
 
         if tile_db.ppips is not None:
-            with open(tile_db.ppips) as f:
+            with OpenSafeFile(tile_db.ppips) as f:
                 self.ppips = read_ppips(f)
         self.tile_segbits = db.get_tile_segbits(self.alias_tile_type)