testrunner.py

from pathlib import Path
from textparser import parse2json
from pyparsing import ParseException
from datetime import datetime
import json
import argparse
from shutil import copyfile

ap = argparse.ArgumentParser()
ap.add_argument('src', help='EM TXT input directory')
ap.add_argument('dst', help='JSON output parent directory')
ap.add_argument('-b', '--base',
                help='set specific baseline directory')
ap.add_argument('-q', '--quick',
                help='ignore footnotes to save a bit of time',
                action='store_true')
args = ap.parse_args()

srcdir = Path(args.src)
dstdir = Path(args.dst)

dtfmt = '%Y-%m-%d_%H-%M-%S'
dt = datetime.now().strftime(dtfmt)
resultsdir = dstdir / dt
investigatedir = resultsdir / 'investigate'

# choose most recent dir for baseline
datedirs = [(datetime.strptime(d.name, dtfmt), d)
            for d in dstdir.glob('[0-9][0-9][0-9][0-9]-*')
            if d.is_dir()]

datedirs.sort(reverse=True, key=lambda p: p[0])
_, basedir = datedirs[0]

# ... unless otherwise specified
if args.base is not None:
    basedir = Path(args.base)

diffFile = investigatedir / 'diff-log.txt'
investigatedir.mkdir(parents=True)


def makecopy(file: Path) -> None:
    copypath = investigatedir / file.name
    copyfile(str(file), str(copypath))


with open(str(diffFile), 'w') as df:
    for emTxt in srcdir.glob('uksiem_*.txt'):
        emJson = emTxt.with_suffix('.json').name
        result = resultsdir / emJson
        base = basedir / emJson
        baseExists = base.exists()
        extraMsg = ''
        processFtnotes = not args.quick
        print('Processing ' + emTxt.name)
        try:
            extraMsg = parse2json(str(emTxt), str(result), processFtnotes)
        except ParseException as ex:
            errMsg = '{0}:\n{1}\n{2}^\n{3}\n'.format(emTxt.name,
                                                     ex.line,
                                                     ' '*(ex.column-1),
                                                     ex)
            # did this parse previously? if so mark "dead"
            if baseExists:
                print('[DEAD] ' + emTxt.name)
                df.write('[DEAD] ' + errMsg)
            # otherwise "fail" = still dead
            else:
                df.write('[FAIL] ' + errMsg)
            makecopy(emTxt)
            continue
        # any extra messages from parse? if so, copy for later investigation
        if len(extraMsg) > 0:
            makecopy(emTxt)
            makecopy(result)
        # if base didn't exist then it's become "live"
        if not baseExists:
            print('[LIVE] {0} {1}'.format(emJson, extraMsg))
            df.write('[LIVE] {0} {1}\n'.format(emJson, extraMsg))
        # otherwise check whether we have a "diff"
        else:
            with open(str(base)) as bf, \
                 open(str(result)) as rf:
                bdict = json.load(bf)
                rdict = json.load(rf)
                if rdict != bdict:
                    # print('[DIFF] {0} {1}'.format(emJson, extraMsg))
                    df.write('[DIFF] {0} {1}\n'.format(emJson, extraMsg))
                else:
                    df.write('[PASS] {0} {1}\n'.format(emJson, extraMsg))