"""compare.py: check a run_all.sh output directory against expected_small/, and the stored full-resolution outputs against the paper. Usage: python compare.py OUTDIR EXPECTED_DIR 1. Every file under EXPECTED_DIR must exist under OUTDIR at the same relative path and agree with it: - JSON files field by field, exactly, except the elapsed-time field "seconds" and the machine-dependent fields "cpu_affinity" and "task_count" that the audit scripts record; - text outputs (.out) line by line, exactly, after removing the leading "[ 12.3s]" elapsed-time stamps and the elapsed times printed inside lines ("done in 79.1s", "(9.4s)"). 2. The stored full-resolution outputs (full/, next to this script) must carry the verdict PASS and a margin at least the value stated in the paper's table of cases. JSON margins are exact fractions; the two text outputs print their margins rounded outward (downward). The stored output of the caps computation must also show sup F < 4 (so a_m <= 5 b'_m), with sup F = sup f / |U| exactly. 3. Each input pickle decoded by run_all.sh.txt must hash to a pickle_sha256 recorded in full/, and every recorded hash must be matched. 4. Each plain-text export inputs/.json (next to this script) must rebuild, value by value and type by type (floats bit for bit), to the object in the decoded pickle of the same name, and must record that pickle's SHA-256. Stored text outputs carry an extra .txt extension (expected_small/sep_p4/cert3_M144000.out.txt is the stored cert3_M144000.out). Prints one line per check and ends with ALL REPRODUCED or the number of failures. """ import hashlib, json, os, re, sys from fractions import Fraction TIME_KEYS = {'seconds', 'cpu_affinity', 'task_count'} # elapsed time; the audit scripts' record of the machine's CPU set and threads STAMP = re.compile(r'^\[\s*[0-9.]+s\]\s?') INLINE_TIME = re.compile(r'(done in |\()[0-9.]+s\b') # elapsed times printed inside lines (sep_p4) HERE = os.path.dirname(os.path.abspath(__file__)) STATED = [ # (stored full output, stated margin in the paper's table of cases) ('full/certb_e11a10b3_M79200.json', '0.08129'), ('full/certb_e14a8_19c_M107520.json', '0.05763'), ('full/cert4v3_v2main_M216000.json', '0.02909'), ('full/cert3_M288000.out.txt', '0.04587'), ('full/cert2v2_p5frozen_M7200.json', '0.03864'), ('full/rig_M432000.out.txt', '0.01609'), ('full/cert2_g93adjQ32_M7680.json', '0.013338'), ('full/cert2_p7r_M23040.json', '0.005347'), ('full/caps_p5frozen_M7200.json', '0.03864'), ] CAPS = 'full/caps_p5frozen_M7200.json' # sup F < 4 gives the cap a_m <= 5 b'_m of the case (1/20, 1/5) def rebuild(x): """Rebuild an object from the tagged JSON of inputs/.json.""" import numpy as np if isinstance(x, list): return [rebuild(v) for v in x] if isinstance(x, dict): (tag, v), = x.items() if tag == 'float': return float.fromhex(v) if tag == 'fraction': return Fraction(v) if tag == 'tuple': return tuple(rebuild(w) for w in v) if tag == 'dict': return {rebuild(k): rebuild(w) for k, w in v} if tag == 'ndarray': dt = np.dtype(v['dtype']) data = [float.fromhex(w) for w in v['data']] if dt.kind == 'f' else v['data'] return np.array(data, dtype=dt).reshape(v['shape']) if tag == 'npscalar': dt = np.dtype(v['dtype']) return dt.type(float.fromhex(v['value']) if dt.kind == 'f' else v['value']) raise ValueError(tag) return x # int, bool, None, str def identical(a, b): """Same type and same value, floats bit for bit (by float.hex), dicts in the same order.""" import numpy as np if type(a) is not type(b): return False if isinstance(a, np.ndarray): if a.dtype != b.dtype or a.shape != b.shape: return False if a.dtype.kind == 'f': return [float(v).hex() for v in a.reshape(-1).tolist()] == [float(v).hex() for v in b.reshape(-1).tolist()] return a.tolist() == b.tolist() if isinstance(a, np.generic): return a.dtype == b.dtype and (float(a).hex() == float(b).hex() if a.dtype.kind == 'f' else a == b) if isinstance(a, float): return a.hex() == b.hex() if isinstance(a, dict): return list(a) == list(b) and all(identical(k, l) for k, l in zip(a, b)) and all(identical(a[k], b[k]) for k in a) if isinstance(a, (list, tuple)): return len(a) == len(b) and all(identical(x, y) for x, y in zip(a, b)) return a == b def strip_time(obj): if isinstance(obj, dict): return {k: strip_time(v) for k, v in obj.items() if k not in TIME_KEYS} if isinstance(obj, list): return [strip_time(v) for v in obj] return obj def first_difference(a, b, path=''): if type(a) != type(b): return path or '/' if isinstance(a, dict): for k in sorted(set(a) | set(b)): if k not in a or k not in b: return path + '/' + k d = first_difference(a[k], b[k], path + '/' + k) if d: return d return None if isinstance(a, list): if len(a) != len(b): return path + ' (length)' for i, (x, y) in enumerate(zip(a, b)): d = first_difference(x, y, '%s[%d]' % (path, i)) if d: return d return None return None if a == b else (path or '/') def main(): out, exp = sys.argv[1], sys.argv[2] fails = 0 print('# 1. small-resolution runs against expected_small/') for dp, dn, fn in sorted(os.walk(exp)): for f in sorted(fn): e = os.path.join(dp, f) rel = os.path.relpath(e, exp) o = os.path.join(out, rel[:-4] if rel.endswith('.out.txt') else rel) # stored text outputs carry an extra .txt if not os.path.exists(o): print('MISSING %s' % rel); fails += 1; continue if f.endswith('.json'): a, b = strip_time(json.load(open(o))), strip_time(json.load(open(e))) d = first_difference(a, b) ok = d is None print('%s %s%s' % ('SAME ' if ok else 'DIFFERENT', rel, '' if ok else ' first difference at ' + d)) else: a = [INLINE_TIME.sub(r'\1Ts', STAMP.sub('', l)) for l in open(o).read().splitlines()] b = [INLINE_TIME.sub(r'\1Ts', STAMP.sub('', l)) for l in open(e).read().splitlines()] ok = a == b if not ok: i = next((k for k, (x, y) in enumerate(zip(a, b)) if x != y), min(len(a), len(b))) print('%s %s%s' % ('SAME ' if ok else 'DIFFERENT', rel, '' if ok else ' first difference at line %d' % (i + 1))) fails += not ok print('# 2. stored full-resolution outputs against the paper\'s stated margins') for rel, stated in STATED: p = os.path.join(HERE, rel) if rel.endswith('.json'): d = json.load(open(p)) m = Fraction(d['margin']) ok = d.get('verdict') == 'PASS' and m >= Fraction(stated) shown = '%s = %.9f' % (d['margin'], float(m)) else: mm = re.findall(r'RESULT M = \d+: scaled margin on the segment >= ([0-9.]+) .*; PASS at this M', open(p).read()) ok = len(mm) == 1 and Fraction(mm[0]) >= Fraction(stated) shown = 'printed (outward) %s' % (mm[0] if mm else 'NONE') print('%s %s: margin %s, stated >= %s' % ('PASS ' if ok else 'FAIL ', rel, shown, stated)) fails += not ok d = json.load(open(os.path.join(HERE, CAPS))) Fsup, supf, U = Fraction(d['F_sup']), Fraction(d['sup_f']), Fraction(d['f1'][1]) ok = d.get('verdict') == 'PASS' and U < 0 and Fsup == supf / (-U) and Fsup < 4 print('%s %s: sup F = %s = %.9f < 4, so 1 + F < 5 (caps 5)' % ('PASS ' if ok else 'FAIL ', CAPS, d['F_sup'], float(Fsup))) fails += not ok print('# 3. the decoded input pickles against the hashes recorded in the stored outputs') recorded = {json.load(open(os.path.join(HERE, 'full', f))).get('pickle_sha256') for f in os.listdir(os.path.join(HERE, 'full')) if f.endswith('.json')} decoded = {} for f in sorted(os.listdir(os.path.join(out, 'inputs'))): if f.endswith('.pkl'): decoded[f] = hashlib.sha256(open(os.path.join(out, 'inputs', f), 'rb').read()).hexdigest() for f, h in decoded.items(): ok = h in recorded print('%s inputs/%s: sha256 %s %s' % ('PASS ' if ok else 'FAIL ', f, h[:16], 'recorded in full/' if ok else 'NOT recorded')) fails += not ok missing = recorded - set(decoded.values()) if missing: print('FAIL recorded pickle hashes with no decoded input: %s' % sorted(m[:16] for m in missing)); fails += 1 print('# 4. the plain-text exports of the inputs against the decoded pickles') import pickle exports = sorted(f for f in os.listdir(os.path.join(HERE, 'inputs')) if f.endswith('.json')) for f in exports: name = f[:-5] pk = os.path.join(out, 'inputs', name + '.pkl') doc = json.load(open(os.path.join(HERE, 'inputs', f))) if not os.path.exists(pk): print('FAIL inputs/%s: no decoded pickle %s.pkl' % (f, name)); fails += 1; continue raw = open(pk, 'rb').read() ok = doc.get('pickle_sha256') == hashlib.sha256(raw).hexdigest() and identical(rebuild(doc['value']), pickle.loads(raw)) print('%s inputs/%s: rebuilds to the object in %s.pkl' % ('PASS ' if ok else 'FAIL ', f, name)) fails += not ok if len(exports) != len(decoded): print('FAIL %d exports for %d decoded pickles' % (len(exports), len(decoded))); fails += 1 print('ALL REPRODUCED' if fails == 0 else '%d FAILURE(S)' % fails) if __name__ == '__main__': main()