# Sources of the experiments behind section 8 Every script and recipe that produced the outputs in `outputs_*.log`, verbatim except as stated for each. Two hashes are given for each file: "as run" is the SHA-256 of the file that produced the outputs (provenance); "as shipped" is the SHA-256 of the text below (integrity), which `unpack.py` checks. For the scripts, the identity line states the relation between the two: the shipped text has neutral docstrings and comments (names and internal references removed) and, where listed, relative paths in place of machine-specific ones; parsed with docstrings dropped, it is identical to the file as run with those path changes applied. The recipes are records of the commands that were run, with machine-specific paths replaced as listed; `cpulimit.sh` was a CPU-time limiter of the machine that ran them and is not shipped. Where the file as run is not kept byte for byte, a "provenance" line says how its "as run" hash is reproduced. #### file: cont_hm.py - kind: script - as run: SHA-256 fb983e4c589ce8cc09577d680f945962bb8cca8b91bcf088d7f4f42b3a70a36d - as shipped: SHA-256 17938884f856e1942514a4b35d23c81a77dd160b0f0a2518b2a865c27c23051d - identity: identical, after dropping docstrings and comments, to the file as run - trailing newline: yes ~~~~~~~~~~~~ python #!/usr/bin/env python3 """cont_hm.py: floating-point continuum linear program for the forced family of the construction (free data f(u) = c[j, k, b] + 1[tau <= alpha] H_j(tau), profile w_k = 1) at general m*. Model: Ford and Maynard's continuum, with Type I information at level 1/2 built in through the fragmentation relation (6.3) and the closed form of the truncated Linnik kernel; Moebius-twisted rows T_jD(sigma) at level alpha; eta-rough support. Here j counts the small components (in [eta, alpha]) and tau is their total, k counts the rough components (in (alpha, 1/2)) and b is the bin of the largest one; H_j lives on [j eta, alpha]. The program keeps every H_j as a variable and minimises f(1) subject to the rows and to f >= -1 at grid points. Floats; not a certificate. Usage: cont_hm.py N eta_num eta_den alpha_num alpha_den EDGES PHIMAX ASUB SSUB [--check] Environment: CONT_REG (minimise sum |c| subject to f(1) <= -1 - REG), CONT_TAG (save a pickle), CONT_NOUPPER=1 (drop the rows f <= PHIMAX), CONT_DUAL=1 (print the duals by class).""" import sys, time, os, pickle from fractions import Fraction as Fr import numpy as np from scipy.signal import fftconvolve from scipy.optimize import linprog from scipy.sparse import csr_matrix t0 = time.time() N = int(sys.argv[1]); eta = Fr(int(sys.argv[2]), int(sys.argv[3])); alpha = Fr(int(sys.argv[4]), int(sys.argv[5])) EDG = [Fr(e) for e in sys.argv[6].split(',')] if sys.argv[6] else [] PHIMAX = float(sys.argv[7]); ASUB = int(sys.argv[8]); SSUB = int(sys.argv[9]) assert eta <= alpha < Fr(1, 2) h = 1.0 / N def idx(x): v = x * N; assert v.denominator == 1, f"{x} not on grid N={N}"; return int(v) ie, ia, ihf = idx(eta), idx(alpha), idx(Fr(1, 2)) edges = [alpha] + EDG + [Fr(1, 2)]; iedges = [idx(e) for e in edges]; NB = len(edges) - 1 inv = np.zeros(N + 1); inv[1:] = 1.0 / (np.arange(1, N + 1) * h) def conv(F, G): return fftconvolve(F, G)[:N + 1] * h delta = np.zeros(N + 1); delta[0] = 1.0 / h def band(lo, hi): F = np.zeros(N + 1); F[lo:hi + 1] = inv[lo:hi + 1]; F[lo] *= 0.5; F[hi] *= 0.5; return F def trunc_ge(F, i0): G = F.copy(); G[:max(i0, 0)] = 0.0 if 0 <= i0 <= N: G[i0] *= 0.5 return G def shift(F, i): G = np.zeros(N + 1); G[i:] = F[:N + 1 - i]; return G s_den = band(ie, ia) JMAX = int(1 / eta) + 1; KMAX = int(1 / alpha) + 1 S = [delta] for j in range(1, JMAX + 1): S.append(conv(S[-1], s_den) / j) S2 = S[2] Rcap = {} for e in iedges[1:]: r = band(ia, e); P = [delta] for k in range(1, KMAX + 1): P.append(conv(P[-1], r) / k) Rcap[e] = P def Rbin(k, b): if k == 0: return delta hi = iedges[b] return Rcap[hi][k] if b == 1 else Rcap[hi][k] - Rcap[iedges[b - 1]][k] Rall = Rcap[ihf] bins_k = lambda k: [0] if k == 0 else list(range(1, NB + 1)) etaf, alf, ef = float(eta), float(alpha), [float(e) for e in edges] def support(j, k, b): if k == 0: return (j * etaf, j * alf) return (j * etaf + (k - 1) * alf + ef[b - 1], j * alf + k * ef[b]) def in_sup(sup, t): return sup[1] > sup[0] and sup[0] - 1e-12 <= t <= sup[1] + 1e-12 print(f"[{time.time()-t0:.1f}s] N={N} eta={eta} alpha={alpha} edges={[str(e) for e in edges]}", flush=True) # ---- Lemma K kernels (c-part) ---- PS = {} for e in iedges[1:]: for bs in range(JMAX + 1): for br in range(KMAX + 1): PS[(e, bs, br)] = conv(S[bs], Rcap[e][br]) if br else S[bs] def Lam_cap(jy, ky, e): K = jy + ky; L = -PS[(e, jy, ky)].copy(); acc = np.zeros(N + 1) for bs in range(jy + 1): for br in range(ky + 1): if (bs, br) in ((0, 0), (jy, ky)): continue acc += (-1) ** (bs + br) * conv(trunc_ge(PS[(e, bs, br)], ihf), PS[(e, jy - bs, ky - br)]) return L + (-1) ** (K + 1) * acc Lam = {} for jy in range(JMAX + 1): for ky in range(KMAX + 1): if jy + ky < 2: continue if ky == 0: Lam[(jy, 0, 0)] = Lam_cap(jy, 0, ihf); continue prev = None for b in range(1, NB + 1): cur = Lam_cap(jy, ky, iedges[b]); Lam[(jy, ky, b)] = cur if prev is None else cur - prev; prev = cur Lam0 = sum(Lam_cap(0, ky, ihf) for ky in range(2, KMAX + 1)) del PS # ---- XS(t; theta) for theta = 1/2 - x, x in [0, ia] ---- XS = {} for x in range(0, ia + 1): acc = np.zeros(N + 1) for k in range(1, KMAX + 1): for br in range(1, k + 1): acc += (-1) ** (k + br) * conv(trunc_ge(Rall[br], ihf - x), Rall[k - br]) XS[x] = acc # ======================= cont_hm: general m* ======================= MS = ia // ie # largest j with j*eta <= alpha (in grid units) while MS * ie > ia: MS -= 1 Hgrid = {j: list(range(j * ie, ia + 1)) for j in range(1, MS + 1)} print(f"[{time.time()-t0:.1f}s] m* (MS) = {MS}; H grids: " + ", ".join(f"H{j}: {len(Hgrid[j])}" for j in Hgrid), flush=True) Rall = Rcap[ihf] Rsgn = sum(((-1) ** k) * Rall[k] for k in range(0, KMAX + 1)) # sum_k (-1)^k R_k (k = 0: delta), all rough structures Runs = sum(Rall[k] for k in range(0, KMAX + 1)) # ---- M_j[sigma, tau] (subset-sum sigma, total tau), densities in tau; M_0 = delta ---- na = ia + 1 M = [np.zeros((na, na)) for _ in range(MS + 1)] M[0][0, 0] = 1.0 / h wsm = h * s_den for j in range(1, MS + 1): acc = np.zeros((na, na)) for t in range(ie, ia + 1): w = wsm[t] if w == 0.0: continue acc[:, t:] += w * M[j - 1][:, :na - t] acc[t:, t:] -= w * M[j - 1][:na - t, :na - t] M[j] = acc / j XSmat = np.array([XS[x] for x in range(0, ia + 1)]) # (na, N+1): XS(.; 1/2 - x h) LJ = {} # LJ[j]: (na, N+1), row tau = Lambda^(j)(.; tau) for j in range(1, MS + 1): A = (M[j].T @ XSmat) * ((-1) ** (j - 1)) # row tau: sum_sigma M_j[sigma, tau] XS[sigma](.) L = np.zeros((na, N + 1)) for tau in Hgrid[j]: L[tau, tau:] = A[tau, :N + 1 - tau] # shift by tau: evaluate at a - tau LJ[j] = L del XSmat LJTr = {j: np.where(np.arange(N + 1)[None, :] >= ihf, LJ[j], 0.0) for j in LJ} for j in LJTr: LJTr[j][:, ihf] *= 0.5 # trunc_ge convention: half weight at a = 1/2 Lam0Tr = trunc_ge(Lam0, ihf) K0sgn = conv(Rsgn, Lam0Tr); K0uns = conv(Runs, Lam0Tr) Ksgn = {}; Kuns = {} for j in range(1, MS + 1): Ks = np.zeros((na, N + 1)); Ku = np.zeros((na, N + 1)) for tau in Hgrid[j]: Ks[tau] = conv(Rsgn, LJTr[j][tau]); Ku[tau] = conv(Runs, LJTr[j][tau]) Ksgn[j] = Ks; Kuns[j] = Ku print(f"[{time.time()-t0:.1f}s] kernels: Lam {len(Lam)}, Lambda^(j) for j <= {MS}", flush=True) if '--kcheck' in sys.argv: # j = 1, 2 against cont_h2b's L1, L2 (densities): L1[i] = inv[i] shift(XS0 - XSi, i); L2[t] = -1/2 shift(sum_l h s s (...), t) for i in [ie + 1, (ie + ia) // 2, ia - 1]: ref = shift(XS[0] - XS[i], i) * inv[i] print(f" kcheck j=1 tau={i}: max|LJ - L1| = {np.abs(LJ[1][i] - ref).max():.3e} (max|L1| {np.abs(ref).max():.3e})") if MS >= 2: for t in [2 * ie + 1, (2 * ie + ia) // 2, ia]: acc = np.zeros(N + 1) for l in range(max(ie, t - ia), min(ia, t - ie) + 1): acc += h * s_den[l] * s_den[t - l] * (XS[0] - XS[l] - XS[t - l] + XS[t]) ref = -0.5 * shift(acc, t) print(f" kcheck j=2 tau={t}: max|LJ - L2| = {np.abs(LJ[2][t] - ref).max():.3e} (max|L2| {np.abs(ref).max():.3e})") a_ = np.arange(ihf, N + 1) print(f" kcheck Lam0 vs -XS(.;1/2) on [1/2,1]: max diff {np.abs(Lam0[a_] + XS[0][a_]).max():.3e} (max|Lam0| {np.abs(Lam0[a_]).max():.3e})") # ---- variables ---- classes = [(j, k, b) for j in range(JMAX + 1) for k in range(KMAX + 1) if j + k >= 1 for b in bins_k(k)] D = {cl: (conv(S[cl[0]], Rbin(cl[1], cl[2])) if cl[1] else S[cl[0]]) for cl in classes}; D[(0, 0, 0)] = delta cid = {cl: n for n, cl in enumerate(classes)}; nc = len(classes) hid = {}; nv = nc for j in range(1, MS + 1): for tau in Hgrid[j]: hid[(j, tau)] = nv; nv += 1 hcols = {j: np.array([hid[(j, tau)] for tau in Hgrid[j]]) for j in Hgrid} def new(): return np.zeros(nv) LamTr = {k_: trunc_ge(L, ihf) for k_, L in Lam.items()} w_classes = [(0, 0, 0)] + [cl for cl in classes if cl[0] + cl[1] >= 1 and in_sup(support(*cl), support(*cl)[0]) and support(*cl)[0] <= 0.5 + 1e-12] _cc = {} def conv_w_lam(wc, key): k_ = (wc, key) if k_ not in _cc: _cc[k_] = LamTr[key] if wc == (0, 0, 0) else conv(D[wc], LamTr[key]) return _cc[k_] # ---- f(1) ---- f1 = new() for key, L in Lam.items(): if (cl := cid.get(key)) is not None: f1[cl] += L[N] for j in range(1, MS + 1): for tau in Hgrid[j]: f1[hid[(j, tau)]] += h * LJ[j][tau, N] # ---- the rows T_jD(sigma) ---- row_list = [(0, 0)] + [(jD, s_) for jD in range(1, MS + 1) for s_ in Hgrid[jD]] rid = {r: n for n, r in enumerate(row_list)}; nr_ = len(row_list) T = np.zeros((nr_, nv)); I = np.zeros((nr_, nv)) DIAG = np.zeros((nr_, nv), dtype=bool) # marks the unweighted diagonal entries (H_jD(sigma) in its own row) for jD in range(0, MS + 1): sig = [0] if jD == 0 else Hgrid[jD] rows_ix = np.array([rid[(jD, s_)] for s_ in sig]); tt = np.array([N - s_ for s_ in sig]) # (A) free completions, c-part for cl in classes: jp, kp, bp = cl tgt = cid.get((jD + jp, kp, bp)) if tgt is None: continue vals = D[cl][tt]; sg = (-1) ** (jD + jp + kp) T[rows_ix, tgt] += sg * vals; I[rows_ix, tgt] += vals # (B) extended completions, c-part (w = empty included: for jD = 0 that is the prime, -f(1)'s c-part) for wc in w_classes: jw, kw, bw = wc; sgn = (-1) ** (1 + jD + jw + kw) for key in Lam: jy, ky, byy = key tgt = cid.get((jD + jw + jy, kw + ky, max(bw, byy))) if tgt is None: continue v = conv_w_lam(wc, key)[tt] T[rows_ix, tgt] += sgn * v; I[rows_ix, tgt] += v # H-parts. Row sigma, column (J, tau), tau >= sigma. for n_, s_ in enumerate(sig): r = rows_ix[n_] if jD >= 1: # diagonal: free (je = 0) and extended (jw = jy = 0) col = hid[(jD, s_)] T[r, col] += ((-1) ** jD) * Rsgn[N - s_] + ((-1) ** (1 + jD)) * K0sgn[N - s_] I[r, col] += Runs[N - s_] + K0uns[N - s_] DIAG[r, col] = True for J in range(max(jD, 1), MS + 1): dj = J - jD taus = np.array([t_ for t_ in Hgrid[J] if t_ >= s_ + (dj * ie if dj > 0 else 0) and t_ <= ia]) if dj == 0 or len(taus) == 0: if not (dj == 0 and jD == 0): continue if len(taus) == 0: continue cols = np.array([hid[(J, t_)] for t_ in taus]); u = taus - s_ # u = small total added to d vt = np.zeros(len(taus)); vu = np.zeros(len(taus)) # (A) free, je = dj >= 1: (-1)^J h S_je(u) Rsgn(N - tau) if dj >= 1: vt += ((-1) ** J) * h * S[dj][u] * Rsgn[N - taus]; vu += h * S[dj][u] * Runs[N - taus] # (B) extended: jw + jy = dj for jw in range(0, dj + 1): jy = dj - jw; sgn = (-1) ** (1 + jD + jw) if jw == 0 and jy == 0: continue # the diagonal, done above (jD >= 1) if jw == 0: # h K_jy(N - sigma; u) vt += sgn * h * Ksgn[jy][u, N - s_]; vu += h * Kuns[jy][u, N - s_] elif jy == 0: # h S_jw(u) K0(N - tau) vt += sgn * h * S[jw][u] * K0sgn[N - taus]; vu += h * S[jw][u] * K0uns[N - taus] else: # h^2 sum_v S_jw(v - sigma) K_jy(N - v; tau - v) for m_, t_ in enumerate(taus): vv = np.arange(s_ + jw * ie, t_ - jy * ie + 1) if len(vv) == 0: continue sw = S[jw][vv - s_] vt[m_] += sgn * h * h * np.dot(sw, Ksgn[jy][t_ - vv, N - vv]) vu[m_] += h * h * np.dot(sw, Kuns[jy][t_ - vv, N - vv]) T[r, cols] += vt; I[r, cols] += vu print(f"[{time.time()-t0:.1f}s] rows: {nr_} (T0 + " + ", ".join(f"T{j}: {len(Hgrid[j])}" for j in Hgrid) + ")", flush=True) T[0] += 0.0 # the prime's H-part sits in T0 through (B) with jw = 0 (Ksgn includes w = empty), so T0 = free + extended incl. -f(1): check Hcols = np.arange(nc, nv) Iw = np.abs(I).copy(); Iw[:, nc:] = np.where(DIAG[:, nc:], Iw[:, nc:], Iw[:, nc:] / h) print(f" Type I check: max|I| c-block {np.abs(I[:, :nc]).max():.3e}; H diagonal (unweighted) {np.abs(I[DIAG]).max() if DIAG.any() else 0:.3e};" f" H integrated / h {np.abs(np.where(DIAG[:, nc:], 0.0, I[:, nc:])).max() / h:.3e}; T0 row: c {np.abs(I[0, :nc]).max():.3e}, H/h {np.abs(I[0, nc:]).max()/h:.3e}", flush=True) for jD in range(0, MS + 1): rr = [rid[(jD, s_)] for s_ in ([0] if jD == 0 else Hgrid[jD])] print(f" rows T{jD}: c {np.abs(I[rr, :nc]).max():.3e}; H(diag) {np.abs(I[rr][:, nc:][DIAG[rr][:, nc:]]).max() if DIAG[rr].any() else 0:.3e};" f" H(int)/h {np.abs(np.where(DIAG[rr][:, nc:], 0.0, I[rr][:, nc:])).max()/h:.3e}", flush=True) if jD >= 0: Bd = [T[rid[(j, s_)], hid[(j, s_)]] for j in Hgrid for s_ in Hgrid[j]] print(f" diagonal B_j(sigma): " + "; ".join(f"B{j} in [{min(T[rid[(j, s_)], hid[(j, s_)]] for s_ in Hgrid[j]):.3f}, {max(T[rid[(j, s_)], hid[(j, s_)]] for s_ in Hgrid[j]):.3f}]" for j in Hgrid), flush=True) if '--check' in sys.argv: sys.exit() # ---- positivity ---- rows = []; desc = [] lb = np.full(nv, -np.inf); ub = np.full(nv, np.inf); lb[:nc] = -1.0; ub[:nc] = PHIMAX for cl in classes: j, k, b = cl if not in_sup(support(j, k, b), 1.0) or j == 0 or j > MS or k == 0: continue supR = support(0, k, b) for tau in Hgrid[j]: if in_sup(supR, 1 - tau * h): v = new(); v[cid[cl]] = 1; v[hid[(j, tau)]] = 1; rows.append(v); desc.append(('free', cl, round(tau * h, 4), None)) nfree = len(rows) alist = list(range(ihf, N, ASUB)) for bc in w_classes: jb, kb, bb = bc if bc == (0, 0, 0): continue supR = support(0, kb, bb) for ai in alist: a = ai * h; tb = N - ai if not in_sup(support(jb, kb, bb), tb * h): continue base = new() for key, L in Lam.items(): jy, ky, bp = key; tgt = cid.get((jb + jy, kb + ky, max(bb, bp))) if tgt is not None: base[tgt] += a * L[ai] if jb == 0: for j in range(1, MS + 1): base[hcols[j]] += h * a * LJ[j][Hgrid[j], ai] rows.append(base); desc.append(('ext', bc, round(a, 4), None)); continue if kb: cands = [t_ for t_ in range(jb * ie, min(jb * ia, tb) + 1, SSUB) if in_sup(supR, (tb - t_) * h)] else: cands = [tb] if jb * ie <= tb <= jb * ia else [] done_const = False for tbeta in cands: if tbeta > ia: if not done_const: rows.append(base.copy()); desc.append(('ext', bc, round(a, 4), 'tau>alpha')); done_const = True continue v = base.copy() if jb <= MS: v[hid[(jb, tbeta)]] += a * Lam0[ai] for jy in range(1, MS - jb + 1): tys = [t_ for t_ in Hgrid[jy] if tbeta + t_ <= ia] for ty in tys: v[hid[(jb + jy, tbeta + ty)]] += h * a * LJ[jy][ty, ai] rows.append(v); desc.append(('ext', bc, round(a, 4), round(tbeta * h, 4))) G = np.array(rows) print(f"[{time.time()-t0:.1f}s] positivity rows: free {nfree}, extended {len(rows) - nfree}; G {G.shape} ({G.nbytes/1e9:.2f} GB)", flush=True) NOUP = bool(os.environ.get('CONT_NOUPPER')) Aub = -G if NOUP else np.vstack([-G, G]); bub = np.ones(len(G)) if NOUP else np.concatenate([np.ones(len(G)), np.full(len(G), PHIMAX)]) del G Aeq = T; beq = np.zeros(T.shape[0]) bnds = np.column_stack([lb, ub]) REG = os.environ.get('CONT_REG') Aub_s = csr_matrix(Aub); del Aub if REG is None: res = linprog(f1, A_ub=Aub_s, b_ub=bub, A_eq=csr_matrix(Aeq), b_eq=beq, bounds=bnds, method='highs-ipm', options={'presolve': True}) else: from scipy.sparse import vstack as svs, hstack as shs, identity as sid kap = float(REG); real_ix = [cid[cl] for cl in classes if in_sup(support(*cl), 1.0)]; nrr = len(real_ix) E = csr_matrix((np.ones(nrr), (np.arange(nrr), real_ix)), shape=(nrr, nv)) Aub2 = svs([shs([Aub_s, csr_matrix((Aub_s.shape[0], nrr))]), shs([E, -sid(nrr)]), shs([-E, -sid(nrr)]), shs([csr_matrix(f1[None, :]), csr_matrix((1, nrr))])]).tocsr() bub2 = np.concatenate([bub, np.zeros(2 * nrr), [-1.0 - kap]]) cost2 = np.concatenate([np.zeros(nv), np.ones(nrr)]) res = linprog(cost2, A_ub=Aub2, b_ub=bub2, A_eq=csr_matrix(np.hstack([Aeq, np.zeros((Aeq.shape[0], nrr))])), b_eq=beq, bounds=np.vstack([bnds, np.column_stack([np.zeros(nrr), np.full(nrr, np.inf)])]), method='highs-ipm', options={'presolve': True}) if res.status == 0: xx = res.x; res.x = xx[:nv]; print(f" regularised: sum|c| = {xx[nv:].sum():.4f}") print(f"[{time.time()-t0:.1f}s] LP status {res.status} ({res.message})", flush=True) if res.status == 0: x = res.x; z = f1 @ x print(f" min f(1) = {z:.6f} -> {'PRIME-FREE after rescaling, margin %.4f' % (1 - 1/abs(z)) if z < -1 else 'NOT prime-free, C- >= %.6f' % (1 + z)}") print(" " + "; ".join(f"H{j} in [{x[hcols[j]].min():.4f}, {x[hcols[j]].max():.4f}]" for j in Hgrid) + f"; max|c| {np.abs(x[:nc]).max():.3f}; |T| max {np.abs(T @ x).max():.1e}") if os.environ.get('CONT_DUAL') and REG is None: try: from collections import defaultdict ml = -np.asarray(res.ineqlin.marginals)[:len(desc)] me = np.asarray(res.eqlin.marginals) print(f" DUALS: sum over f >= -1 rows {ml.sum():.4f}; T0 dual {me[0]:+.4f}; other T rows |.| {np.abs(me[1:]).sum():.4f}") byc = defaultdict(float) for w, d in zip(ml, desc): byc[(d[0], d[1])] += w print(" DUALS by class (f >= -1):", "; ".join(f"{k[0]} {k[1]}: {v:.4f}" for k, v in sorted(byc.items(), key=lambda kv: -kv[1])[:12] if v > 1e-6)) print(" top rows:", "; ".join(f"{desc[i]} {ml[i]:.4f}" for i in np.argsort(-ml)[:14] if ml[i] > 1e-6)) except Exception as ex: print(" DUALS unavailable:", ex) tag = os.environ.get('CONT_TAG') if tag: pickle.dump(dict(N=N, eta=eta, alpha=alpha, edges=edges, classes=classes, Hgrid=Hgrid, x=x, f1=z, PHIMAX=PHIMAX, realisable=[cl for cl in classes if in_sup(support(*cl), 1.0)]), open(f"cont_hm_sol_{tag}.pkl", 'wb')) print(f" saved cont_hm_sol_{tag}.pkl") ~~~~~~~~~~~~ #### file: wall_check.py - kind: script - as run: SHA-256 6bbc991e42ce01573153bbc6017ee47e016f6b562398d45dc5acc096f7b9978e - as shipped: SHA-256 9c5086460503eded3f90c9522f7808ed90ef71c14b99ba923ab3602c10cc43ad - identity: identical, after dropping docstrings and comments, to the file as run with 1 declared change(s) of a machine-specific path, which now read: `'cont_hm.py'` - trailing newline: yes ~~~~~~~~~~~~ python #!/usr/bin/env python3 """wall_check.py: for alpha > 1/3, reports the largest coefficient of the forced functions H_j in cont_hm.py's twisted rows T_jD(sigma) with sigma in [1 - 2 alpha, alpha] (the rows of the paper's Lemma 8.1) and, as a positive control, in the rows with sigma < 1 - 2 alpha. Reads cont_hm.py from the working directory. Floats. Usage: wall_check.py N eta_num eta_den alpha_num alpha_den""" import sys, numpy as np argv = sys.argv[1:6] src = open('cont_hm.py').read() src = src[:src.index("if '--check' in sys.argv: sys.exit()")] sys.argv = ['cont_hm.py'] + argv + ['', '4', '2', '2'] g = {} exec(compile(src, 'cont_hm', 'exec'), g) T, rid, nc, Hgrid, h, N, ia = g['T'], g['rid'], g['nc'], g['Hgrid'], g['h'], g['N'], g['ia'] lo = int(round((1 - 2 * ia * h) * N)) dead = [(j, s) for j in Hgrid for s in Hgrid[j] if s >= lo] live = [(j, s) for j in Hgrid for s in Hgrid[j] if s < lo] md = max((np.abs(T[rid[r], nc:]).max() for r in dead), default=float('nan')) ml = max((np.abs(T[rid[r], nc:]).max() for r in live), default=float('nan')) mc = max((np.abs(T[rid[r], :nc]).max() for r in dead), default=float('nan')) inner = [(j, s) for j in Hgrid for s in Hgrid[j] if s >= lo + 3] mi = max((np.abs(T[rid[r], nc:]).max() for r in inner), default=float('nan')) print(f" inner dead rows (sigma >= 1 - 2 alpha + 3h): {len(inner)}, max |H-coefficient| = {mi:.3e}") print(f"WALL alpha = {ia*h:.4f}, 1 - 2 alpha = {1 - 2*ia*h:.4f}: dead rows {len(dead)} (sigma >= {lo*h:.4f}): max |H-coefficient| = {md:.3e}, max |c-coefficient| = {mc:.3e};" f" live rows {len(live)}: max |H-coefficient| = {ml:.3e}") ~~~~~~~~~~~~ #### file: twist_otf.py - kind: script - as run: SHA-256 2dc875fad2517d4736edf3b8128764e1dc9a58477c3db50676bc93ac68d0e58c - as shipped: SHA-256 2eeb607718b3fc935be8d9a682756b7ba6f2087102416ac262a714eb733f5f7b - identity: identical, after dropping docstrings and comments, to the file as run - trailing newline: yes ~~~~~~~~~~~~ python """twist_otf.py: the discretised problem with fully free data: the Moebius-twisted shape linear program on a grid of K, solved by column generation, with exact weak-duality certificates for positive values. A shape u is a partition of K with all parts >= m0 = eta K, with weight B(u) = prod(1/u_i)/prod(mult!); a = b (1 + phi), phi >= -1 (phi <= c_sw - 1 on composite shapes if capped); Type I rows at level thI: for every v with |v| <= thI, sum_{u >= v} B(u) c(u, v) phi(u) = 0, c(u, v) = prod_j C(m_j(u), m_j(v)); Moebius-twisted rows at level thT: the same with the sign (-1)^(len u - len v); minimise phi at the prime shape; C^- = 1 + min. The solves are floats; the certificates are exact. Imports modm_otf_lowmem2.py from its directory. Usage: twist_otf.py K m0 thI thT [c_sw | inf] [add]; the environment variables ETA, THETA_RULE (half | below_half | level:p/q) and ALPHA (thT/K) define m0, thI and thT, and the command line must match.""" import math import os import sys import time from fractions import Fraction as F import numpy as np from scipy.optimize._highspy import _core from scipy.sparse import csc_matrix import modm_otf_lowmem2 as B0 # Universe (Type I pairs, exact coefficients via coef_slice), rss_gb, partitions_min: read-only use time.time = time.monotonic # the WSL wall clock jumps class TwistRows: def __init__(self, U, thT): sizes = np.array([sum(v) for v in U.vrows], dtype=np.int64) nT = int(np.sum(sizes <= thT)) assert thT <= U.theta, "twisted level above the Type I level" assert np.all(sizes[:nT] <= thT) and np.all(sizes[nT:] > thT), "Type I rows are not ordered by |v|" assert U.vrows[0] == (), "row 0 is not v = ()" self.nT, self.thT = nT, thT self.lam = np.where(U.lens % 2 == 0, 1, -1).astype(np.int64) def price(U, T, y, c): """reduced costs d = c - A^T y over all shapes (binary64).""" d = c.copy() yI, yT = y[:U.nI], y[U.nI:U.nI + T.nT] accT = np.zeros(U.S) for r0, r1 in U.chunks(): p0, p1 = U.rowptr[r0], U.rowptr[r1] cf = U.coef_slice(p0, p1).astype(np.float64) d -= np.bincount(U.ids[p0:p1], weights=np.repeat(yI[r0:r1], U.rowlen[r0:r1]) * cf, minlength=U.S) if r0 < T.nT: r1t = min(r1, T.nT); nt = int(U.rowptr[r1t] - p0) accT += np.bincount(U.ids[p0:p0 + nt], weights=np.repeat(yT[r0:r1t], U.rowlen[r0:r1t]) * cf[:nt], minlength=U.S) d -= T.lam * accT return d def restricted_matrix(U, T, cidx): """psi-form matrix (Type I rows, then twisted rows) x working columns cidx (CSC).""" colmap = np.full(U.S, -1, dtype=np.int64) colmap[cidx] = np.arange(len(cidx)) R_, C_, V_ = [], [], [] for r0, r1 in U.chunks(): p0, p1 = U.rowptr[r0], U.rowptr[r1] ids = U.ids[p0:p1] cm = colmap[ids] sel = cm >= 0 if not sel.any(): continue rows = np.repeat(np.arange(r0, r1, dtype=np.int64), U.rowlen[r0:r1]) cf = U.coef_slice(p0, p1).astype(np.float64) R_.append(rows[sel]); C_.append(cm[sel]); V_.append(cf[sel]) if r0 < T.nT: r1t = min(r1, T.nT); nt = int(U.rowptr[r1t] - p0) st = sel[:nt] R_.append(U.nI + rows[:nt][st]); C_.append(cm[:nt][st]); V_.append(cf[:nt][st] * T.lam[ids[:nt][st]]) if R_: R_ = np.concatenate(R_); C_ = np.concatenate(C_); V_ = np.concatenate(V_) else: R_ = np.zeros(0, dtype=np.int64); C_ = np.zeros(0, dtype=np.int64); V_ = np.zeros(0) return csc_matrix((V_, (R_, C_)), shape=(U.nI + T.nT, len(cidx))) def _accumulate_exact(U, Ylist, nrows_used, L): """exact per-shape sums sum_{rows r < nrows_used} c(u, v_r) Y_r as int64 limbs of L bits (returns lists of Python ints per limb).""" nbits = max([abs(v).bit_length() for v in Ylist] + [1]) nl = nbits // L + 1 mask = (1 << L) - 1 limbs = [] for k in range(nl): if k < nl - 1: limbs.append(np.array([(v >> (k * L)) & mask for v in Ylist], dtype=np.int64)) # 0 <= limb < 2^L else: limbs.append(np.array([v >> (k * L) for v in Ylist], dtype=np.int64)) # signed top limb, |.| < 2^L for lb in limbs: assert len(lb) == 0 or int(np.max(np.abs(lb))) < 2 ** L W = [np.zeros(U.S, dtype=np.int64) for _ in range(nl)] for r0, r1 in U.chunks(4_000_000): if r0 >= nrows_used: break r1u = min(r1, nrows_used) p0, p1 = U.rowptr[r0], U.rowptr[r1u] cf = U.coef_slice(p0, p1) idc = U.ids[p0:p1] for k in range(nl): np.add.at(W[k], idc, np.repeat(limbs[k][r0:r1u], U.rowlen[r0:r1u]) * cf) return [w.tolist() for w in W], nl def certificate(U, T, y, c_sw, log, p=None): """exact weak-duality lower bound for C- over ALL shapes (docstring).""" t2 = time.time() K, S = U.K, U.S if p is None: p = int(os.environ.get("CERT_P", "50")) Yf = np.asarray(y, dtype=np.float64) Y = [int(v) for v in np.rint(np.ldexp(Yf, p)).tolist()] maxlen = int(U.lens.max()) L = 60 - maxlen # |limb| < 2^L and sum_v c(u, v) <= 2^{len u}: every partial sum < 2^60 assert L >= 8, "shapes too long for int64 limbs" WI, nlI = _accumulate_exact(U, Y[:U.nI], U.nI, L) WT, nlT = _accumulate_exact(U, Y[U.nI:U.nI + T.nT], T.nT, L) lam = T.lam.tolist() Z = [] for i in range(S): wi = sum(int(WI[k][i]) << (k * L) for k in range(nlI)) wt = sum(int(WT[k][i]) << (k * L) for k in range(nlT)) Z.append((K << p) * (i == U.P) - (wi + lam[i] * wt)) delta = 0 if Z[U.P] < 0: delta = -Z[U.P] Z = [z + delta for z in Z] # = lowering Y on the v = () Type I row (coefficient 1 on every shape) zf = np.array([math.ldexp(float(z), -p) for z in Z]) # float(z): one rounding; ldexp exact here comp = U.lens >= 2 assert np.all(comp[zf < 0]), "negative reduced cost on the prime after repair" if math.isinf(c_sw): if np.any(zf < 0): log(f" CERT: none (uncapped and {int(np.sum(zf < 0))} composite reduced costs < 0; {time.time() - t2:.0f} s)", flush=True) return -math.inf terms = -zf * U.Bf else: capm1 = c_sw - 1.0 assert capm1 == math.floor(capm1) terms = np.where(zf >= 0, -zf * U.Bf, capm1 * zf * U.Bf) tot = math.fsum(terms.tolist()) err = math.fsum(np.abs(terms).tolist()) * 2.0 ** -48 + 2.0 ** -1000 cert = float(np.nextafter(1.0 + tot - err, -np.inf)) log(f" CERT: certified C- >= {cert:.9f} (all {S} shapes; exact duals at 2^-{p}; {len(Y)} rows; repair {delta / 2.0 ** p:.2e}; " f"rounding allowance {err:.1e}; {time.time() - t2:.0f} s)", flush=True) return cert def new_cols_matrix(U, T, cols): A = restricted_matrix(U, T, cols).tocsc() A.sort_indices() return A.indptr[:-1].astype(np.int32), A.indices.astype(np.int32), A.data.astype(np.float64), A.nnz def main(): K = int(sys.argv[1]); m0 = int(sys.argv[2]); thI = int(sys.argv[3]); thT = int(sys.argv[4]) c_sw = float(sys.argv[5]) if len(sys.argv) > 5 else 100.0 add = int(sys.argv[6]) if len(sys.argv) > 6 else 4000 BCUT = float(os.environ.get("BCUT", "1e-11")) MAXROUNDS = int(os.environ.get("MAXROUNDS", "200")) RSS_GB = float(os.environ.get("RSS_GB", "2.9")) STRATEGY = int(os.environ.get("STRATEGY", "1")) STOP_AT_POS = os.environ.get("STOP_AT_POS", "0") == "1" STAB = float(os.environ.get("STAB", "0.7")) LP = os.environ.get("LP", "auto") AUTO_SWITCH = float(os.environ.get("AUTO_SWITCH", "1e-4")) ZERO_TOL = float(os.environ.get("ZERO_TOL", "1e-5")) SUPP_TOLS = [float(v) for v in os.environ.get("SUPP_TOLS", "1e-2,1e-4,1e-6").split(",")] RES_TOL = float(os.environ.get("RES_TOL", "1e-9")) OUTPFX = os.environ.get("OUTPFX", "tw") capname = "inf" if math.isinf(c_sw) else f"{c_sw:g}" tag = f"TWIST K={K} m0={m0} thI={thI} thT={thT} cap={capname}" stem = f"{OUTPFX}_K{K}_m0{m0}_thI{thI}_thT{thT}_cap{capname}" log = print ts = time.time() log(f"{tag}: start", flush=True) # parameters DERIVED from the model's definition, checked against the command line, printed eta = F(os.environ["ETA"]); alpha = F(os.environ["ALPHA"]); rule = os.environ.get("THETA_RULE", "half") m0_def = eta * K if rule == "half": thI_def = F(K, 2); thI_txt = f"theta = K/2 = {thI_def} (Type I at level exactly 1/2, inclusive: the law's convention)" elif rule == "below_half": thI_def = F((K - 1) // 2); thI_txt = f"largest theta with theta/K < 1/2 -> {thI_def}" elif rule.startswith("level:"): lev = F(rule.split(":", 1)[1]); thI_def = lev * K; thI_txt = f"fixed level {lev} -> theta = {thI_def}" else: raise SystemExit(f"unknown THETA_RULE {rule}") thT_def = alpha * K for name, val_, dfn in (("m0", m0, m0_def), ("thI", thI, thI_def), ("thT", thT, thT_def)): if dfn.denominator != 1 or int(dfn) != val_: raise SystemExit(f"PARAMS MISMATCH: command line {name}={val_}; model definition gives {dfn} " f"(eta={eta}, alpha={alpha}, THETA_RULE={rule})") log(f"PARAMS (derived from the model's definition, checked): K = {K}; eta = {eta} -> m0 = {m0}; Type I: {thI_txt} -> thI/K = " f"{thI}/{K} = {thI / K:.6f}; Moebius-twisted level alpha = {alpha} -> thT = {thT} (thT/K = {thT / K:.6f}; alpha " f"{'>' if alpha > F(1, 3) else '<=' } 1/3); caps: {'none' if math.isinf(c_sw) else f'x = 1 + phi <= {c_sw:g} on composites'}; " f"phi >= -1; objective min phi(prime)", flush=True) U = B0.Universe(K, m0, thI, log=log) T = TwistRows(U, thT) S = U.S nrows = U.nI + T.nT keep = U.Bf >= BCUT lo = -U.Bf hi = np.where(U.lens >= 2, (c_sw - 1.0) * U.Bf, np.inf) if not math.isinf(c_sw) else np.full(S, np.inf) c = np.zeros(S); c[U.P] = float(K) sfac = np.sqrt(U.Bf) h = _core._Highs() for k, v in (("output_flag", False), ("presolve", "off"), ("solver", "simplex"), ("primal_feasibility_tolerance", 1e-10), ("dual_feasibility_tolerance", 1e-10), ("simplex_strategy", STRATEGY)): h.setOptionValue(k, v) INF = h.getInfinity() h.addRows(nrows, np.zeros(nrows), np.zeros(nrows), 0, np.zeros(nrows, dtype=np.int32), np.zeros(0, dtype=np.int32), np.zeros(0)) work = np.zeros(S, dtype=bool) order = [] def append(cols): cols = np.asarray(cols, dtype=np.int64) st, ind, val, nnz = new_cols_matrix(U, T, cols) sc = sfac[cols] val = val * np.repeat(sc, np.diff(np.append(st, nnz))) hh = np.where(np.isinf(hi[cols]), INF, hi[cols] / sc) z = np.zeros(len(cols)) h.addCols(len(cols), c[cols] * sc, z, hh, nnz, st, ind, val) # xi+ in [0, hi/s] h.addCols(len(cols), -c[cols] * sc, z, -lo[cols] / sc, nnz, st, ind, -val) # xi- in [0, B/s] work[cols] = True order.append(cols) init = np.nonzero(keep & (U.lens <= 3))[0] init = np.union1d(init, [U.P]) append(init) log(f"{tag}: {S} shapes ({int(keep.sum())} with B >= {BCUT:g}), {U.nI} Type I rows + {T.nT} twisted rows; RSS {B0.rss_gb():.2f} GB; " f"{time.time() - ts:.0f} s", flush=True) best_cert = None val = None lb_best = [-np.inf]; y_best = [None]; certified = [None] def impact(dd): im = np.where(dd > 0, dd * (-lo), np.where(dd < 0, -dd * np.where(np.isinf(hi), 1e30, hi), 0.0)) im[work] = 0.0; im[~keep] = 0.0 return im def float_lb(dd): dl = max(0.0, -float(dd[U.P])) z = dd + dl if math.isinf(c_sw) and np.any(z[U.lens >= 2] < 0): return -np.inf return 1.0 + float(np.sum(np.where(z >= 0, z * lo, z * np.where(np.isinf(hi), 0.0, hi)))) def save_fake(cols, phis, how): np.savez(f"{stem}_fake.npz", ids=np.asarray(cols, dtype=np.int64), phi=np.asarray(phis, dtype=np.float64), K=K, m0=m0, thI=thI, thT=thT, cap=c_sw, how=how) for it in range(1, MAXROUNDS + 1): if B0.rss_gb() > RSS_GB: log(f" ABORT: RSS {B0.rss_gb():.2f} GB above {RSS_GB} GB", flush=True) sys.exit(3) t1 = time.time() use_ipm = LP == "ipm" or (LP == "auto" and (val is None or val > AUTO_SWITCH)) def solve(ipm): h.clearSolver() if ipm: h.setOptionValue("solver", "ipm"); h.setOptionValue("run_crossover", "off") else: h.setOptionValue("solver", "simplex"); h.setOptionValue("simplex_strategy", STRATEGY) h.run() return h.getModelStatus() st = solve(use_ipm) if use_ipm and h.modelStatusToString(st) != "Optimal": log(f" round {it}: IPM status {h.modelStatusToString(st)}; dual simplex instead", flush=True) use_ipm = False st = solve(False) if h.modelStatusToString(st) != "Optimal": log(f" round {it}: model status {h.modelStatusToString(st)}; retrying with the dual simplex from scratch", flush=True) h.clearSolver(); h.setOptionValue("simplex_strategy", 1); h.run(); st = h.getModelStatus() h.setOptionValue("simplex_strategy", STRATEGY) if h.modelStatusToString(st) != "Optimal": log(f" round {it}: still {h.modelStatusToString(st)}; stop", flush=True) log(f"RESULT {tag}: UNDECIDED (solver status {h.modelStatusToString(st)} at round {it}; best certified " f"{best_cert if best_cert is not None else float('nan')}) [{time.time() - ts:.0f} s]", flush=True) sys.exit(1) sol = h.getSolution() y = np.asarray(sol.row_dual, dtype=np.float64) xall = np.asarray(sol.col_value, dtype=np.float64) cdall = np.asarray(sol.col_dual, dtype=np.float64) xs, cds, ords = [], [], [] pos = 0 for blk in order: nb = len(blk) sc = sfac[blk] xs.append((xall[pos:pos + nb] - xall[pos + nb:pos + 2 * nb]) * sc) cds.append(cdall[pos:pos + nb] / sc); cds.append(-cdall[pos + nb:pos + 2 * nb] / sc) ords.append(blk); ords.append(blk) pos += 2 * nb ord_ = np.concatenate(order) x = np.concatenate(xs) val = 1.0 + float(np.dot(c[ord_], x)) d = price(U, T, y, c) cd = np.concatenate(cds); ordd = np.concatenate(ords) dchk = float(np.max(np.abs(cd - d[ordd]))) if len(ordd) else 0.0 lb = float_lb(d) if lb > lb_best[0]: lb_best[:] = [lb]; y_best[:] = [y.copy()] dp = d; tagp = "y" if STAB > 0 and y_best[0] is not None and y_best[0] is not y: ys = STAB * y_best[0] + (1.0 - STAB) * y ds = price(U, T, ys, c) lbs = float_lb(ds) if lbs > lb_best[0]: lb_best[:] = [lbs]; y_best[:] = [ys] imp_s = impact(ds) if int((imp_s > 1e-9).sum()) > 0: dp = ds; tagp = f"smoothed({STAB:g})" imp = impact(dp) nbad = int((imp > 1e-9).sum()) if nbad == 0 and dp is not d: imp = impact(d); nbad = int((imp > 1e-9).sum()); tagp = "y (mispricing fallback)" phi = x / U.Bf[ord_] comp = U.lens[ord_] >= 2 log(f" round {it}: {len(ord_)} columns, C- (restricted) = {val:.9f}, float Lagrangian bound {lb:.6f} (best {lb_best[0]:.6f}); " f"improving outside {nbad} by {tagp}, max impact {imp.max():.2e}; phi in [{phi.min():.6f}, " f"{phi[comp].max() if comp.any() else 0:.6f}]; dual check {dchk:.1e}; {'IPM' if use_ipm else 'simplex'}; " f"round {time.time() - t1:.0f} s; RSS {B0.rss_gb():.2f} GB", flush=True) if dchk > 1e-6: log(" WARNING: row-dual sign/convention check failed; pricing may be wrong", flush=True) if val <= ZERO_TOL and use_ipm and RES_TOL > 0: Aall = restricted_matrix(U, T, ord_) res = Aall @ x absA = abs(Aall) @ np.abs(x) rmax = float(np.max(np.abs(res))) if len(res) else 0.0 rrel = float(np.max(np.abs(res) / np.maximum(absA, 1e-300))) if len(res) else 0.0 blo = float(np.min(phi)); bhi = float(np.max(phi[U.lens[ord_] >= 2])) ok = rmax <= RES_TOL and blo >= -1.0 - 1e-6 and bhi <= c_sw - 1.0 + 1e-6 and val <= 1e-7 log(f" round {it}: interior point check: value {val:.2e}, max |row residual| {rmax:.2e} (max relative {rrel:.2e}) over " f"{Aall.shape[0]} rows, phi in [{blo:.9f}, {bhi:.9f}] -> {'accepted' if ok else 'not accepted'}", flush=True) del Aall, absA if ok: save_fake(ord_, phi, f"interior point, round {it}") log(f"RESULT {tag}: ZERO (an interior fake on {len(ord_)} working columns at round {it}: max |row residual| {rmax:.2e}, " f"phi in [{blo:.6f}, {bhi:.6f}]; honest to solver tolerance; saved {stem}_fake.npz) [{time.time() - ts:.0f} s]", flush=True) return if val <= ZERO_TOL and use_ipm: last_n = -1 for stol in SUPP_TOLS: supp = np.union1d(ord_[np.abs(phi) > stol], [U.P]) if len(supp) == last_n: continue last_n = len(supp) log(f" round {it}: IPM value {val:.2e}; support-reduced simplex (|phi| > {stol:g}) on {len(supp)} of {len(ord_)} shapes", flush=True) hr = _core._Highs() for k_, v_ in (("output_flag", False), ("presolve", "off"), ("solver", "simplex"), ("primal_feasibility_tolerance", 1e-10), ("dual_feasibility_tolerance", 1e-10), ("simplex_strategy", 1)): hr.setOptionValue(k_, v_) hr.addRows(nrows, np.zeros(nrows), np.zeros(nrows), 0, np.zeros(nrows, dtype=np.int32), np.zeros(0, dtype=np.int32), np.zeros(0)) st_, ind_, val_, nnz_ = new_cols_matrix(U, T, supp) sc_ = sfac[supp] val_ = val_ * np.repeat(sc_, np.diff(np.append(st_, nnz_))) hh_ = np.where(np.isinf(hi[supp]), INF, hi[supp] / sc_) z_ = np.zeros(len(supp)) hr.addCols(len(supp), c[supp] * sc_, z_, hh_, nnz_, st_, ind_, val_) hr.addCols(len(supp), -c[supp] * sc_, z_, -lo[supp] / sc_, nnz_, st_, ind_, -val_) hr.run() str_ = hr.modelStatusToString(hr.getModelStatus()) if str_ == "Optimal": xr = np.asarray(hr.getSolution().col_value, dtype=np.float64) n_ = len(supp) psir = (xr[:n_] - xr[n_:]) * sc_ valr = 1.0 + float(np.dot(c[supp], psir)) phir = psir / U.Bf[supp] compr = U.lens[supp] >= 2 log(f" round {it}: support-reduced simplex value {valr:.9f}; phi in [{phir.min():.6f}, {phir[compr].max():.6f}]", flush=True) if valr <= 1e-7: save_fake(supp, phir, f"support-reduced simplex |phi| > {stol:g}, round {it}") log(f"RESULT {tag}: ZERO (a fake within {len(supp)} support columns (|phi| > {stol:g}) of the round-{it} interior " f"solution; min phi {phir.min():.6f}, max composite phi {phir[compr].max():.6f}; honest to solver tolerance; " f"saved {stem}_fake.npz) [{time.time() - ts:.0f} s]", flush=True) return else: log(f" round {it}: support-reduced simplex status {str_}", flush=True) del hr log(f" round {it}: confirming with the full dual simplex", flush=True) st = solve(False) sol = h.getSolution() xall = np.asarray(sol.col_value, dtype=np.float64) xs = [] pos = 0 for blk in order: nb = len(blk) xs.append((xall[pos:pos + nb] - xall[pos + nb:pos + 2 * nb]) * sfac[blk]) pos += 2 * nb x = np.concatenate(xs) val = 1.0 + float(np.dot(c[ord_], x)) phi = x / U.Bf[ord_] log(f" round {it}: dual simplex value {val:.9f} (status {h.modelStatusToString(st)})", flush=True) if val <= 1e-7: save_fake(ord_, phi, f"simplex, round {it}") log(f"RESULT {tag}: ZERO (a fake within {len(ord_)} working columns; min phi {phi.min():.6f}, max composite phi " f"{phi[comp].max():.6f}; honest to solver tolerance; saved {stem}_fake.npz) [{time.time() - ts:.0f} s]", flush=True) return if lb_best[0] > 1e-6 and y_best[0] is not certified[0]: certified[0] = y_best[0] np.save(f"{stem}_ybest.npy", y_best[0]) cert = certificate(U, T, y_best[0], c_sw, log) if best_cert is None or cert > best_cert: best_cert = cert if STOP_AT_POS and cert > 1e-6: log(f"RESULT {tag}: POSITIVE, C- in [{best_cert:.9f}, {val:.9f}] (certified lower end over all {S} shapes; restricted upper " f"end; stopped at the first positive certificate, round {it}; {time.time() - ts:.0f} s)", flush=True) return if nbad == 0: if certified[0] is not y: np.save(f"{stem}_yfinal.npy", y) cert = certificate(U, T, y, c_sw, log) if best_cert is None or cert > best_cert: best_cert = cert break append(np.argsort(-imp)[:min(add, nbad)]) log(f"RESULT {tag}: C- in [{best_cert if best_cert is not None else float('nan'):.9f}, {val:.9f}] (certified lower end, restricted " f"upper end; {it} rounds; {time.time() - ts:.0f} s)", flush=True) if __name__ == "__main__": main() ~~~~~~~~~~~~ #### file: modm_otf_lowmem2.py - kind: script - as run: SHA-256 2b6c25c17ca07b6155414e40092bb9d9ad651d7bdcb46a2bfa2afded6a8c33e6 - as shipped: SHA-256 92cbd0366ff6d251630b450da919482ba43a87491ede1aa22a51a9ac58d64a51 - identity: identical, after dropping docstrings and comments, to the file as run - trailing newline: yes ~~~~~~~~~~~~ python """modm_otf_lowmem2.py: the shape universe and Type I machinery used by twist_otf.py and agg_lp5.py: partitions of K with parts >= m0, the coefficients c(u, v) stored compactly with an exact overflow table, column generation with pricing passes, and an exact multi-limb weak-duality certificate. (Its own command line runs a residue-class model that the paper does not use.)""" import math import os import resource import sys import time from collections import Counter import numpy as np from scipy.optimize import linprog from scipy.sparse import csc_matrix time.time = time.monotonic # the WSL wall clock jumps def rss_gb(): with open("/proc/self/status") as f: for line in f: if line.startswith("VmRSS:"): return int(line.split()[1]) / 1048576.0 return 0.0 def partitions_min(n, m0, maxpart=None): if maxpart is None: maxpart = n if n == 0: yield () return for k in range(min(n, maxpart), m0 - 1, -1): rest = n - k if rest == 0 or rest >= m0: for r in partitions_min(rest, m0, k): yield (k,) + r def keys_of(parts_list, R, m0): """64-bit random-projection hash of each multiset (wraparound int64 sum of R[p - m0]).""" out = np.zeros(len(parts_list), dtype=np.int64) if not parts_list: return out maxlen = max(len(p) for p in parts_list) CH = 200000 for c0 in range(0, len(parts_list), CH): chunk = parts_list[c0:c0 + CH] P = np.zeros((len(chunk), maxlen), dtype=np.int64) msk = np.zeros((len(chunk), maxlen), dtype=bool) for i, p in enumerate(chunk): P[i, :len(p)] = p msk[i, :len(p)] = True vals = np.where(msk, R[np.clip(P - m0, 0, len(R) - 1)], 0) with np.errstate(over="ignore"): out[c0:c0 + len(chunk)] = vals.sum(axis=1) return out class Universe: def __init__(self, K, m0, theta, seed=12345, log=print): t0 = time.time() self.K, self.m0, self.theta = K, m0, theta L = K - m0 + 1 rng = np.random.default_rng(seed) self.R = rng.integers(1, 2**62, size=L, dtype=np.int64) | 1 self.shapes = list(partitions_min(K, m0)) S = self.S = len(self.shapes) self.lens = np.array([len(u) for u in self.shapes], dtype=np.int16) self.mins = np.array([u[-1] for u in self.shapes], dtype=np.int16) keys = keys_of(self.shapes, self.R, m0) self.order = np.argsort(keys, kind="stable") self.sk = keys[self.order] if np.any(self.sk[1:] == self.sk[:-1]): raise RuntimeError("hash collision among shapes; change seed") self.P = self.shapes.index((K,)) # exact integer denominators and binary64 masses self.Bf = np.empty(S) for i, u in enumerate(self.shapes): d = math.prod(u) for mlt in Counter(u).values(): d *= math.factorial(mlt) self.Bf[i] = 1.0 / float(d) del self.shapes # free the tuples before the pair build (P, lens, mins, keys, Bf are kept) log(f" universe: {S} shapes (parts >= {m0}), {time.time() - t0:.0f} s, RSS {rss_gb():.2f} GB", flush=True) # Type I pairs, stored per row v: ids (int32) and c(u, v) (int32), preallocated (counts by a partition-count DP) pc = np.zeros(K + 1, dtype=np.int64) # pc[n] = number of partitions of n with parts >= m0 pc[0] = 1 for part in range(m0, K + 1): for n in range(part, K + 1): pc[n] += pc[n - part] ks = [k for k in range(0, theta + 1) if (k == 0 or k >= m0) and (K - k == 0 or K - k >= m0)] total = int(sum(pc[k] * pc[K - k] for k in ks)) nrows = int(sum(pc[k] for k in ks)) self.ids = np.empty(total, dtype=np.int32) self.coef = np.empty(total, dtype=np.uint16) ov_i, ov_v = [], [] self.rowlen = np.empty(nrows, dtype=np.int64) self.vrows = [] q = 0 for k in ks: n = K - k V = list(partitions_min(k, m0)) Wp = list(partitions_min(n, m0)) assert len(V) == pc[k] and len(Wp) == pc[n] wkeys = keys_of(Wp, self.R, m0) if k > 0: Wm = np.zeros((len(Wp), k - m0 + 1), dtype=np.int64) for i, w in enumerate(Wp): for part in w: if part <= k: Wm[i, part - m0] += 1 vkeys = keys_of(V, self.R, m0) if k > 0 else np.zeros(1, dtype=np.int64) for v, vkey in zip(V, vkeys): with np.errstate(over="ignore"): uk = wkeys + vkey pos = np.searchsorted(self.sk, uk) pos = np.minimum(pos, S - 1) if not np.all(self.sk[pos] == uk): raise RuntimeError(f"lookup failure for v = {v}") coef = np.ones(len(Wp), dtype=np.int64) for part, mv in Counter(v).items(): mw = Wm[:, part - m0] cc = np.ones(len(Wp), dtype=np.int64) for i in range(1, mv + 1): cc = cc * (mw + i) // i coef *= cc assert coef.max() < 2**62 nw = len(Wp) self.ids[q:q + nw] = self.order[pos] big = coef >= 65535 if big.any(): bi = np.nonzero(big)[0] ov_i.append(q + bi); ov_v.append(coef[bi].copy()) self.coef[q:q + nw] = np.minimum(coef, 65535).astype(np.uint16) self.rowlen[len(self.vrows)] = nw self.vrows.append(v) q += nw del Wp, wkeys assert q == total and len(self.vrows) == nrows self.ov_idx = np.concatenate(ov_i).astype(np.int64) if ov_i else np.zeros(0, dtype=np.int64) self.ov_val = np.concatenate(ov_v).astype(np.int64) if ov_v else np.zeros(0, dtype=np.int64) self.rowptr = np.concatenate([[0], np.cumsum(self.rowlen)]) self.nI = len(self.vrows) self.vrow0 = self.vrows.index(()) log(f" Type I: {self.nI} rows, {total} pairs ({(self.ids.nbytes + self.coef.nbytes) / 2**30:.2f} GB stored; {len(self.ov_idx)} coefficients >= 65535 in the overflow table), " f"{time.time() - t0:.0f} s, RSS {rss_gb():.2f} GB", flush=True) def coef_slice(self, p0, p1): """exact int64 coefficients of pairs p0..p1-1 (uint16 storage patched from the overflow table).""" c = self.coef[p0:p1].astype(np.int64) if len(self.ov_idx): a = np.searchsorted(self.ov_idx, p0); b = np.searchsorted(self.ov_idx, p1) if b > a: c[self.ov_idx[a:b] - p0] = self.ov_val[a:b] return c def chunks(self, target=8_000_000): """row ranges [r0, r1) with about `target` pairs each.""" r0 = 0 while r0 < self.nI: r1 = int(np.searchsorted(self.rowptr, self.rowptr[r0] + target, side="right")) - 1 r1 = max(r1, r0 + 1) r1 = min(r1, self.nI) yield r0, r1 r0 = r1 class OracleRows: def __init__(self, U, m, t0): self.U, self.m, self.t0 = U, m, t0 self.keys = [(t, r) for t in range(U.m0, t0 + 1) for r in range(1, m)] self.lmod = (U.lens % m).astype(np.int64) def weight(self, r): """per shape: 1[len = r mod m] - 1[len = 0 mod m].""" return (self.lmod == r).astype(np.int64) - (self.lmod == 0).astype(np.int64) def ATy(self, yM, integer=False): """per shape: sum_{t = m0..min(t0, min u)} sum_r yM[(t, r)] w_r(len u). integer=True: yM are Python ints, exact (object).""" U = self.U nt = self.t0 - U.m0 + 1 if integer: G = [[0] * self.m for _ in range(nt + 1)] else: G = np.zeros((nt + 1, self.m)) for it in range(nt): for cc in range(self.m): acc = 0 if integer else 0.0 for r in range(1, self.m): wv = (1 if cc == r else 0) - (1 if cc == 0 else 0) if wv: acc += wv * yM[it * (self.m - 1) + (r - 1)] G[it + 1][cc] = G[it][cc] + acc tt = np.clip(U.mins.astype(np.int64), U.m0 - 1, self.t0) - (U.m0 - 1) if integer: flat = np.array([G[a][b] for a in range(nt + 1) for b in range(self.m)], dtype=object) return flat[tt * self.m + self.lmod] return G[tt, self.lmod] def price(U, O, y, c): """reduced costs d = c - A^T y over all shapes (binary64).""" d = c.copy() yI, yM = y[:U.nI], y[U.nI:] for r0, r1 in U.chunks(): p0, p1 = U.rowptr[r0], U.rowptr[r1] w = np.repeat(yI[r0:r1], U.rowlen[r0:r1]) * U.coef_slice(p0, p1) d -= np.bincount(U.ids[p0:p1], weights=w, minlength=U.S) d -= O.ATy(yM) return d def restricted_matrix(U, O, cidx): """psi-form matrix (all Type I rows + oracle rows) x working columns cidx (CSC).""" S = U.S colmap = np.full(S, -1, dtype=np.int64) colmap[cidx] = np.arange(len(cidx)) R_, C_, V_ = [], [], [] for r0, r1 in U.chunks(): p0, p1 = U.rowptr[r0], U.rowptr[r1] ids = U.ids[p0:p1] cm = colmap[ids] sel = cm >= 0 if not sel.any(): continue rows = np.repeat(np.arange(r0, r1, dtype=np.int64), U.rowlen[r0:r1])[sel] R_.append(rows); C_.append(cm[sel]); V_.append(U.coef_slice(p0, p1)[sel].astype(np.float64)) nI = U.nI for k, (t, r) in enumerate(O.keys): w = O.weight(r)[cidx] rough = U.mins[cidx] >= t sel = np.nonzero(rough & (w != 0))[0] R_.append(np.full(len(sel), nI + k, dtype=np.int64)); C_.append(sel.astype(np.int64)); V_.append(w[sel].astype(np.float64)) R_ = np.concatenate(R_); C_ = np.concatenate(C_); V_ = np.concatenate(V_) return csc_matrix((V_, (R_, C_)), shape=(nI + len(O.keys), len(cidx))) def certificate(U, O, y, c_sw, log, p=None): """exact weak-duality lower bound for C- over ALL shapes (see the docstring). Y = round(y 2^p) as Python ints; the Type I part of A^T Y is accumulated exactly in int64 limbs of L = 60 - max len u bits (0 <= lower limbs < 2^L, signed top limb |.| < 2^L), so every partial sum is below 2^L * 2^{len u} <= 2^60, then recombined as Python ints.""" t2 = time.time() K, S = U.K, U.S Yf = np.asarray(y, dtype=np.float64) if p is None: ymax = float(np.max(np.abs(Yf))) if len(Yf) else 1.0 p = min(50, 31 + (61 - int(U.lens.max())) - int(math.ceil(math.log2(ymax + 1.0))) - 1) Y = [int(v) for v in np.rint(np.ldexp(Yf, p)).tolist()] maxlen = int(U.lens.max()) L = 60 - maxlen # limb width: |limb| < 2^L and sum_v c(u, v) <= 2^{len u} keep sums < 2^60 assert L >= 8, "shapes too long for int64 limbs" YI = Y[:U.nI] nbits = max([abs(v).bit_length() for v in YI] + [1]) nl = nbits // L + 1 mask = (1 << L) - 1 limbs = [] for k in range(nl): if k < nl - 1: limbs.append(np.array([(v >> (k * L)) & mask for v in YI], dtype=np.int64)) # 0 <= limb < 2^L else: limbs.append(np.array([v >> (k * L) for v in YI], dtype=np.int64)) # signed top limb, |.| < 2^L for lb in limbs: assert int(np.max(np.abs(lb))) < 2 ** L W = [np.zeros(S, dtype=np.int64) for _ in range(nl)] for r0, r1 in U.chunks(4_000_000): p0, p1 = U.rowptr[r0], U.rowptr[r1] cf = U.coef_slice(p0, p1) idc = U.ids[p0:p1] for k in range(nl): np.add.at(W[k], idc, np.repeat(limbs[k][r0:r1], U.rowlen[r0:r1]) * cf) WM = O.ATy(Y[U.nI:], integer=True) Wl = [w.tolist() for w in W] Z = [] for i in range(S): wi = sum(int(Wl[k][i]) << (k * L) for k in range(nl)) + int(WM[i]) Z.append((K << p) * (i == U.P) - wi) delta = 0 if Z[U.P] < 0: delta = -Z[U.P] Z = [z + delta for z in Z] # = moving y on the v = () row (coefficient 1 on every shape) capm1 = c_sw - 1.0 assert capm1 == math.floor(capm1) zf = np.array([math.ldexp(float(z), -p) for z in Z]) # float(z): one rounding; ldexp exact (no underflow at these sizes) terms = np.where(zf >= 0, -zf * U.Bf, capm1 * zf * U.Bf) comp = U.lens >= 2 assert np.all(comp[zf < 0]), "negative reduced cost on the prime after repair" tot = math.fsum(terms.tolist()) err = math.fsum(np.abs(terms).tolist()) * 2.0 ** -48 + 2.0 ** -1000 cert = float(np.nextafter(1.0 + tot - err, -np.inf)) log(f" CERT: certified C- >= {cert:.9f} (all {S} shapes; exact duals at 2^-{p}; repair {delta / 2.0 ** p:.2e}; rounding allowance " f"{err:.1e}; {time.time() - t2:.0f} s)", flush=True) return cert def main(): K = int(sys.argv[1]); m0 = int(sys.argv[2]); th = int(sys.argv[3]); m = int(sys.argv[4]); t0 = int(sys.argv[5]) c_sw = float(sys.argv[6]) if len(sys.argv) > 6 else 4.0 add = int(sys.argv[7]) if len(sys.argv) > 7 else 4000 BCUT = float(os.environ.get("BCUT", "1e-11")) METHOD = os.environ.get("METHOD", "highs-ipm") CERT_FROM = int(os.environ.get("CERT_FROM", "2")) MAXROUNDS = int(os.environ.get("MAXROUNDS", "60")) RSS_GB = float(os.environ.get("RSS_GB", "2.9")) tag = f"OTF K={K} m0={m0} theta={th} m={m} t0={t0} caps={c_sw}" log = print ts = time.time() log(f"{tag}: start", flush=True) U = Universe(K, m0, th, log=log) O = OracleRows(U, m, t0) S = U.S keep = U.Bf >= BCUT lo = -U.Bf hi = np.where(U.lens >= 2, (c_sw - 1.0) * U.Bf, np.inf) c = np.zeros(S); c[U.P] = float(K) work = np.zeros(S, dtype=bool); work[U.P] = True work |= keep & (U.lens <= 3) log(f"{tag}: {S} shapes ({int(keep.sum())} with B >= {BCUT:g}), {U.nI} Type I rows + {len(O.keys)} oracle rows; " f"RSS {rss_gb():.2f} GB; {time.time() - ts:.0f} s", flush=True) best_cert = None val = None for it in range(1, MAXROUNDS + 1): if rss_gb() > RSS_GB: log(f" ABORT: RSS {rss_gb():.2f} GB above {RSS_GB} GB", flush=True) sys.exit(3) cidx = np.nonzero(work)[0] t1 = time.time() Aw = restricted_matrix(U, O, cidx) meths = [METHOD] + [mm for mm in ("highs-ipm", "highs-ds", "highs") if mm != METHOD] r = None for meth in meths: opts = {"presolve": meth == "highs", "primal_feasibility_tolerance": 1e-10, "dual_feasibility_tolerance": 1e-10} r = linprog(c[cidx], A_eq=Aw, b_eq=np.zeros(Aw.shape[0]), bounds=np.column_stack([lo[cidx], hi[cidx]]), method=meth, options=opts) if r.status == 0: break log(f" round {it}: {meth} status {r.status} ({str(r.message)[:80]}); next method", flush=True) if r is None or r.status != 0: log(" all methods failed", flush=True) sys.exit(1) val = 1.0 + r.fun y = np.asarray(r.eqlin.marginals, dtype=np.float64) d = price(U, O, y, c) imp = np.where(d > 0, d * (-lo), np.where(d < 0, -d * np.where(np.isinf(hi), 1e30, hi), 0.0)) imp[work] = 0.0; imp[~keep] = 0.0 nbad = int((imp > 1e-9).sum()) phi = r.x / U.Bf[cidx] comp = U.lens[cidx] >= 2 log(f" round {it}: {len(cidx)} columns, C- (restricted) = {val:.9f}, improving outside {nbad}, max impact {imp.max():.2e}; " f"phi in [{phi.min():.6f}, {phi[comp].max() if comp.any() else 0:.6f}]; LP {time.time() - t1:.0f} s; RSS {rss_gb():.2f} GB", flush=True) if val <= 1e-7: log(f"RESULT {tag}: ZERO (a fake within {len(cidx)} working columns; min phi {phi.min():.6f}, max composite phi " f"{phi[comp].max():.6f}; honest to solver tolerance) [{time.time() - ts:.0f} s]", flush=True) return if it >= CERT_FROM: np.save(f"otf_y_K{K}_m0{m0}_m{m}_t{t0}_r{it}.npy", y) cert = certificate(U, O, y, c_sw, log) if best_cert is None or cert > best_cert: best_cert = cert if nbad == 0: break work[np.argsort(-imp)[:min(add, nbad)]] = True log(f"RESULT {tag}: C- in [{best_cert if best_cert is not None else float('nan'):.9f}, {val:.9f}] (certified lower end, restricted " f"upper end; {it} rounds; {time.time() - ts:.0f} s)", flush=True) if __name__ == "__main__": main() ~~~~~~~~~~~~ #### file: agg_lp5.py - kind: script - as run: SHA-256 a0be7edea25221b895e3360cc027d2e0e785e5c1d9e3dd164fab54e7ce4a711b - as shipped: SHA-256 ebc241712ba5e099fa4ffcfd2c0b3d8774cae305d65438494b157452ef772337 - identity: identical, after dropping docstrings and comments, to the file as run with 1 declared change(s) of a machine-specific path, which now read: `sys.path.insert(0, __import__('os').path.dirname(__import__('os').path.abspath(__file__)))` - trailing newline: yes ~~~~~~~~~~~~ python """agg_lp5.py: the lever test on the grid: twist_otf.py's free twisted shape program restricted to a family of data depending on chosen features of a shape (the number of small parts and their total, the number of rough parts, the largest rough part, the largest small parts, ...): phi(u) = g[class(u)]. A family's zero is a fake of the free program; a positive value means that the family's features do not suffice on that grid. Floats (HiGHS: interior point without crossover, then dual simplex); not certified. Usage: agg_lp5.py K m0 thI thT cap fam1 [fam2 ...] (environment ETA, ALPHA, THETA_RULE as twist_otf.py); DROP_DEAD=1 drops the twisted rows of Lemma 8.1 (a control).""" import math import os import sys import time from collections import Counter from fractions import Fraction as F import numpy as np from scipy.optimize import linprog from scipy.sparse import coo_matrix, csr_matrix sys.path.insert(0, __import__('os').path.dirname(__import__('os').path.abspath(__file__))) import modm_otf_lowmem2 as B0 # noqa: E402 from twist_otf import TwistRows # noqa: E402 time.time = time.monotonic def features(u, K, thT): sm = [x for x in u if x <= thT] ro = [x for x in u if thT < x < K / 2] la = [x for x in u if x >= K / 2] return {"j": len(sm), "tau": sum(sm), "k": len(ro), "rmax": ro[0] if ro else 0, "L": la[0] if la else 0, "smax": sm[0] if sm else 0, "s2": sm[1] if len(sm) > 1 else 0, "S": tuple(sm), "R": tuple(ro), "top3": tuple(sm[:3]), "top4": tuple(sm[:4]), "top5": tuple(sm[:5]), "top6": tuple(sm[:6]), "bot2": tuple(sm[-2:]), "nsm_ge": tuple(sum(1 for x in sm if x >= t) for t in (4, 8, 12, 16))} FAMS = {"ct": ("j", "tau", "k", "rmax", "L"), "ct+smax": ("j", "tau", "k", "rmax", "L", "smax"), "ct+s2": ("j", "tau", "k", "rmax", "L", "smax", "s2"), "S": ("S", "k", "rmax", "L"), "R": ("j", "tau", "R", "L"), "ct+s3": ("j", "tau", "k", "rmax", "L", "top3"), "ct+s4": ("j", "tau", "k", "rmax", "L", "top4"), "ct+s5": ("j", "tau", "k", "rmax", "L", "top5"), "ct+s6": ("j", "tau", "k", "rmax", "L", "top6"), "ct+s2+bot2": ("j", "tau", "k", "rmax", "L", "smax", "s2", "bot2"), "ct+cnt": ("j", "tau", "k", "rmax", "L", "nsm_ge")} def main(): K = int(sys.argv[1]); m0 = int(sys.argv[2]); thI = int(sys.argv[3]); thT = int(sys.argv[4]); cap = float(sys.argv[5]) fams = sys.argv[6:] eta = F(os.environ["ETA"]); alpha = F(os.environ["ALPHA"]); rule = os.environ.get("THETA_RULE", "half") thI_def = {"half": F(K, 2), "below_half": F((K - 1) // 2)}[rule] assert eta * K == m0 and alpha * K == thT and thI_def == thI, "PARAMS MISMATCH" log = print ts = time.time() log(f"PARAMS (checked): K = {K}; eta = {eta} -> m0 = {m0}; Type I thI = {thI} ({rule}); alpha = {alpha} -> thT = {thT}; " f"cap x <= {cap:g}; small <= {thT} < rough < {K / 2} <= large (grid units)", flush=True) U = B0.Universe(K, m0, thI, log=log) T = TwistRows(U, thT) shapes = list(B0.partitions_min(K, m0)) assert len(shapes) == U.S and shapes[U.P] == (K,) feats = [features(u, K, thT) for u in shapes] nrows = U.nI + T.nT DROP_DEAD = os.environ.get("DROP_DEAD", "0") == "1" sizes_T = np.array([sum(v) for v in U.vrows[:T.nT]], dtype=np.int64) dead_T = (sizes_T >= K - 2 * thT) & (sizes_T <= thT) log(f" dead twisted rows (|v| in [{K - 2 * thT}, {thT}]): {int(dead_T.sum())} of {T.nT}; " f"{'DROPPED (control run)' if DROP_DEAD else 'enforced'}", flush=True) log(f" {U.S} shapes, {U.nI} Type I + {T.nT} twisted rows; features {time.time() - ts:.0f} s; RSS {B0.rss_gb():.2f} GB", flush=True) for fam in fams: t1 = time.time() if fam == "free": cls = np.arange(U.S, dtype=np.int64) else: keyf = FAMS[fam] index = {} cls = np.empty(U.S, dtype=np.int64) for i, ft in enumerate(feats): key = ("PRIME",) if i == U.P else tuple(ft[n] for n in keyf) cls[i] = index.setdefault(key, len(index)) nC = int(cls.max()) + 1 cP = int(cls[U.P]) assert np.sum(cls == cP) == 1 A = csr_matrix((nrows, nC)) for r0, r1 in U.chunks(6_000_000): p0, p1 = U.rowptr[r0], U.rowptr[r1] ids = U.ids[p0:p1] rows = np.repeat(np.arange(r0, r1, dtype=np.int64), U.rowlen[r0:r1]) w = U.coef_slice(p0, p1).astype(np.float64) * U.Bf[ids] A = A + coo_matrix((w, (rows, cls[ids])), shape=(nrows, nC)).tocsr() if r0 < T.nT: r1t = min(r1, T.nT); nt = int(U.rowptr[r1t] - p0) A = A + coo_matrix((w[:nt] * T.lam[ids[:nt]], (U.nI + rows[:nt], cls[ids[:nt]])), shape=(nrows, nC)).tocsr() A = A.tocsr() A.eliminate_zeros() # row equilibration (rows are homogeneous equalities, so scaling them changes nothing): row r divided by max|A[r, :]| rowmax = np.asarray(abs(A).max(axis=1).todense()).ravel() keep_r = rowmax > 0 if DROP_DEAD: # control: drop the twisted rows with |v| in [K - 2 thT, thT] keep_r[U.nI + np.nonzero(dead_T)[0]] = False A = csr_matrix((1.0 / np.where(keep_r, rowmax, 1.0), (np.arange(nrows), np.arange(nrows))), shape=(nrows, nrows)) @ A A = A[np.nonzero(keep_r)[0]].tocsc() nrows_eff = A.shape[0] # column scaling: g_c = s_c * h_c with s_c = 1 / max|A[:, c]| colmax = np.asarray(abs(A).max(axis=0).todense()).ravel() s = np.where(colmax > 0, 1.0 / np.maximum(colmax, 1e-4), 1.0) # scale capped at 1e4 (v5) As = A @ csr_matrix((s, (np.arange(nC), np.arange(nC))), shape=(nC, nC)) lo = -np.ones(nC) / s hi = np.full(nC, cap - 1.0) / s hi[cP] = np.inf c = np.zeros(nC); c[cP] = s[cP] build = time.time() - t1 from scipy.optimize._highspy import _core Asc = As.tocsc(); Asc.sort_indices() meth = None; xsol = None for mode in os.environ.get("AGG_MODES", "ipm,simplex").split(","): hh = _core._Highs() for k_, v_ in (("output_flag", False), ("presolve", "off"), ("primal_feasibility_tolerance", 1e-10), ("dual_feasibility_tolerance", 1e-10), ("time_limit", float(os.environ.get("AGG_TL", "900")))): hh.setOptionValue(k_, v_) if mode == "ipm": hh.setOptionValue("solver", "ipm"); hh.setOptionValue("run_crossover", "off") else: hh.setOptionValue("solver", "simplex"); hh.setOptionValue("simplex_strategy", 1) INF = hh.getInfinity() hh.addRows(nrows_eff, np.zeros(nrows_eff), np.zeros(nrows_eff), 0, np.zeros(nrows_eff, dtype=np.int32), np.zeros(0, dtype=np.int32), np.zeros(0)) hh.addCols(nC, c, lo, np.where(np.isinf(hi), INF, hi), Asc.nnz, Asc.indptr[:-1].astype(np.int32), Asc.indices.astype(np.int32), Asc.data.astype(np.float64)) hh.run() st = hh.modelStatusToString(hh.getModelStatus()) if st == "Optimal": xsol = np.asarray(hh.getSolution().col_value, dtype=np.float64); meth = f"core-{mode}" break log(f" family {fam}: core {mode} status {st}; next", flush=True) if xsol is None: log(f"FAMILY {fam}: solver failed (all modes)", flush=True) continue class _R: pass r = _R(); r.x = xsol g_raw = r.x * s glo = np.full(nC, -1.0); ghi = np.full(nC, cap - 1.0); ghi[cP] = np.inf g = np.clip(g_raw, glo, ghi) # v5: judge the clipped point clip = float(np.max(np.abs(g - g_raw))) if nC else 0.0 val = 1.0 + float(g[cP]) res = float(np.max(np.abs(A @ g))) if nrows_eff else 0.0 # on the row-equilibrated matrix, at the clipped point comp = np.arange(nC) != cP log(f"FAMILY {fam}{' [DEAD ROWS DROPPED]' if DROP_DEAD else ''} (key {FAMS.get(fam, 'free')}): {nC} classes; C- = {val:.9f}{' ZERO (a fake in this family)' if (val <= 1e-7 and res <= 1e-9) else (' (value ~0 but residual above 1e-9: NOT a zero)' if val <= 1e-7 else '')}; clip {clip:.1e}; " f"phi in [{g.min():.6f}, {g[comp].max():.6f}]; max |row residual| {res:.2e}; build {build:.0f} s, solve {time.time() - t1 - build:.0f} s " f"({meth}); RSS {B0.rss_gb():.2f} GB", flush=True) if val <= 1e-7 and res <= 1e-9 and fam != "free": np.savez(f"agg_K{K}_m0{m0}_thI{thI}_thT{thT}_cap{cap:g}_{fam}{'_dropdead' if DROP_DEAD else ''}.npz", cls=cls, g=g) if __name__ == "__main__": main() ~~~~~~~~~~~~ #### file: cert2_f3.py - kind: script - as run: SHA-256 e5f7278044c5a0a68c22ae7d9848667fdeae2e4ea4d94b3b638e871dbbeba22d - as shipped: SHA-256 29ff1626c2783a121ded47a41374672eee3bcc84f19497b87bf2decd1abd35ac - identity: identical, after dropping docstrings and comments, to the file as run with 2 declared change(s) of a machine-specific path, which now read: `sys.path.insert(0, __import__('os').path.dirname(__import__('os').path.abspath(__file__)))`; `open(f'cert2_{TAG_}_M{M}.json', 'w')` - trailing newline: yes ~~~~~~~~~~~~ python #!/usr/bin/env python # cert2_f3.py: the second-order engine of the paper (engine2/cert2_f2.py, used for the case (1/30, 4/15)) with the guard on the # number of forced levels widened from m* <= 9 to m* <= 19; nothing else changes. Integer arithmetic with directed rounding # (dpl.py, the shipped engine2/dpl.py). Here it was used to price the candidate at (1/36, 17/60). # Usage: python cert2_f3.py M NB PICKLE TAG (environment as cert2_f2.py). Writes cert2__M.json. import sys, os, time, json, hashlib, pickle from fractions import Fraction as Fr from math import comb sys.path.insert(0, __import__('os').path.dirname(__import__('os').path.abspath(__file__))) import dpl T_START = time.time() def log(*a): print(f"[{time.time() - T_START:7.1f}s]", *a, flush=True) M_, NB_, PKL_, TAG_ = sys.argv[1:5] M = int(M_); NB = int(NB_) dpl.init(M) from dpl import (D, SCALE, fl, ce, fz, dz, d_add, d_neg, d_scale, d_div, d_trunc, d_cell, d_ends, d_ends_rev, d_at1, d_at0, d_from_cells, d_const_cells, d_minlow, d_maxup, d_maxwidth, d_kernel, conv_nn, conv_sn, conv_ss, corr_sn, corr_nn, cell_mul, cell_neg_quot, cell_add, cell_contains0, iv_mul, iv_div, add_iv, iv_scale, _pos_lo, _pos_hi, _neg_lo, _neg_hi) raw = open(PKL_, 'rb').read(); PKL_SHA = hashlib.sha256(raw).hexdigest(); dpk = pickle.loads(raw) ETA, ALPHA = Fr(dpk['eta']), Fr(dpk['alpha']) assert (M * ETA).denominator == 1 and (M * ALPHA).denominator == 1 and M % 2 == 0 iE, iA, iH = int(M * ETA), int(M * ALPHA), M // 2 KPROF = None if os.environ.get('KPROF', 'all') == 'all' else set(int(x) for x in os.environ['KPROF'].split(',')) assert M > 0 and NB > 0 # audit fix 2 assert 0 < ETA < ALPHA < Fr(1, 2) assert 6 * ALPHA >= 1, "rough-count cutoff 5 is insufficient" assert 20 * ETA >= ALPHA, "the forced-level search range 1..19 of MSTAR is insufficient" # addendum 9 (was 10*ETA >= ALPHA, m* <= 9) MSTAR = max(j for j in range(1, 20) if j * ETA < ALPHA) log(f"cert2 (second order): M={M} NB={NB} NBQ={os.environ.get('NBQ', NB)} pickle sha256={PKL_SHA} tag={TAG_} KPROF={'all' if KPROF is None else sorted(KPROF)} KW={os.environ.get('KW')} eta={ETA} alpha'={ALPHA} m*={MSTAR}") # ---------------------------------------------------------------- bins and constants _ed = dpk.get('edges') if isinstance(_ed, str): _ed = [Fr(t) for t in _ed.split(',') if t.strip()] # some input pickles store the interior edges as text EDGES = [Fr(e) for e in _ed] if _ed else [ALPHA, Fr(1, 2)] if EDGES[-1] != Fr(1, 2): EDGES = EDGES + [Fr(1, 2)] if EDGES[0] != ALPHA: EDGES = [ALPHA] + [e for e in EDGES if e > ALPHA] assert EDGES[-1] == Fr(1, 2), EDGES assert len(EDGES) >= 2 and EDGES[0] == ALPHA assert all(a < b for a, b in zip(EDGES, EDGES[1:])), "unordered bin edges" NBIN = len(EDGES) - 1 for q in EDGES: assert (q * M).denominator == 1, f"M does not put {q} on the grid" iEdge = [int(e * M) for e in EDGES] def class_total_range(j, k, b): if k == 0: return (j * ETA, j * ALPHA) lo_b, hi_b = EDGES[b - 1], EDGES[b] return (j * ETA + (k - 1) * ALPHA + lo_b, j * ALPHA + k * hi_b) def realisable_bin(j, k, b): lo, hi = class_total_range(j, k, b); return lo < 1 < hi TEXTS = {'v2main': [(0, 3), (0, 4), (2, 2), (2, 3), (2, 4), (3, 2), (3, 3), (4, 1), (4, 2), (5, 1), (5, 2), (6, 1), (6, 2)]} TEXT = TEXTS.get(TAG_) C3 = {} for cls, x in zip(dpk['classes'], dpk['x']): j, k, b = (int(t) for t in cls) if abs(float(x)) == 0: continue if k == 0: b = 0 if NBIN == 1 and k >= 1: b = 1 if (k == 0 and b != 0) or (k >= 1 and not 1 <= b <= NBIN): continue if TEXT is not None and (j, k) not in TEXT: continue if not realisable_bin(j, k, b): continue assert (j, k, b) not in C3 C3[(j, k, b)] = Fr(float(x)) if TEXT is not None: assert sorted((j, k) for (j, k, b) in C3) == sorted(TEXT), sorted(C3) MAXS = max(9, max(j for (j, k, b) in C3), int(os.environ.get('MAXS', '0'))) # small-count power cutoff: every constant's small count, the forced levels, and env MAXS (oracle bases) assert MAXS >= MSTAR def abel(cdict): Dd = {b: {} for b in range(1, NBIN + 1)} for (j, k, b), v in cdict.items(): if k == 0: Dd[NBIN][(j, 0)] = Dd[NBIN].get((j, 0), 0) + v else: Dd[b][(j, k)] = Dd[b].get((j, k), 0) + v if b >= 2: Dd[b - 1][(j, k)] = Dd[b - 1].get((j, k), 0) - v return {b: {jk: v for jk, v in dd.items() if v != 0} for b, dd in Dd.items()} GBIN = int(os.environ.get('GBIN', str(NBIN))) GCLS = (0, 3, GBIN if NBIN > 1 else 1) # env GVEC 'b:w,b:w' = the T0 correction direction g = sum_b w_b e_(0,3,b) (H-neutral: no small parts); default e_GCLS GDICT = {(0, 3, int(a)): Fr(w) for a, w in (t.split(':') for t in os.environ['GVEC'].split(','))} if os.environ.get('GVEC') else {GCLS: Fr(1)} for _g in GDICT: assert realisable_bin(*_g), f'g class {_g} not realisable' DB = abel(C3); DG = abel(GDICT) log(f"bins: edges {[str(e) for e in EDGES]}; {len(C3)} constants; g = {GDICT}") log("c:", {kk: round(float(v), 6) for kk, v in sorted(C3.items())}) # ---------------------------------------------------------------- kernels (global: uncut rough kernel), powers, Lemma K kernels phi S1 = d_kernel(iE, iA); R1 = d_kernel(iA, iH) def build_powers(Rker): Pd = {}; PSd = {(1, 0): S1, (0, 1): Rker} for b in range(0, MAXS + 1): for bp in range(0, 6): if b + bp < 2 or b * ETA + bp * ALPHA > 1: continue if bp >= 1: prev, ker, div = (b, bp - 1), Rker, bp else: prev, ker, div = (b - 1, 0), S1, b if prev not in PSd: continue Pd[(b, bp)] = d_div(conv_nn(PSd[prev], ker), div); PSd[(b, bp)] = Pd[(b, bp)] return Pd, PSd P, PS = build_powers(R1) log(f"powers: {len(P)}") def Rk(k): return R1 if k == 1 else PS[(0, k)] def Sst(r): return PS[(r, 0)] class Phis: def __init__(self, Pd, PSd): self.P, self.PS, self.TR, self.PHI = Pd, PSd, {}, {} def trs(self, bb): if bb not in self.TR: self.TR[bb] = None if (sum(bb) < 2 or bb not in self.P) else d_trunc(self.P[bb], iH) return self.TR[bb] def phi(self, j, k): if (j, k) in self.PHI: return self.PHI[(j, k)] acc = dz() for bs in range(0, j + 1): for br in range(0, k + 1): if bs + br < 2 or (bs, br) not in self.P: continue sign = (-1) ** (j + k + 1) * (-1) ** (bs + br); comp = (j - bs, k - br) if comp == (0, 0): X = d_trunc(self.P[(bs, br)], iH) else: if comp not in self.PS: continue X = conv_nn(self.trs((bs, br)), self.PS[comp]) acc = d_add(acc, X, sign) self.PHI[(j, k)] = d_trunc(acc, iH); return self.PHI[(j, k)] def lin_phi(self, coef): acc = dz() for jk, cv in coef.items(): if cv != 0: acc = d_add(acc, d_scale(self.phi(*jk), cv)) return acc def H_beta(self, jb, kb, dd): coef = {} for (cj, ck), cv in dd.items(): ju, ku = cj - jb, ck - kb if ju >= 0 and ku >= 0 and ju + ku >= 2: coef[(ju, ku)] = coef.get((ju, ku), 0) + cv return self.lin_phi(coef) if coef else None GPH = Phis(P, PS) phi = GPH.phi def rsum(ms): s = dz() for m in ms: if m == 1: s = d_add(s, R1) elif (0, m) in P: s = d_add(s, P[(0, m)]) return s RALL = rsum([1, 2, 3, 4, 5]); EP = rsum([2, 4]); EM = rsum([1, 3, 5]) def E_conv(X, rho_c, twisted=True, nodelta=False): """E_rho^{(c)} * X: twisted (-1)^c (delta + E'+ - E'-) * X; untwisted (delta + R_all) * X""" if twisted: out = dz() if nodelta else X out = d_add(out, conv_sn(X, EP)); out = d_add(out, conv_sn(X, EM), -1) return d_neg(out) if (rho_c or 0) % 2 == 1 else out out = dz() if nodelta else X return d_add(out, conv_sn(X, RALL)) log("kernels ready") # exact values of (f*g)(1) and int f g, without building the whole convolution (no curvature needed at a grid point) def _N_at(F, G, k, mode): a, b = F; c, d = G; tot = 0 if mode == 'conv': # n + m = k - 1 for n in range(max(0, k - M), min(M, k)): m = k - 1 - n if m < 0 or m >= M: continue an, bn, cm, dm = a[n], b[n], c[m], d[m] if an or bn: tot += (an * cm + bn * dm) + 2 * (an * dm + bn * cm) else: # corr at s = 0: n = m for n in range(M): an, bn, cm, dm = a[n], b[n], c[n], d[n] if (an or bn) and (cm or dm): tot += 2 * (an * cm + bn * dm) + (an * dm + bn * cm) return tot def val_sn(f, g, mode): """interval of (f*g)(1) (mode 'conv') or int f g (mode 'corr'), f signed, g >= 0""" S6 = 6 * M * SCALE glo = _pos_lo(g.la, g.lb); ghi = _pos_hi(g.ua, g.ub) lo = fl(_N_at(_pos_lo(f.la, f.lb), glo, M, mode), S6) - ce(_N_at(_neg_hi(f.la, f.lb), ghi, M, mode), S6) hi = ce(_N_at(_pos_hi(f.ua, f.ub), ghi, M, mode), S6) - fl(_N_at(_neg_lo(f.ua, f.ub), glo, M, mode), S6) return (lo, hi) # ---------------------------------------------------------------- one bin machine per edge (rough kernel cut at the edge) class Mach(Phis): def __init__(self, ecell): if ecell == iH: Pd, PSd = P, PS else: Pd, PSd = build_powers(d_kernel(iA, ecell)) Phis.__init__(self, Pd, PSd) if ecell == iH: self.PHI = GPH.PHI; self.TR = GPH.TR def f1_c(self, dd): acc = (0, 0) for jk, cv in dd.items(): if sum(jk) >= 2: acc = add_iv(acc, iv_scale(d_at1(self.phi(*jk)), cv)) return acc def T0_c(self, dd, twisted): rho = (lambda d: (-1) ** d) if twisted else (lambda d: 1) acc = (0, 0) if not dd: return acc for (j, k), cv in dd.items(): if (j, k) in self.P: acc = add_iv(acc, iv_scale(d_at1(self.P[(j, k)]), cv * rho(j + k))) if j + k >= 2: acc = add_iv(acc, iv_scale(d_at1(self.phi(j, k)), cv * rho(1))) jmax = max(j for j, k in dd); kmax = max(k for j, k in dd) for jb in range(0, jmax + 1): for kb in range(0, kmax + 1): if (jb, kb) == (0, 0) or (jb, kb) not in self.PS: continue Hb = self.H_beta(jb, kb, dd) if Hb is None: continue acc = add_iv(acc, iv_scale(val_sn(d_trunc(Hb, iH), self.PS[(jb, kb)], 'conv'), rho(1 + jb + kb))) return acc def A_dpl(self, base, twisted, dd): """A_base(x).c as a DPL in x = 1 - sigma (free part + extended part), DESIGN_GM""" rho = (lambda d: (-1) ** d) if twisted else (lambda d: 1) A = dz() if not dd: return A for (cj, ck), cv in dd.items(): if cj >= base: jz, kz = cj - base, ck if jz + kz >= 2 and (jz, kz) in self.P: A = d_add(A, d_scale(self.P[(jz, kz)], cv * rho(base + jz + kz))) jmax = max(j for j, k in dd); kmax = max(k for j, k in dd) for jp in range(0, jmax + 1): for kp in range(0, kmax + 1): coef = {} for (cj, ck), cv in dd.items(): ju, ku = cj - base - jp, ck - kp if ju >= 0 and ku >= 0 and ju + ku >= 2: coef[(ju, ku)] = coef.get((ju, ku), 0) + cv if not coef: continue Gk = self.lin_phi(coef) if (jp, kp) == (0, 0): X = Gk else: if (jp, kp) not in self.PS: continue X = conv_sn(d_trunc(Gk, iH), self.PS[(jp, kp)]) A = d_add(A, d_scale(X, rho(base + 1 + jp + kp))) return A MACH = {b: Mach(iEdge[b]) for b in range(1, NBIN + 1)} log(f"bin machines ready: {NBIN}") # ---------------------------------------------------------------- rough-count profile: B_{j,K}, brackets, slab chains KALL = list(range(2, 6)) # weighted rough-count profile (env KW, e.g. '2:1,3:-1'): H_j(sigma, k) = w_k h_j(sigma); KPROF = the support of KW. Unset: KPROF as before (w = 1). KW_ENV = os.environ.get('KW') KW = {int(a): Fr(b) for a, b in (t.split(':') for t in KW_ENV.split(','))} if KW_ENV else None if KW is not None: KPROF = set(k for k, v in KW.items() if v != 0) def wk(k): if KW is not None: return KW.get(k, Fr(0)) return Fr(1) if (KPROF is None or k in KPROF) else Fr(0) def kin(k): return wk(k) != 0 def kws_of(kb): """fragment rough counts ky with the weight w(kb + ky) of the completed vector's total rough count (KPROF all, no KW: ky = 1..5, w = 1)""" if KPROF is None and KW is None: return tuple((ky, Fr(1)) for ky in range(1, 6)) return tuple((k - kb, wk(k)) for k in range(1, 6) if wk(k) != 0 and k - kb >= 1) KWS = tuple((k, wk(k)) for k in KALL if wk(k) != 0) # f(1)'s Omega: rough counts >= 2, no beta def rho_(d, twisted): return (-1) ** d if twisted else 1 _RPHI = {} def RPHI(kw, ky): if (kw, ky) not in _RPHI: if (0, ky) not in P: _RPHI[(kw, ky)] = None elif kw == 0: _RPHI[(kw, ky)] = phi(0, ky) elif (0, kw) not in PS: _RPHI[(kw, ky)] = None else: _RPHI[(kw, ky)] = conv_sn(d_trunc(phi(0, ky), iH), PS[(0, kw)]) return _RPHI[(kw, ky)] _BK = {} def Bk(j, twisted): key = ('B', j % 2 if twisted else 0, twisted) if key not in _BK: acc = dz() for k in KALL: if not kin(k): continue if (0, k) in P: acc = d_add(acc, d_scale(P[(0, k)], wk(k) * rho_(j + k, twisted))) for kw in range(0, k - 1): X = RPHI(kw, k - kw) if X is None: continue acc = d_add(acc, d_scale(X, wk(k) * rho_(j + kw + 1, twisted))) _BK[key] = acc return _BK[key] def EB(c, twisted): """free-completion bracket sum_{k in K} rho(c+k) R_k + sum_{k in K} sum_{kw+ky=k, ky>=2} rho(c+1+kw) (R_kw * Phi_{0,ky}tr), as a DPL in x""" key = ('E', c % 2 if twisted else 0, twisted) if key not in _BK: acc = dz() for k in KALL: if not kin(k): continue if (0, k) in P: acc = d_add(acc, d_scale(P[(0, k)], wk(k) * rho_(c + k, twisted))) for kw in range(0, k - 1): X = RPHI(kw, k - kw) if X is None: continue acc = d_add(acc, d_scale(X, wk(k) * rho_(c + 1 + kw, twisted))) _BK[key] = acc return _BK[key] def blocks(lo_c, hi_c): n_c = hi_c - lo_c; bl = max(1, -(-n_c // NB)) return [(c0, min(c0 + bl, hi_c)) for c0 in range(lo_c, hi_c, bl)] NBQ = int(os.environ.get('NBQ', str(NB))) # sigma_beta blocks in positivity's (j_b >= 1, k_b >= 1) Omega terms (first order there) NBQLO = int(os.environ.get('NBQLO', str(NBQ))) # finer sigma_beta blocks for j_b <= 2 (the binding (1,1), (2,1) classes at 1/4) def blocksQ(lo_c, hi_c, nbq=None): n_c = hi_c - lo_c; bl = max(1, -(-n_c // (nbq or NBQ))) return [(c0, min(c0 + bl, hi_c)) for c0 in range(lo_c, hi_c, bl)] NBQJ = {int(a): int(b) for a, b in (u.split(':') for u in os.environ['NBQJ'].split(','))} if os.environ.get('NBQJ') else {} # (A) assert all(v >= 1 for v in NBQJ.values()) def nbq_of(jb): return NBQJ.get(jb, NBQLO if jb <= 2 else NBQ) # (A): the sigma_beta block count for j_beta QOUTER = os.environ.get('QOUTER', '0') == '1' # (B) QADAPT = {} # (C): (jb, kb) -> (scaled target, split, depth) for _it in (os.environ['QADAPT'].split(';') if os.environ.get('QADAPT') else []): _cls, _tg, _sp, _dp = _it.split(':'); _jb, _kb = (int(x) for x in _cls.split(',')); _tf = Fr(_tg) QADAPT[(_jb, _kb)] = (fl(_tf.numerator * SCALE, _tf.denominator), int(_sp), int(_dp)) assert int(_sp) >= 2 and int(_dp) >= 0 assert QOUTER or not QADAPT, 'QADAPT needs QOUTER=1' def slab_br(br, t_lo, t_hi): Rb = Rk(br) return (d_trunc(Rb, iH - t_lo, iH), d_trunc(Rb, iH - t_hi, iH)) def conv_chain(kw, sl, m): cur = sl if kw >= 1: cur = conv_nn(cur, Rk(kw)) if m >= 1: cur = conv_nn(cur, Rk(m)) return cur def coefK(j, i, b, kw, ky, br, twisted): tot = 0 for i1 in range(0, i - b + 1): i2 = i - i1 s_out = rho_(j + i1 + kw + 1, twisted) tot += comb(i - b, i1) * s_out * (-1) ** (i2 + ky + 1) * (-1) ** b * (-1) ** br return tot def H_times_x(Hd, X, lo_u): """f(u) = H(u) X(1 - u) on u-cells [lo_u, iA) (X a DPL in x)""" return d_from_cells([cell_mul(d_ends(Hd, u), d_ends_rev(X, M - u - 1)) for u in range(lo_u, iA)], lo_u) def H_times_u(Hd, X, lo_u): """f(u) = H(u) X(u) on u-cells [lo_u, iA)""" return d_from_cells([cell_mul(d_ends(Hd, u), d_ends(X, u)) for u in range(lo_u, iA)], lo_u) # ---------------------------------------------------------------- refined threshold blocks (gm2/DESIGN_SO.md addendum 1): Part A exact at the block # start tau0, Part B the sawtooth rho(tau) = int_{tau0}^{tau} R_br(1/2 - t) dt times the range of g = R_kw * R_m on a window of width dtau. # BLOCKS=plain restores the narrow/wide (first-order) block bound for comparison. BLOCKMODE = os.environ.get('BLOCKS', 'refined') ALLCOMBOS = sorted(set((kw, br, k - kw - br) for k in KALL if kin(k) for kw in range(0, k) for br in range(1, k - kw + 1) if not ((br >= 2 and (0, br) not in PS) or (kw >= 2 and (0, kw) not in PS) or (k - kw - br >= 2 and (0, k - kw - br) not in PS)))) _GD = {} def g_dpl(kw, m): if (kw, m) not in _GD: if kw == 0 and m == 0: _GD[(kw, m)] = None elif kw == 0: _GD[(kw, m)] = Rk(m) elif m == 0: _GD[(kw, m)] = Rk(kw) else: _GD[(kw, m)] = conv_nn(Rk(kw), Rk(m)) return _GD[(kw, m)] def _sparse(vals, op): tab = [vals] while (1 << len(tab)) <= len(vals): h = 1 << (len(tab) - 1); a = tab[-1] tab.append([op(a[i], a[i + h]) for i in range(len(a) - h)]) def q(l, r): k = (r - l + 1).bit_length() - 1 return op(tab[k][l], tab[k][r - (1 << k) + 1]) return q _GR = {} def g_range(kw, m, dtc, kind): """(Gl, Gu) per cell n: the range of g = R_kw * R_m (>= 0) on the window of g-cells [iH-1-n-dtc, iH-1-n] (kind 'corr': argument 1/2 - y - s, s in [0, dtau], y in cell n) or [n-iH-dtc, n-iH] (kind 'conv': argument z - 1/2 - s, z in cell n); off-grid cells count as 0""" key = (kw, m, dtc, kind) if key in _GR: return _GR[key] g = g_dpl(kw, m) qlo = _sparse([min(a, b) for a, b in zip(g.la, g.lb)], min); qhi = _sparse([max(a, b) for a, b in zip(g.ua, g.ub)], max) Gl = [0] * M; Gu = [0] * M for n in range(M): c_lo, c_hi = (iH - 1 - n - dtc, iH - 1 - n) if kind == 'corr' else (n - iH - dtc, n - iH) vals_lo = []; vals_hi = [] if c_lo < 0 or c_hi >= M: vals_lo.append(0); vals_hi.append(0) a_, b_ = max(c_lo, 0), min(c_hi, M - 1) if a_ <= b_: vals_lo.append(qlo(a_, b_)); vals_hi.append(qhi(a_, b_)) Gl[n] = max(0, min(vals_lo)); Gu[n] = max(0, max(vals_hi)) _GR[key] = (Gl, Gu) return _GR[key] def rho_dpl(br, bl): """rho(tau) = int_{tau0(block)}^{tau} R_br(1/2 - t) dt on the tau-cells of the blocks bl, as a DPL (linear bounds from the min/max of R_br's enclosure on the mirrored cell; the cumulative sums are exact integrals of R_br's linear bounds)""" Rb = Rk(br); la = [0] * M; lb = [0] * M; ua = [0] * M; ub = [0] * M for (c0, c1) in bl: acc_lo = acc_hi = 0 for c in range(c0, c1): n = iH - 1 - c rl = max(0, min(Rb.la[n], Rb.lb[n])); ru = max(Rb.ua[n], Rb.ub[n]) la[c] = acc_lo; lb[c] = acc_lo + fl(rl, M); ua[c] = acc_hi; ub[c] = acc_hi + ce(ru, M) acc_lo += fl(max(Rb.la[n] + Rb.lb[n], 0), 2 * M) # audit fix 1 acc_hi += ce(Rb.ua[n] + Rb.ub[n], 2 * M) return D(la, lb, ua, ub) def W_dpl(b, br, bl): rho = rho_dpl(br, bl); S = Sst(b); lo_c, hi_c = bl[0][0], bl[-1][1] return d_from_cells([cell_mul(d_ends(S, c), d_ends(rho, c)) for c in range(lo_c, hi_c)], lo_c) def H_pm(Hd): lp = _pos_lo(Hd.la, Hd.lb); up = _pos_hi(Hd.ua, Hd.ub); ln = _neg_lo(Hd.ua, Hd.ub); un = _neg_hi(Hd.la, Hd.lb) return D(lp[0], lp[1], up[0], up[1]), D(ln[0], ln[1], un[0], un[1]) def _gI(G, Ip, Im, c): """c * [Gl I+ - Gu I-, Gu I+ - Gl I-] per cell (G constant per cell, I+- >= 0 DPLs), as a DPL""" Gl, Gu = G lo_a = [fl(gl * ipa - gu * ima, SCALE) for gl, gu, ipa, ima in zip(Gl, Gu, Ip.la, Im.ua)] lo_b = [fl(gl * ipb - gu * imb, SCALE) for gl, gu, ipb, imb in zip(Gl, Gu, Ip.lb, Im.ub)] hi_a = [ce(gu * ipa - gl * ima, SCALE) for gl, gu, ipa, ima in zip(Gl, Gu, Ip.ua, Im.la)] hi_b = [ce(gu * ipb - gl * imb, SCALE) for gl, gu, ipb, imb in zip(Gl, Gu, Ip.ub, Im.lb)] X = D(lo_a, lo_b, hi_a, hi_b) return d_scale(X, c) def slab_corr(Hd, lo_u, i, coefs, combos, at0=False): """sum_b sum_combos coef(b, kw, br, m) * int int H(u) X_tau(1 - u) S_b(tau) S_{i-b}(u - tau - sigma) dtau du for each coefficient function in coefs; DPLs in sigma (or intervals at sigma = 0 if at0). kw = m = 0 terms vanish here (argument 1 - u > 1/2 is outside the slab).""" combos = [cb for cb in combos if cb[0] + cb[2] >= 1] res = [((0, 0) if at0 else dz()) for _ in coefs] def acc_(o, X): if at0: res[o] = add_iv(res[o], X) else: res[o] = d_add(res[o], X) for b in range(1, i + 1): bl = blocks(b * iE, iA); dtc = max(c1 - c0 for c0, c1 in bl) cmb = [cb for cb in combos if any(cf(b, *cb) != 0 for cf in coefs)] if not cmb: continue for (c0, c1) in bl: Kb = d_trunc(Sst(b), c0, c1); K = Kb if b == i else conv_nn(Kb, Sst(i - b)) if BLOCKMODE == 'refined': X0 = {cb: conv_chain(cb[0], d_trunc(Rk(cb[1]), iH - c0, iH), cb[2]) for cb in cmb} for o, cf in enumerate(coefs): Q = dz() for cb, X in X0.items(): cv = cf(b, *cb) if cv: Q = d_add(Q, d_scale(X, cv)) f = H_times_x(Hd, Q, lo_u) acc_(o, val_sn(f, K, 'corr') if at0 else corr_sn(f, K)) else: ch = {} for cb in cmb: sn, sw = slab_br(cb[1], c0, c1); ch[cb] = (conv_chain(cb[0], sn, cb[2]), conv_chain(cb[0], sw, cb[2])) for o, cf in enumerate(coefs): Q = dz() for cb, (Xn, Xw) in ch.items(): cv = cf(b, *cb) if cv: Q = d_add(Q, d_scale(D(Xn.la, Xn.lb, Xw.ua, Xw.ub), cv)) # d_scale swaps the sides when cv < 0 f = H_times_x(Hd, Q, lo_u) acc_(o, val_sn(f, K, 'corr') if at0 else corr_sn(f, K)) if BLOCKMODE != 'refined': continue Hp, Hm = H_pm(Hd) for br in sorted(set(cb[1] for cb in cmb)): W = W_dpl(b, br, bl) Ip, Im = corr_nn(Hp, W), corr_nn(Hm, W) for o, cf in enumerate(coefs): PB = None for cb in cmb: if cb[1] != br: continue cv = cf(b, *cb) if not cv: continue t_ = _gI(g_range(cb[0], cb[2], dtc, 'corr'), Ip, Im, cv) PB = t_ if PB is None else d_add(PB, t_) if PB is None: continue if b < i: acc_(o, val_sn(PB, Sst(i - b), 'corr') if at0 else corr_sn(PB, Sst(i - b))) else: acc_(o, (PB.la[0], PB.ua[0]) if at0 else PB) return res def coupling(j, i, Hd): """(twisted, untwisted) C_{j,i}[H_{j+i}](sigma) as DPLs in sigma (DESIGN_GM): free + i2 = 0 part, then the i2 >= 1 slab terms""" lo_u = (j + i) * iE res = [corr_sn(H_times_x(Hd, EB(j + i, tw), lo_u), Sst(i)) for tw in (True, False)] ext = slab_corr(Hd, lo_u, i, [lambda b, kw, br, m, tw=tw: coefK(j, i, b, kw, br + m, br, tw) * wk(kw + br + m) for tw in (True, False)], ALLCOMBOS) return d_add(res[0], ext[0]), d_add(res[1], ext[1]) def coupling_at0(i, Hd): """(twisted, untwisted) C_{0,i}[H_i] at sigma = 0, as intervals""" lo_u = i * iE res = [val_sn(H_times_x(Hd, EB(i, tw), lo_u), Sst(i), 'corr') for tw in (True, False)] ext = slab_corr(Hd, lo_u, i, [lambda b, kw, br, m, tw=tw: coefK(0, i, b, kw, br + m, br, tw) * wk(kw + br + m) for tw in (True, False)], ALLCOMBOS, at0=True) return add_iv(res[0], ext[0]), add_iv(res[1], ext[1]) # ---------------------------------------------------------------- Step A: top-down forcing H = {}; untw_bad = {} for j in range(MSTAR, 0, -1): lo_c = j * iE; cells = range(lo_c, iA) At = dz(); Au = dz() for b in range(1, NBIN + 1): At = d_add(At, MACH[b].A_dpl(j, True, DB[b])); Au = d_add(Au, MACH[b].A_dpl(j, False, DB[b])) Btw, Bun = Bk(j, True), Bk(j, False) num = [d_ends_rev(At, M - u - 1) for u in cells] bad = sum(1 for u in cells if not cell_contains0(d_ends_rev(Au, M - u - 1))) + sum(1 for u in cells if not cell_contains0(d_ends_rev(Bun, M - u - 1))) wA = max(max(x[2] - x[0], x[3] - x[1]) for x in num) for i in range(1, MSTAR - j + 1): Ct, Cu = coupling(j, i, H[j + i]) bu = sum(1 for u in cells if not cell_contains0(d_ends(Cu, u))) bad += bu num = [cell_add(x, d_ends(Ct, u)) for x, u in zip(num, cells)] log(f" T{j}: coupling to H{j+i}: twisted width max {fz(d_maxwidth(Ct, lo_c, iA)):.3e}; untwisted cells not containing 0: {bu}/{len(cells)}") Bc = [d_ends_rev(Btw, M - u - 1) for u in cells] zc = [u for u, bb in zip(cells, Bc) if not ((bb[0] > 0 and bb[1] > 0) or (bb[2] < 0 and bb[3] < 0))] if zc: log(f"T{j}: B may vanish on {len(zc)} cells, sigma in [{zc[0]/M:.5f}, {zc[-1]/M:.5f}]"); sys.exit(2) Hc = [cell_neg_quot(n_, b_) for n_, b_ in zip(num, Bc)] H[j] = d_from_cells(Hc, lo_c); untw_bad[j] = bad log(f"T{j}: B in [{fz(min(min(x[0], x[1]) for x in Bc)):+.4f}, {fz(max(max(x[2], x[3]) for x in Bc)):+.4f}]; " f"H{j} in [{fz(d_minlow(H[j], lo_c, iA)):+.4f}, {fz(d_maxup(H[j], lo_c, iA)):+.4f}]; A-part width max {fz(wA):.3e}; " f"H width max {fz(d_maxwidth(H[j], lo_c, iA)):.3e}; untwisted A/B/C cell ends not enclosing 0: {bad}") # ---------------------------------------------------------------- Step B: T0 and f(1) def Omega_rK(Hd, r, kws): """Omega^K_r[H](a) as a DPL in a (DESIGN_GM addendum 2; b = r term by the per-br slab identity; 1 <= b <= r-1 terms by blocks: refined (Part A at tau0 + Part B, DESIGN_SO addendum 1) for m >= 1, narrow/wide for m = 0)""" out = dz() HS = H_times_u(Hd, Sst(r), r * iE) Hp = Hm = None for (k, wgt) in kws: if k < 1 or wgt == 0: continue for br in range(1, k + 1): m = k - br if (br >= 2 and (0, br) not in PS) or (m >= 2 and (0, m) not in PS): continue X = d_trunc(conv_sn(HS, d_trunc(Rk(br), 0, iH)), iH) if m >= 1: X = conv_sn(X, Rk(m)) out = d_add(out, d_scale(X, wgt * (-1) ** (r + k + 1) * (-1) ** r * (-1) ** br)) for b in range(1, r): sgb = wgt * (-1) ** (r + k + 1) * (-1) ** b * (-1) ** br bl = blocks(b * iE, iA); dtc = max(c1 - c0 for c0, c1 in bl) if m == 0 or BLOCKMODE != 'refined': for (c0, c1) in bl: Kk = conv_nn(d_trunc(Sst(b), c0, c1), Sst(r - b)) w = H_times_u(Hd, Kk, r * iE) sn, sw = slab_br(br, c0, c1) Xn, Xw = conv_chain(0, sn, m), conv_chain(0, sw, m) out = d_add(out, d_scale(conv_ss(w, D(Xn.la, Xn.lb, Xw.ua, Xw.ub)), sgb)) continue for (c0, c1) in bl: # Part A: the chain exact at tau0 = c0 Kk = conv_nn(d_trunc(Sst(b), c0, c1), Sst(r - b)) w = H_times_u(Hd, Kk, r * iE) out = d_add(out, d_scale(conv_sn(w, conv_chain(0, d_trunc(Rk(br), iH - c0, iH), m)), sgb)) if Hp is None: Hp, Hm = H_pm(Hd) # Part B: S_{r-b} J+- convolved with the window range of R_m W = W_dpl(b, br, bl) Jp, Jm = corr_nn(Hp, W), corr_nn(Hm, W) S_ = Sst(r - b) Ap = d_from_cells([cell_mul(d_ends(S_, y), d_ends(Jp, y)) for y in range(M)], 0) Am = d_from_cells([cell_mul(d_ends(S_, y), d_ends(Jm, y)) for y in range(M)], 0) Gl, Gu = g_range(0, m, dtc, 'conv') Gd = D(list(Gl), list(Gl), list(Gu), list(Gu)) out = d_add(out, d_scale(d_add(conv_nn(Ap, Gd), conv_nn(Am, Gd), -1), sgb)) return out KSET = KWS T0t = (0, 0); T0u = (0, 0) for _b in range(1, NBIN + 1): T0t = add_iv(T0t, MACH[_b].T0_c(DB[_b], True)); T0u = add_iv(T0u, MACH[_b].T0_c(DB[_b], False)) log(f"T0's c-part: twisted [{fz(T0t[0]):+.6f}, {fz(T0t[1]):+.6f}]; untwisted [{fz(T0u[0]):+.6f}, {fz(T0u[1]):+.6f}]") for i in range(1, MSTAR + 1): ct, cu = coupling_at0(i, H[i]) log(f"T0's H{i} part: twisted [{fz(ct[0]):+.6f}, {fz(ct[1]):+.6f}]; untwisted [{fz(cu[0]):+.6f}, {fz(cu[1]):+.6f}] (must contain 0)") T0t = add_iv(T0t, ct); T0u = add_iv(T0u, cu) log(f"untwisted T0 (actual data): [{fz(T0u[0]):+.4e}, {fz(T0u[1]):+.4e}] (must contain 0)") # ---------------------------------------------------------------- adjoint T0 (env T0ADJ=1; DESIGN_SO addendum 5): with L H = -A c (the forcing), # sum_m = -sum_m where L* lambda = kappa is solved bottom-up: lambda_m = (kappa_m - sum_{j coupling_at0(i, H) (twisted). The A-part widths then enter once, weighted by |lambda|. T0ADJ = os.environ.get('T0ADJ', '0') == '1' def val_ss(f, g): """interval of int f g, both signed DPLs: f against g+ and g- separately (each by the exact half-hat dot products of val_sn)""" gpl, gph, gml, gmh = _pos_lo(g.la, g.lb), _pos_hi(g.ua, g.ub), _neg_lo(g.ua, g.ub), _neg_hi(g.la, g.lb) a = val_sn(f, D(gpl[0], gpl[1], gph[0], gph[1]), 'corr'); b_ = val_sn(f, D(gml[0], gml[1], gmh[0], gmh[1]), 'corr') return (a[0] - b_[1], a[1] - b_[0]) def _cf_tw(j, i): return lambda b, kw, br, m: coefK(j, i, b, kw, br + m, br, True) * wk(kw + br + m) def _slab_parts(j, i, b): combos = [cb for cb in ALLCOMBOS if cb[0] + cb[2] >= 1] cf = _cf_tw(j, i) return [cb for cb in combos if cf(b, *cb) != 0], cf def kappa_dpl(i): """representer of H -> C_{0,i}[H](0) (twisted) on u-cells [i iE, iA)""" lo_u = i * iE; cells = range(lo_u, iA) acc = d_from_cells([cell_mul(d_ends_rev(EB(i, True), M - u - 1), d_ends(Sst(i), u)) for u in cells], lo_u) for b in range(1, i + 1): cmb, cf = _slab_parts(0, i, b) if not cmb: continue bl = blocks(b * iE, iA); dtc = max(c1 - c0 for c0, c1 in bl) for (c0, c1) in bl: # Part A: chain exact at the block start Kb = d_trunc(Sst(b), c0, c1); K = Kb if b == i else conv_nn(Kb, Sst(i - b)) Q = dz() for cb in cmb: Q = d_add(Q, d_scale(conv_chain(cb[0], d_trunc(Rk(cb[1]), iH - c0, iH), cb[2]), cf(b, *cb))) acc = d_add(acc, d_from_cells([cell_mul(d_ends_rev(Q, M - u - 1), d_ends(K, u)) for u in cells], lo_u)) for br in sorted(set(cb[1] for cb in cmb)): # Part B: W * (G-range x S_{i-b}), all factors >= 0 W = W_dpl(b, br, bl) for cb in cmb: if cb[1] != br: continue Gl, Gu = g_range(cb[0], cb[2], dtc, 'corr') if b == i: kB = d_from_cells([cell_mul(d_ends(W, u), (Gl[0], Gl[0], Gu[0], Gu[0])) for u in cells], lo_u) else: S_ = Sst(i - b) kB = conv_nn(W, d_from_cells([cell_mul((Gl[y], Gl[y], Gu[y], Gu[y]), d_ends(S_, y)) for y in range(M)], 0)) acc = d_add(acc, d_scale(d_trunc(kB, lo_u, iA), cf(b, *cb))) return acc def coefK_prime(i, b, kw, ky, br, twisted): """the beta-empty part of coefK(0, i, ...): i1 = 0 small parts and kw = 0 rough parts outside the large part, i.e. the prime's own fragmentations inside the d = 1 row (its sign rho(1) is the prime's -f(1))""" if kw != 0: return 0 return rho_(1, twisted) * (-1) ** (i + ky + 1) * (-1) ** b * (-1) ** br def kappa_F(i): """representer of H -> Omega_i^K[H](1), f(1)'s H_i part (F1ADJ): minus the prime's part of kappa_dpl(i). kappa_dpl's EB term has no prime part (its vectors keep all i small parts outside the large part), so only the slab terms with kw = 0 and i1 = 0 enter; Parts A and B exactly as in kappa_dpl""" lo_u = i * iE; cells = range(lo_u, iA) acc = dz() cf = lambda b, kw, br, m: coefK_prime(i, b, kw, br + m, br, True) * wk(kw + br + m) combos = [cb for cb in ALLCOMBOS if cb[0] + cb[2] >= 1] for b in range(1, i + 1): cmb = [cb for cb in combos if cf(b, *cb) != 0] if not cmb: continue bl = blocks(b * iE, iA); dtc = max(c1 - c0 for c0, c1 in bl) for (c0, c1) in bl: # Part A: chain exact at the block start Kb = d_trunc(Sst(b), c0, c1); K = Kb if b == i else conv_nn(Kb, Sst(i - b)) Q = dz() for cb in cmb: Q = d_add(Q, d_scale(conv_chain(cb[0], d_trunc(Rk(cb[1]), iH - c0, iH), cb[2]), cf(b, *cb))) acc = d_add(acc, d_from_cells([cell_mul(d_ends_rev(Q, M - u - 1), d_ends(K, u)) for u in cells], lo_u)) for br in sorted(set(cb[1] for cb in cmb)): # Part B: W * (G-range x S_{i-b}), all factors >= 0 W = W_dpl(b, br, bl) for cb in cmb: if cb[1] != br: continue Gl, Gu = g_range(cb[0], cb[2], dtc, 'corr') if b == i: kB = d_from_cells([cell_mul(d_ends(W, u), (Gl[0], Gl[0], Gu[0], Gu[0])) for u in cells], lo_u) else: S_ = Sst(i - b) kB = conv_nn(W, d_from_cells([cell_mul((Gl[y], Gl[y], Gu[y], Gu[y]), d_ends(S_, y)) for y in range(M)], 0)) acc = d_add(acc, d_scale(d_trunc(kB, lo_u, iA), cf(b, *cb))) return d_neg(acc) def adj_coupling(j, i, lam): """C*_{j,i}[lam](u) on u-cells [(j+i) iE, iA) (twisted); lam lives on sigma-cells [j iE, iA)""" lo_u = (j + i) * iE; cells = range(lo_u, iA) LS = conv_sn(lam, Sst(i)) acc = d_from_cells([cell_mul(d_ends_rev(EB(j + i, True), M - u - 1), d_ends(LS, u)) for u in cells], lo_u) for b in range(1, i + 1): cmb, cf = _slab_parts(j, i, b) if not cmb: continue bl = blocks(b * iE, iA); dtc = max(c1 - c0 for c0, c1 in bl) for (c0, c1) in bl: Kb = d_trunc(Sst(b), c0, c1); K = Kb if b == i else conv_nn(Kb, Sst(i - b)) Q = dz() for cb in cmb: Q = d_add(Q, d_scale(conv_chain(cb[0], d_trunc(Rk(cb[1]), iH - c0, iH), cb[2]), cf(b, *cb))) LK = conv_sn(lam, K) acc = d_add(acc, d_from_cells([cell_mul(d_ends_rev(Q, M - u - 1), d_ends(LK, u)) for u in cells], lo_u)) Lam = lam if b == i else conv_sn(lam, Sst(i - b)) Lp, Lm = H_pm(Lam) for br in sorted(set(cb[1] for cb in cmb)): W = W_dpl(b, br, bl) for cb in cmb: if cb[1] != br: continue Gl, Gu = g_range(cb[0], cb[2], dtc, 'corr') GLp = d_from_cells([cell_mul((Gl[y], Gl[y], Gu[y], Gu[y]), d_ends(Lp, y)) for y in range(M)], 0) GLm = d_from_cells([cell_mul((Gl[y], Gl[y], Gu[y], Gu[y]), d_ends(Lm, y)) for y in range(M)], 0) gB = d_add(conv_nn(W, GLp), conv_nn(W, GLm), -1) acc = d_add(acc, d_scale(d_trunc(gB, lo_u, iA), cf(b, *cb))) return acc _T0_forward_for_json = T0t # audit fix 6 if T0ADJ: LAM = {}; T0adj = (0, 0) for _b in range(1, NBIN + 1): T0adj = add_iv(T0adj, MACH[_b].T0_c(DB[_b], True)) for m_ in range(1, MSTAR + 1): lo_c = m_ * iE; cells = range(lo_c, iA) rhs = kappa_dpl(m_) for j_ in range(1, m_): rhs = d_add(rhs, adj_coupling(j_, m_ - j_, LAM[j_]), -1) Bc = [d_ends_rev(Bk(m_, True), M - u - 1) for u in cells] LAM[m_] = d_from_cells([cell_neg_quot(d_ends(d_neg(rhs), u), b_) for u, b_ in zip(cells, Bc)], lo_c) Am = dz() for _b in range(1, NBIN + 1): if DB[_b]: Am = d_add(Am, MACH[_b].A_dpl(m_, True, DB[_b])) As = d_from_cells([d_ends_rev(Am, M - u - 1) for u in cells], lo_c) part = val_ss(LAM[m_], As) T0adj = add_iv(T0adj, (-part[1], -part[0])) log(f" adjoint T0: lambda_{m_} in [{fz(d_minlow(LAM[m_], lo_c, iA)):+.4f}, {fz(d_maxup(LAM[m_], lo_c, iA)):+.4f}] width max {fz(d_maxwidth(LAM[m_], lo_c, iA)):.3e}; " f"- = [{fz(-part[1]):+.6f}, {fz(-part[0]):+.6f}]") log(f"T0 forward [{fz(T0t[0]):+.6f}, {fz(T0t[1]):+.6f}]; T0 adjoint [{fz(T0adj[0]):+.6f}, {fz(T0adj[1]):+.6f}]") assert T0adj[0] <= T0t[1] and T0t[0] <= T0adj[1], "forward and adjoint T0 enclosures are disjoint: an engine error" T0t = (max(T0t[0], T0adj[0]), min(T0t[1], T0adj[1])) log(f"T0 used (intersection): [{fz(T0t[0]):+.6f}, {fz(T0t[1]):+.6f}]") F1c = (0, 0) for _b in range(1, NBIN + 1): F1c = add_iv(F1c, MACH[_b].f1_c(DB[_b])) log(f"f(1)'s c-part: [{fz(F1c[0]):+.6f}, {fz(F1c[1]):+.6f}]") F1cp = F1c OMK = {} for r in range(1, MSTAR + 1): OMK[r] = Omega_rK(H[r], r, KSET) part = d_at1(OMK[r]) log(f"f(1)'s H{r} part: [{fz(part[0]):+.6f}, {fz(part[1]):+.6f}]") F1c = add_iv(F1c, part) # ---------------------------------------------------------------- adjoint f(1) (env F1ADJ=1; DESIGN_SO_DEV_addendum7.md): f(1)'s H-part is # sum_m = -sum_m with L* nu = kappaF, solved bottom-up exactly as T0ADJ's lambda (same adj_coupling, same B). F1ADJ = os.environ.get('F1ADJ', '0') == '1' F1_forward_at_c_for_json = F1c # audit fix 4 F1_identity_checks_for_json = {} if F1ADJ: NU = {}; F1adj = F1cp for m_ in range(1, MSTAR + 1): lo_c = m_ * iE; cells = range(lo_c, iA) kF = kappa_F(m_) chk = val_ss(kF, H[m_]); fwd = d_at1(OMK[m_]) F1_identity_checks_for_json[str(m_)] = { # audit fix 4 'representer_H': [str(Fr(x, SCALE)) for x in chk], 'forward_H': [str(Fr(x, SCALE)) for x in fwd], } log(f" adjoint f(1): = [{fz(chk[0]):+.6f}, {fz(chk[1]):+.6f}] vs forward [{fz(fwd[0]):+.6f}, {fz(fwd[1]):+.6f}]") assert chk[0] <= fwd[1] and fwd[0] <= chk[1], "kappa_F does not represent f(1)'s H-part: an engine error" rhs = kF for j_ in range(1, m_): rhs = d_add(rhs, adj_coupling(j_, m_ - j_, NU[j_]), -1) Bc = [d_ends_rev(Bk(m_, True), M - u - 1) for u in cells] NU[m_] = d_from_cells([cell_neg_quot(d_ends(d_neg(rhs), u), b_) for u, b_ in zip(cells, Bc)], lo_c) Am = dz() for _b in range(1, NBIN + 1): if DB[_b]: Am = d_add(Am, MACH[_b].A_dpl(m_, True, DB[_b])) As = d_from_cells([d_ends_rev(Am, M - u - 1) for u in cells], lo_c) part = val_ss(NU[m_], As) F1adj = add_iv(F1adj, (-part[1], -part[0])) log(f" adjoint f(1): nu_{m_} in [{fz(d_minlow(NU[m_], lo_c, iA)):+.4f}, {fz(d_maxup(NU[m_], lo_c, iA)):+.4f}] width max {fz(d_maxwidth(NU[m_], lo_c, iA)):.3e}; " f"- = [{fz(-part[1]):+.6f}, {fz(-part[0]):+.6f}]") log(f"f(1) at c: forward [{fz(F1c[0]):+.6f}, {fz(F1c[1]):+.6f}]; adjoint [{fz(F1adj[0]):+.6f}, {fz(F1adj[1]):+.6f}]") assert F1adj[0] <= F1c[1] and F1c[0] <= F1adj[1], "forward and adjoint f(1) enclosures are disjoint: an engine error" F1c = (max(F1c[0], F1adj[0]), min(F1c[1], F1adj[1])) log(f"f(1) at c used (intersection): [{fz(F1c[0]):+.6f}, {fz(F1c[1]):+.6f}]") T0g = (0, 0); PHIG1 = (0, 0) for _b in range(1, NBIN + 1): T0g = add_iv(T0g, MACH[_b].T0_c(DG[_b], True)); PHIG1 = add_iv(PHIG1, MACH[_b].f1_c(DG[_b])) tiv = iv_div((-T0t[1], -T0t[0]), T0g) F1 = add_iv(F1c, iv_mul(tiv, PHIG1)) log(f"T0(c) in [{fz(T0t[0]):+.6f}, {fz(T0t[1]):+.6f}]; T0(g) in [{fz(T0g[0]):+.6f}, {fz(T0g[1]):+.6f}]; t in [{fz(tiv[0]):+.6f}, {fz(tiv[1]):+.6f}]") log(f"f(1) at c: [{fz(F1c[0]):+.6f}, {fz(F1c[1]):+.6f}]; on the segment: [{fz(F1[0]):+.6f}, {fz(F1[1]):+.6f}]") # ---------------------------------------------------------------- Step C: positivity (per-cell ranges: local first-order losses only) def realisable_free(j, k): if k == 0: return j * ETA < 1 < j * ALPHA return j * ETA + k * ALPHA < 1 < j * ALPHA + Fr(k, 2) def kset_of(kb): return kws_of(kb) free_min = 0; free_arg = None for j in range(0, int(1 / ETA) + 1): for k in range(0, 6): for bb in ([0] if k == 0 else range(1, NBIN + 1)): if not realisable_bin(j, k, bb): continue v = C3.get((j, k, bb), Fr(0)); lo = fl(v.numerator * SCALE, v.denominator) if (j, k, bb) in GDICT: w_g = GDICT[(j, k, bb)]; lo += min(0, fl(w_g.numerator * tiv[0], w_g.denominator), fl(w_g.numerator * tiv[1], w_g.denominator)) cands = [lo] if 1 <= j <= MSTAR and kin(k): w_ = wk(k) hlo = fl(w_.numerator * d_minlow(H[j], j * iE, iA), w_.denominator) if w_ > 0 else fl(w_.numerator * d_maxup(H[j], j * iE, iA), w_.denominator) cands = [lo + hlo] if j == 1 else [lo + hlo, lo] for x in cands: if x < free_min: free_min, free_arg = x, (j, k, bb) log(f"free min >= {fz(free_min):+.6f} at {free_arg}") _L0K = {} def L0K(kb): ks = tuple(kset_of(kb)) if ks not in _L0K: acc = dz() for (ky, w_) in ks: if ky >= 2 and (0, ky) in P: acc = d_add(acc, d_scale(phi(0, ky), w_)) _L0K[ks] = acc return _L0K[ks] _POK = {} def POK(kb): ks = tuple(kset_of(kb)) if ks not in _POK: acc = dz() for r in range(1, MSTAR + 1): acc = d_add(acc, OMK[r] if ks == KSET else Omega_rK(H[r], r, ks)) _POK[ks] = acc return _POK[ks] def J_fnK(jb, i2, ks): """(j_b >= 1, k_b = 0) h-term with i2 >= 1 small fragments, per rough count, as a DPL in sigma_beta (slab terms via slab_corr: kw = 0)""" wd = dict(ks) combos = sorted(set((0, br, k - br) for (k, w_) in ks for br in range(1, k + 1) if not ((br >= 2 and (0, br) not in PS) or (k - br >= 2 and (0, k - br) not in PS)))) def cf(b, kw, br, m): k = br + m if kw != 0 or wd.get(k, 0) == 0: return 0 return (-1) ** (i2 + k + 1) * (-1) ** b * (-1) ** br * wd[k] return slab_corr(H[jb + i2], (jb + i2) * iE, i2, [cf], combos)[0] JK = {jb: {i2: J_fnK(jb, i2, kset_of(0)) for i2 in range(1, MSTAR - jb + 1)} for jb in range(1, MSTAR)} log("J functions ready:", {jb: list(JK[jb]) for jb in JK}) ext_min = 0; ext_arg = None; EXT = {} def upd(val, arg): global ext_min, ext_arg key = arg[:2] if key not in EXT or val[0] < EXT[key][0]: EXT[key] = (val[0], arg) if val[0] < ext_min: ext_min, ext_arg = val[0], arg def make_rq(j): lo_c = j * iE lo_ = [min(H[j].la[u], H[j].lb[u]) for u in range(lo_c, iA)]; hi_ = [max(H[j].ua[u], H[j].ub[u]) for u in range(lo_c, iA)] SPmin, SPmax = [lo_], [hi_] while (1 << len(SPmin)) <= len(lo_): kk = len(SPmin); h = 1 << (kk - 1); a_, b_ = SPmin[-1], SPmax[-1] SPmin.append([min(a_[i], a_[i + h]) for i in range(len(a_) - h)]); SPmax.append([max(b_[i], b_[i + h]) for i in range(len(b_) - h)]) def rq(l, r): out_lo = []; out_hi = [] if r >= iA: out_lo.append(0); out_hi.append(0); r = iA - 1 if l < lo_c: out_lo.append(0); out_hi.append(0); l = lo_c if l <= r: l2, r2 = l - lo_c, r - lo_c; kk = (r2 - l2 + 1).bit_length() - 1 out_lo.append(min(SPmin[kk][l2], SPmin[kk][r2 - (1 << kk) + 1])); out_hi.append(max(SPmax[kk][l2], SPmax[kk][r2 - (1 << kk) + 1])) return (min(out_lo), max(out_hi)) if out_lo else (0, 0) return rq RQ = {j: make_rq(j) for j in range(1, MSTAR + 1)} QB = {} def block_oms(jb, kb, c0, c1): """the Omega DPLs of sigma_beta block [c0, c1), i2 = 1 .. MSTAR - jb: Hsh carries H_{jb+i2}'s range over cells c0+i .. c1+i at shift cell i""" oms = [] for i2 in range(1, MSTAR - jb + 1): Hsh = d_const_cells([RQ[jb + i2](c0 + i, c1 + i) for i in range(i2 * iE, iA)], i2 * iE) oms.append(Omega_rK(Hsh, i2, kset_of(kb))) return oms def get_QB(jb, kb): key = (jb, tuple(kset_of(kb))) if key not in QB: lst = [] for (c0, c1) in blocksQ(jb * iE, iA, nbq_of(jb)): hr = RQ[jb](c0, c1 - 1) lst.append((c0, c1, hr, block_oms(jb, kb, c0, c1))) QB[key] = lst return QB[key] HBB = {}; HBG = {} for jb in range(0, int(1 / (2 * ETA)) + 1): for kb in range(0, 4): for b in range(1, NBIN + 1): HBB[(jb, kb, b)] = MACH[b].H_beta(jb, kb, DB[b]); HBG[(jb, kb, b)] = MACH[b].H_beta(jb, kb, DG[b]) def sum_d(lst): acc = None for x in lst: if x is None: continue acc = x if acc is None else d_add(acc, x) return acc POS2D = os.environ.get('POS2D', 'fast') def _sp(vals, op): tab = [vals] while (1 << len(tab)) <= len(vals): hh = 1 << (len(tab) - 1); a_ = tab[-1] tab.append([op(a_[i], a_[i + hh]) for i in range(len(a_) - hh)]) def q(l, r): k = (r - l + 1).bit_length() - 1 return op(tab[k][l], tab[k][r - (1 << k) + 1]) return q HRQ = {} for _j in range(1, MSTAR + 1): _Hj = H[_j] HRQ[_j] = (_sp([min(_Hj.la[u], _Hj.lb[u]) for u in range(M)], min), _sp([max(_Hj.ua[u], _Hj.ub[u]) for u in range(M)], max)) def min_aq(n, L0_, L1_): """lower bound of min over a in a-cell n of a * L(a), L the linear function with (scaled) end values L0_, L1_ (a > 0): the endpoint values minus the convexity slack (h^2/8) * 2 L' = (h/4) max(L1 - L0, 0)""" return min(fl(n * L0_, M), fl((n + 1) * L1_, M)) - ce(max(L1_ - L0_, 0), 4 * M) def t_ends(Hg, n): """lower bound ends of t * Hg on a-cell n, t in tiv (the min of linear functions is concave, so the chord through the end minima is below)""" if Hg is None: return (0, 0) out = [] for lo_, hi_ in ((Hg.la[n], Hg.ua[n]), (Hg.lb[n], Hg.ub[n])): out.append(min(fl(tv * hv, SCALE) for tv in tiv for hv in (lo_, hi_))) return tuple(out) def mulL(Hv, L0e, e): """lower bound of Hv * L0(a) at a-end e (Hv a constant)""" return fl(Hv * (L0e[e] if Hv >= 0 else L0e[2 + e]), SCALE) LEAVES = {} def ext_blockouter(jb, kb): """addendum 8 (B)/(C): the 'elif hasH' branch of the loop below (its prelude and its POS2D fast path) for one class (jb, kb) with sigma_beta blocks, evaluated block by block. Pass 1 stores the per-(bin, a-cell) prelude (and makes the 'noH' updates); pass 2 computes one block's Omegas at a time and commits the block's minimum (only leaves, under QADAPT). Returns [(c0, c1, depth, value or None)].""" assert POS2D == 'fast', 'QOUTER needs the POS2D fast path' L0b = L0K(kb); rqmin, rqmax = HRQ[jb]; pre = [] for bbeta in range(1, NBIN + 1): tlo, thi = class_total_range(jb, kb, bbeta) if tlo >= Fr(1, 2): continue Hb = sum_d([HBB[(jb, kb, b)] for b in range(max(bbeta, 1), NBIN + 1)]) Hg = sum_d([HBG[(jb, kb, b)] for b in range(max(bbeta, 1), NBIN + 1)]) for n in range(iH, M): tau_lo, tau_hi = 1 - Fr(n + 1, M), 1 - Fr(n, M) if tau_hi <= tlo or tau_lo >= thi: continue tg = t_ends(Hg, n) b0 = (Hb.la[n] if Hb is not None else 0) + tg[0]; b1 = (Hb.lb[n] if Hb is not None else 0) + tg[1] sg_lo = max(jb * ETA, tau_lo - kb * EDGES[bbeta]); sg_hi = min(jb * ALPHA, tau_hi - EDGES[bbeta - 1] - (kb - 1) * ALPHA) if sg_lo > sg_hi: continue if sg_hi > ALPHA: upd((min_aq(n, b0, b1),), (jb, kb, n / M, 'noH')) L0e = d_ends(L0b, n) u_lo = max(jb * iE, int(sg_lo * M) - 1); u_hi = min(iA - 1, int(min(sg_hi, ALPHA) * M)) while u_lo <= u_hi and Fr(u_lo + 1, M) < sg_lo: u_lo += 1 while u_hi >= u_lo and Fr(u_hi, M) > min(sg_hi, ALPHA): u_hi -= 1 if u_lo > u_hi: continue pre.append((bbeta, n, b0, b1, L0e, u_lo, u_hi)) if not pre: return [] def eval_block(c0, c1): if not any(max(e[5], c0) <= min(e[6], c1 - 1) for e in pre): return None # no part: the n-outer loop makes no update here either oms = block_oms(jb, kb, c0, c1); best = None for (bbeta, n, b0, b1, L0e, u_lo, u_hi) in pre: p0, p1 = max(u_lo, c0), min(u_hi, c1 - 1) if p0 > p1: continue hmn, hmx = rqmin(p0, p1), rqmax(p0, p1) L0_ = b0 + min(mulL(hmn, L0e, 0), mulL(hmx, L0e, 0)); L1_ = b1 + min(mulL(hmn, L0e, 1), mulL(hmx, L0e, 1)) for om in oms: L0_ += om.la[n]; L1_ += om.lb[n] v = min_aq(n, L0_, L1_) if best is None or v < best[0]: best = (v, (jb, kb, n / M, p0 / M, 'b%d' % bbeta)) return best ad = QADAPT.get((jb, kb)); leaves = [] roots = blocksQ(jb * iE, iA, nbq_of(jb)) assert roots[0][0] == jb * iE and roots[-1][1] == iA and all(a < b for (a, b) in roots) and \ all(roots[i][1] == roots[i + 1][0] for i in range(len(roots) - 1)), 'sigma_beta base blocks do not tile [jb iE, iA): an engine error' for (r0, r1) in roots: stack = [(r0, r1, 0)]; mine = [] while stack: c0, c1, d = stack.pop() best = eval_block(c0, c1) if ad is not None and best is not None and best[0] < ad[0] and d < ad[2] and c1 - c0 > 1: for (s0, s1) in reversed(blocksQ(c0, c1, ad[1])): stack.append((s0, s1, d + 1)) # refine: the parent is not committed continue if best is not None: upd((best[0],), best[1]) mine.append((c0, c1, d, None if best is None else best[0])) mine.sort(key=lambda x: (x[0], x[1])) # check: the committed leaves tile [r0, r1) exactly assert mine and mine[0][0] == r0 and mine[-1][1] == r1 and all(x[0] < x[1] for x in mine) and \ all(mine[i][1] == mine[i + 1][0] for i in range(len(mine) - 1)), f'leaves of block [{r0}, {r1}) do not tile it: an engine error' if ad is not None: log(f" block-outer {(jb, kb)} block [{r0}, {r1}): {len(mine)} leaves") leaves.extend(mine) return leaves for jb in range(0, int(1 / (2 * ETA)) + 1): for kb in range(0, 4): QB.pop((jb, tuple(kset_of(kb - 1))), None) if kb >= 1 else None # memory: drop the previous k_b's sigma-block Omegas (one (j_b, k_b) cached at a time) if QOUTER and 1 <= jb < MSTAR and kb >= 1 and len(kset_of(kb)) > 0: # (B)/(C): exactly the classes whose QBl is not None below LEAVES[(jb, kb)] = ext_blockouter(jb, kb); continue for bbeta in ([0] if kb == 0 else range(1, NBIN + 1)): if (jb, kb) == (0, 0): continue tlo, thi = class_total_range(jb, kb, bbeta) if tlo >= Fr(1, 2): continue Hb = sum_d([HBB[(jb, kb, b)] for b in range(max(bbeta, 1), NBIN + 1)]) Hg = sum_d([HBG[(jb, kb, b)] for b in range(max(bbeta, 1), NBIN + 1)]) L0b = L0K(kb); hasH = (jb >= 1 and jb <= MSTAR and len(kset_of(kb)) > 0) QBl = get_QB(jb, kb) if (hasH and kb >= 1 and jb < MSTAR) else None for n in range(iH, M): tau_lo, tau_hi = 1 - Fr(n + 1, M), 1 - Fr(n, M) if tau_hi <= tlo or tau_lo >= thi: continue tg = t_ends(Hg, n) b0 = (Hb.la[n] if Hb is not None else 0) + tg[0]; b1 = (Hb.lb[n] if Hb is not None else 0) + tg[1] if jb == 0: Pk = POK(kb); upd((min_aq(n, b0 + Pk.la[n], b1 + Pk.lb[n]),), (jb, kb, n / M)) elif hasH and kb == 0: ib = M - n - 1 if ib >= iA: upd((min_aq(n, b0, b1),), (jb, kb, n / M)); continue if ib < jb * iE: continue Hj = H[jb] pr = cell_mul((Hj.lb[ib], Hj.la[ib], Hj.ub[ib], Hj.ua[ib]), d_ends(L0b, n)) L0_, L1_ = b0 + pr[0], b1 + pr[1] for i2 in range(1, MSTAR - jb + 1): Jd = JK[jb][i2]; L0_ += Jd.lb[ib]; L1_ += Jd.la[ib] upd((min_aq(n, L0_, L1_),), (jb, kb, n / M, ib / M)) elif hasH: # beta's kb rough parts: the largest in bin bbeta, the others in [alpha', E[bbeta]], so their total lies in # [E[bbeta-1] + (kb-1) alpha', kb E[bbeta]]; with one bin this is the old [kb alpha', kb/2] sg_lo = max(jb * ETA, tau_lo - kb * EDGES[bbeta]); sg_hi = min(jb * ALPHA, tau_hi - EDGES[bbeta - 1] - (kb - 1) * ALPHA) if sg_lo > sg_hi: continue if sg_hi > ALPHA: upd((min_aq(n, b0, b1),), (jb, kb, n / M, 'noH')) L0e = d_ends(L0b, n); Hj = H[jb] u_lo = max(jb * iE, int(sg_lo * M) - 1); u_hi = min(iA - 1, int(min(sg_hi, ALPHA) * M)) while u_lo <= u_hi and Fr(u_lo + 1, M) < sg_lo: u_lo += 1 while u_hi >= u_lo and Fr(u_hi, M) > min(sg_hi, ALPHA): u_hi -= 1 if u_lo > u_hi: continue if POS2D == 'slow': for u in range(u_lo, u_hi + 1): Hv = (Hj.la[u], Hj.lb[u], Hj.ua[u], Hj.ub[u]) m0 = min(mulL(h_, L0e, 0) for h_ in Hv); m1 = min(mulL(h_, L0e, 1) for h_ in Hv) L0_, L1_ = b0 + m0, b1 + m1 if QBl is not None: for (c0, c1, hr, oms) in QBl: if c0 <= u < c1: for om in oms: L0_ += om.la[n]; L1_ += om.lb[n] break upd((min_aq(n, L0_, L1_),), (jb, kb, n / M, u / M, 'b%d' % bbeta)) continue # fast: per sigma-block, the H extremes over the sub-range (phi(h) = h * L0-bound is concave in h, so its minimum over a # range of H values sits at the range's extremes); the bracket is concave in sigma's H, so per a-end minima give a valid chord rqmin, rqmax = HRQ[jb] parts = [(u_lo, u_hi, ())] if QBl is None else \ [(max(u_lo, c0), min(u_hi, c1 - 1), oms) for (c0, c1, hr, oms) in QBl if max(u_lo, c0) <= min(u_hi, c1 - 1)] for (p0, p1, oms) in parts: hmn, hmx = rqmin(p0, p1), rqmax(p0, p1) L0_ = b0 + min(mulL(hmn, L0e, 0), mulL(hmx, L0e, 0)); L1_ = b1 + min(mulL(hmn, L0e, 1), mulL(hmx, L0e, 1)) for om in oms: L0_ += om.la[n]; L1_ += om.lb[n] upd((min_aq(n, L0_, L1_),), (jb, kb, n / M, p0 / M, 'b%d' % bbeta)) else: upd((min_aq(n, b0, b1),), (jb, kb, n / M)) for _key in sorted(LEAVES): # (B)/(C) _lv = LEAVES[_key]; _byd = {} for (_c0, _c1, _d, _v) in _lv: _byd[_d] = _byd.get(_d, 0) + 1 _vals = [_v for (_c0, _c1, _d, _v) in _lv if _v is not None] log(f" block-outer {_key}: {len(_lv)} leaves, by depth {dict(sorted(_byd.items()))}, min leaf " + (f"{fz(min(_vals)):+.6f}" if _vals else "none")) log(f"extended min >= {fz(ext_min):+.6f} at {ext_arg}") for key in sorted(EXT): log(f" extended min for (j_b, k_b) = {key}: {fz(EXT[key][0]):+.6f} at {EXT[key][1]}") m_ = min(free_min, ext_min) margin = 1 - Fr(-m_) / Fr(-F1[1]) if (m_ < 0 and F1[1] < 0) else Fr(-1) verdict = "PASS" if (F1[1] < 0 and m_ > F1[1] and all(v == 0 for v in untw_bad.values()) and T0u[0] <= 0 <= T0u[1]) else "FAIL" log(f"VERDICT {verdict}: min f >= {fz(m_):+.6f}, f(1)_hi = {fz(F1[1]):+.6f}, scaled margin >= {float(margin):.5f} (print rounding: quote the JSON value rounded down); untwisted bad {untw_bad}") json.dump(dict(M=M, NB=NB, tag=TAG_, engine='second-order DPL (gm2/cert2.py, development)', script_sha256=hashlib.sha256(open(__file__, 'rb').read()).hexdigest(), dpl_sha256=hashlib.sha256(open(dpl.__file__, 'rb').read()).hexdigest(), block_mode=BLOCKMODE, pos2d=POS2D, nbq=NBQ, nbqlo=NBQLO, t0adj=T0ADJ, f1adj=F1ADJ, maxs=MAXS, kprof=('all' if KPROF is None else sorted(KPROF)), kw=(None if KW is None else {str(k): str(v) for k, v in KW.items()}), eta=str(ETA), alpha=str(ALPHA), mstar=MSTAR, edges=[str(e) for e in EDGES], gcls=list(GCLS), gvec={str(k): str(v) for k, v in GDICT.items()}, pickle_sha256=PKL_SHA, c={str(kk): str(v) for kk, v in sorted(C3.items())}, T0=[str(Fr(x, SCALE)) for x in T0t], T0u=[str(Fr(x, SCALE)) for x in T0u], T0g=[str(Fr(x, SCALE)) for x in T0g], t=[str(Fr(x, SCALE)) for x in tiv], f1=[str(Fr(x, SCALE)) for x in F1], free_min=str(Fr(free_min, SCALE)), ext_min=str(Fr(ext_min, SCALE)), ext_arg=str(ext_arg), untwisted_bad=untw_bad, verdict=verdict, margin=str(margin), nbqj={str(k): v for k, v in sorted(NBQJ.items())}, qouter=QOUTER, # addendum 8 qadapt={f'{k[0]},{k[1]}': [str(Fr(v[0], SCALE)), v[1], v[2]] for k, v in sorted(QADAPT.items())}, leaves={f'{k[0]},{k[1]}': [[a, b, d, (None if v is None else str(Fr(v, SCALE)))] for (a, b, d, v) in LEAVES[k]] for k in sorted(LEAVES)}, T0_forward=[str(Fr(x, SCALE)) for x in _T0_forward_for_json], # audit fix 6 T0_adjoint=([str(Fr(x, SCALE)) for x in T0adj] if T0ADJ else None), free_bound_arg=str(free_arg), free_bound_domain='H over [j*eta,alpha] independently of class; no-H candidate for j>=2', gcls_legacy=bool(os.environ.get('GVEC')), F1_constant_part=[str(Fr(x, SCALE)) for x in F1cp], # audit fix 4 F1_forward_at_c=[str(Fr(x, SCALE)) for x in F1_forward_at_c_for_json], F1_adjoint_at_c=([str(Fr(x, SCALE)) for x in F1adj] if F1ADJ else None), F1_used_at_c=[str(Fr(x, SCALE)) for x in F1c], F1_identity_checks=F1_identity_checks_for_json), open(f'cert2_{TAG_}_M{M}.json', 'w'), indent=1) ~~~~~~~~~~~~ #### file: cert2_f3_diag.py - kind: script - as run: SHA-256 f960076234009c61147281e35b952d1ac75d9e68aded72a51ad1a6f74287d04b - as shipped: SHA-256 ecae1cc633092ca69849dabb13a960f16ed3d90dfe540f1194e515765c30532a - identity: identical, after dropping docstrings and comments, to the file as run with 2 declared change(s) of a machine-specific path, which now read: `sys.path.insert(0, __import__('os').path.dirname(__import__('os').path.abspath(__file__)))`; `open(f'cert2_{TAG_}_M{M}.json', 'x')` - trailing newline: yes ~~~~~~~~~~~~ python #!/usr/bin/env python # cert2_f3_diag.py: cert2_f3.py with the JSON output written to the working directory as a new file (mode 'x'). A diagnostic copy. import sys, os, time, json, hashlib, pickle from fractions import Fraction as Fr from math import comb sys.path.insert(0, __import__('os').path.dirname(__import__('os').path.abspath(__file__))) import dpl T_START = time.time() def log(*a): print(f"[{time.time() - T_START:7.1f}s]", *a, flush=True) M_, NB_, PKL_, TAG_ = sys.argv[1:5] M = int(M_); NB = int(NB_) dpl.init(M) from dpl import (D, SCALE, fl, ce, fz, dz, d_add, d_neg, d_scale, d_div, d_trunc, d_cell, d_ends, d_ends_rev, d_at1, d_at0, d_from_cells, d_const_cells, d_minlow, d_maxup, d_maxwidth, d_kernel, conv_nn, conv_sn, conv_ss, corr_sn, corr_nn, cell_mul, cell_neg_quot, cell_add, cell_contains0, iv_mul, iv_div, add_iv, iv_scale, _pos_lo, _pos_hi, _neg_lo, _neg_hi) raw = open(PKL_, 'rb').read(); PKL_SHA = hashlib.sha256(raw).hexdigest(); dpk = pickle.loads(raw) ETA, ALPHA = Fr(dpk['eta']), Fr(dpk['alpha']) assert (M * ETA).denominator == 1 and (M * ALPHA).denominator == 1 and M % 2 == 0 iE, iA, iH = int(M * ETA), int(M * ALPHA), M // 2 KPROF = None if os.environ.get('KPROF', 'all') == 'all' else set(int(x) for x in os.environ['KPROF'].split(',')) assert M > 0 and NB > 0 # audit fix 2 assert 0 < ETA < ALPHA < Fr(1, 2) assert 6 * ALPHA >= 1, "rough-count cutoff 5 is insufficient" assert 20 * ETA >= ALPHA, "the forced-level search range 1..19 of MSTAR is insufficient" # addendum 9 (was 10*ETA >= ALPHA, m* <= 9) MSTAR = max(j for j in range(1, 20) if j * ETA < ALPHA) log(f"cert2 (second order): M={M} NB={NB} NBQ={os.environ.get('NBQ', NB)} pickle sha256={PKL_SHA} tag={TAG_} KPROF={'all' if KPROF is None else sorted(KPROF)} KW={os.environ.get('KW')} eta={ETA} alpha'={ALPHA} m*={MSTAR}") # ---------------------------------------------------------------- bins and constants _ed = dpk.get('edges') if isinstance(_ed, str): _ed = [Fr(t) for t in _ed.split(',') if t.strip()] # some input pickles store the interior edges as text EDGES = [Fr(e) for e in _ed] if _ed else [ALPHA, Fr(1, 2)] if EDGES[-1] != Fr(1, 2): EDGES = EDGES + [Fr(1, 2)] if EDGES[0] != ALPHA: EDGES = [ALPHA] + [e for e in EDGES if e > ALPHA] assert EDGES[-1] == Fr(1, 2), EDGES assert len(EDGES) >= 2 and EDGES[0] == ALPHA assert all(a < b for a, b in zip(EDGES, EDGES[1:])), "unordered bin edges" NBIN = len(EDGES) - 1 for q in EDGES: assert (q * M).denominator == 1, f"M does not put {q} on the grid" iEdge = [int(e * M) for e in EDGES] def class_total_range(j, k, b): if k == 0: return (j * ETA, j * ALPHA) lo_b, hi_b = EDGES[b - 1], EDGES[b] return (j * ETA + (k - 1) * ALPHA + lo_b, j * ALPHA + k * hi_b) def realisable_bin(j, k, b): lo, hi = class_total_range(j, k, b); return lo < 1 < hi TEXTS = {'v2main': [(0, 3), (0, 4), (2, 2), (2, 3), (2, 4), (3, 2), (3, 3), (4, 1), (4, 2), (5, 1), (5, 2), (6, 1), (6, 2)]} TEXT = TEXTS.get(TAG_) C3 = {} for cls, x in zip(dpk['classes'], dpk['x']): j, k, b = (int(t) for t in cls) if abs(float(x)) == 0: continue if k == 0: b = 0 if NBIN == 1 and k >= 1: b = 1 if (k == 0 and b != 0) or (k >= 1 and not 1 <= b <= NBIN): continue if TEXT is not None and (j, k) not in TEXT: continue if not realisable_bin(j, k, b): continue assert (j, k, b) not in C3 C3[(j, k, b)] = Fr(float(x)) if TEXT is not None: assert sorted((j, k) for (j, k, b) in C3) == sorted(TEXT), sorted(C3) MAXS = max(9, max(j for (j, k, b) in C3), int(os.environ.get('MAXS', '0'))) # small-count power cutoff: every constant's small count, the forced levels, and env MAXS (oracle bases) assert MAXS >= MSTAR def abel(cdict): Dd = {b: {} for b in range(1, NBIN + 1)} for (j, k, b), v in cdict.items(): if k == 0: Dd[NBIN][(j, 0)] = Dd[NBIN].get((j, 0), 0) + v else: Dd[b][(j, k)] = Dd[b].get((j, k), 0) + v if b >= 2: Dd[b - 1][(j, k)] = Dd[b - 1].get((j, k), 0) - v return {b: {jk: v for jk, v in dd.items() if v != 0} for b, dd in Dd.items()} GBIN = int(os.environ.get('GBIN', str(NBIN))) GCLS = (0, 3, GBIN if NBIN > 1 else 1) # env GVEC 'b:w,b:w' = the T0 correction direction g = sum_b w_b e_(0,3,b) (H-neutral: no small parts); default e_GCLS GDICT = {(0, 3, int(a)): Fr(w) for a, w in (t.split(':') for t in os.environ['GVEC'].split(','))} if os.environ.get('GVEC') else {GCLS: Fr(1)} for _g in GDICT: assert realisable_bin(*_g), f'g class {_g} not realisable' DB = abel(C3); DG = abel(GDICT) log(f"bins: edges {[str(e) for e in EDGES]}; {len(C3)} constants; g = {GDICT}") log("c:", {kk: round(float(v), 6) for kk, v in sorted(C3.items())}) # ---------------------------------------------------------------- kernels (global: uncut rough kernel), powers, Lemma K kernels phi S1 = d_kernel(iE, iA); R1 = d_kernel(iA, iH) def build_powers(Rker): Pd = {}; PSd = {(1, 0): S1, (0, 1): Rker} for b in range(0, MAXS + 1): for bp in range(0, 6): if b + bp < 2 or b * ETA + bp * ALPHA > 1: continue if bp >= 1: prev, ker, div = (b, bp - 1), Rker, bp else: prev, ker, div = (b - 1, 0), S1, b if prev not in PSd: continue Pd[(b, bp)] = d_div(conv_nn(PSd[prev], ker), div); PSd[(b, bp)] = Pd[(b, bp)] return Pd, PSd P, PS = build_powers(R1) log(f"powers: {len(P)}") def Rk(k): return R1 if k == 1 else PS[(0, k)] def Sst(r): return PS[(r, 0)] class Phis: def __init__(self, Pd, PSd): self.P, self.PS, self.TR, self.PHI = Pd, PSd, {}, {} def trs(self, bb): if bb not in self.TR: self.TR[bb] = None if (sum(bb) < 2 or bb not in self.P) else d_trunc(self.P[bb], iH) return self.TR[bb] def phi(self, j, k): if (j, k) in self.PHI: return self.PHI[(j, k)] acc = dz() for bs in range(0, j + 1): for br in range(0, k + 1): if bs + br < 2 or (bs, br) not in self.P: continue sign = (-1) ** (j + k + 1) * (-1) ** (bs + br); comp = (j - bs, k - br) if comp == (0, 0): X = d_trunc(self.P[(bs, br)], iH) else: if comp not in self.PS: continue X = conv_nn(self.trs((bs, br)), self.PS[comp]) acc = d_add(acc, X, sign) self.PHI[(j, k)] = d_trunc(acc, iH); return self.PHI[(j, k)] def lin_phi(self, coef): acc = dz() for jk, cv in coef.items(): if cv != 0: acc = d_add(acc, d_scale(self.phi(*jk), cv)) return acc def H_beta(self, jb, kb, dd): coef = {} for (cj, ck), cv in dd.items(): ju, ku = cj - jb, ck - kb if ju >= 0 and ku >= 0 and ju + ku >= 2: coef[(ju, ku)] = coef.get((ju, ku), 0) + cv return self.lin_phi(coef) if coef else None GPH = Phis(P, PS) phi = GPH.phi def rsum(ms): s = dz() for m in ms: if m == 1: s = d_add(s, R1) elif (0, m) in P: s = d_add(s, P[(0, m)]) return s RALL = rsum([1, 2, 3, 4, 5]); EP = rsum([2, 4]); EM = rsum([1, 3, 5]) def E_conv(X, rho_c, twisted=True, nodelta=False): """E_rho^{(c)} * X: twisted (-1)^c (delta + E'+ - E'-) * X; untwisted (delta + R_all) * X""" if twisted: out = dz() if nodelta else X out = d_add(out, conv_sn(X, EP)); out = d_add(out, conv_sn(X, EM), -1) return d_neg(out) if (rho_c or 0) % 2 == 1 else out out = dz() if nodelta else X return d_add(out, conv_sn(X, RALL)) log("kernels ready") # exact values of (f*g)(1) and int f g, without building the whole convolution (no curvature needed at a grid point) def _N_at(F, G, k, mode): a, b = F; c, d = G; tot = 0 if mode == 'conv': # n + m = k - 1 for n in range(max(0, k - M), min(M, k)): m = k - 1 - n if m < 0 or m >= M: continue an, bn, cm, dm = a[n], b[n], c[m], d[m] if an or bn: tot += (an * cm + bn * dm) + 2 * (an * dm + bn * cm) else: # corr at s = 0: n = m for n in range(M): an, bn, cm, dm = a[n], b[n], c[n], d[n] if (an or bn) and (cm or dm): tot += 2 * (an * cm + bn * dm) + (an * dm + bn * cm) return tot def val_sn(f, g, mode): """interval of (f*g)(1) (mode 'conv') or int f g (mode 'corr'), f signed, g >= 0""" S6 = 6 * M * SCALE glo = _pos_lo(g.la, g.lb); ghi = _pos_hi(g.ua, g.ub) lo = fl(_N_at(_pos_lo(f.la, f.lb), glo, M, mode), S6) - ce(_N_at(_neg_hi(f.la, f.lb), ghi, M, mode), S6) hi = ce(_N_at(_pos_hi(f.ua, f.ub), ghi, M, mode), S6) - fl(_N_at(_neg_lo(f.ua, f.ub), glo, M, mode), S6) return (lo, hi) # ---------------------------------------------------------------- one bin machine per edge (rough kernel cut at the edge) class Mach(Phis): def __init__(self, ecell): if ecell == iH: Pd, PSd = P, PS else: Pd, PSd = build_powers(d_kernel(iA, ecell)) Phis.__init__(self, Pd, PSd) if ecell == iH: self.PHI = GPH.PHI; self.TR = GPH.TR def f1_c(self, dd): acc = (0, 0) for jk, cv in dd.items(): if sum(jk) >= 2: acc = add_iv(acc, iv_scale(d_at1(self.phi(*jk)), cv)) return acc def T0_c(self, dd, twisted): rho = (lambda d: (-1) ** d) if twisted else (lambda d: 1) acc = (0, 0) if not dd: return acc for (j, k), cv in dd.items(): if (j, k) in self.P: acc = add_iv(acc, iv_scale(d_at1(self.P[(j, k)]), cv * rho(j + k))) if j + k >= 2: acc = add_iv(acc, iv_scale(d_at1(self.phi(j, k)), cv * rho(1))) jmax = max(j for j, k in dd); kmax = max(k for j, k in dd) for jb in range(0, jmax + 1): for kb in range(0, kmax + 1): if (jb, kb) == (0, 0) or (jb, kb) not in self.PS: continue Hb = self.H_beta(jb, kb, dd) if Hb is None: continue acc = add_iv(acc, iv_scale(val_sn(d_trunc(Hb, iH), self.PS[(jb, kb)], 'conv'), rho(1 + jb + kb))) return acc def A_dpl(self, base, twisted, dd): """A_base(x).c as a DPL in x = 1 - sigma (free part + extended part), DESIGN_GM""" rho = (lambda d: (-1) ** d) if twisted else (lambda d: 1) A = dz() if not dd: return A for (cj, ck), cv in dd.items(): if cj >= base: jz, kz = cj - base, ck if jz + kz >= 2 and (jz, kz) in self.P: A = d_add(A, d_scale(self.P[(jz, kz)], cv * rho(base + jz + kz))) jmax = max(j for j, k in dd); kmax = max(k for j, k in dd) for jp in range(0, jmax + 1): for kp in range(0, kmax + 1): coef = {} for (cj, ck), cv in dd.items(): ju, ku = cj - base - jp, ck - kp if ju >= 0 and ku >= 0 and ju + ku >= 2: coef[(ju, ku)] = coef.get((ju, ku), 0) + cv if not coef: continue Gk = self.lin_phi(coef) if (jp, kp) == (0, 0): X = Gk else: if (jp, kp) not in self.PS: continue X = conv_sn(d_trunc(Gk, iH), self.PS[(jp, kp)]) A = d_add(A, d_scale(X, rho(base + 1 + jp + kp))) return A MACH = {b: Mach(iEdge[b]) for b in range(1, NBIN + 1)} log(f"bin machines ready: {NBIN}") # ---------------------------------------------------------------- rough-count profile: B_{j,K}, brackets, slab chains KALL = list(range(2, 6)) # weighted rough-count profile (env KW, e.g. '2:1,3:-1'): H_j(sigma, k) = w_k h_j(sigma); KPROF = the support of KW. Unset: KPROF as before (w = 1). KW_ENV = os.environ.get('KW') KW = {int(a): Fr(b) for a, b in (t.split(':') for t in KW_ENV.split(','))} if KW_ENV else None if KW is not None: KPROF = set(k for k, v in KW.items() if v != 0) def wk(k): if KW is not None: return KW.get(k, Fr(0)) return Fr(1) if (KPROF is None or k in KPROF) else Fr(0) def kin(k): return wk(k) != 0 def kws_of(kb): """fragment rough counts ky with the weight w(kb + ky) of the completed vector's total rough count (KPROF all, no KW: ky = 1..5, w = 1)""" if KPROF is None and KW is None: return tuple((ky, Fr(1)) for ky in range(1, 6)) return tuple((k - kb, wk(k)) for k in range(1, 6) if wk(k) != 0 and k - kb >= 1) KWS = tuple((k, wk(k)) for k in KALL if wk(k) != 0) # f(1)'s Omega: rough counts >= 2, no beta def rho_(d, twisted): return (-1) ** d if twisted else 1 _RPHI = {} def RPHI(kw, ky): if (kw, ky) not in _RPHI: if (0, ky) not in P: _RPHI[(kw, ky)] = None elif kw == 0: _RPHI[(kw, ky)] = phi(0, ky) elif (0, kw) not in PS: _RPHI[(kw, ky)] = None else: _RPHI[(kw, ky)] = conv_sn(d_trunc(phi(0, ky), iH), PS[(0, kw)]) return _RPHI[(kw, ky)] _BK = {} def Bk(j, twisted): key = ('B', j % 2 if twisted else 0, twisted) if key not in _BK: acc = dz() for k in KALL: if not kin(k): continue if (0, k) in P: acc = d_add(acc, d_scale(P[(0, k)], wk(k) * rho_(j + k, twisted))) for kw in range(0, k - 1): X = RPHI(kw, k - kw) if X is None: continue acc = d_add(acc, d_scale(X, wk(k) * rho_(j + kw + 1, twisted))) _BK[key] = acc return _BK[key] def EB(c, twisted): """free-completion bracket sum_{k in K} rho(c+k) R_k + sum_{k in K} sum_{kw+ky=k, ky>=2} rho(c+1+kw) (R_kw * Phi_{0,ky}tr), as a DPL in x""" key = ('E', c % 2 if twisted else 0, twisted) if key not in _BK: acc = dz() for k in KALL: if not kin(k): continue if (0, k) in P: acc = d_add(acc, d_scale(P[(0, k)], wk(k) * rho_(c + k, twisted))) for kw in range(0, k - 1): X = RPHI(kw, k - kw) if X is None: continue acc = d_add(acc, d_scale(X, wk(k) * rho_(c + 1 + kw, twisted))) _BK[key] = acc return _BK[key] def blocks(lo_c, hi_c): n_c = hi_c - lo_c; bl = max(1, -(-n_c // NB)) return [(c0, min(c0 + bl, hi_c)) for c0 in range(lo_c, hi_c, bl)] NBQ = int(os.environ.get('NBQ', str(NB))) # sigma_beta blocks in positivity's (j_b >= 1, k_b >= 1) Omega terms (first order there) NBQLO = int(os.environ.get('NBQLO', str(NBQ))) # finer sigma_beta blocks for j_b <= 2 (the binding (1,1), (2,1) classes at 1/4) def blocksQ(lo_c, hi_c, nbq=None): n_c = hi_c - lo_c; bl = max(1, -(-n_c // (nbq or NBQ))) return [(c0, min(c0 + bl, hi_c)) for c0 in range(lo_c, hi_c, bl)] NBQJ = {int(a): int(b) for a, b in (u.split(':') for u in os.environ['NBQJ'].split(','))} if os.environ.get('NBQJ') else {} # (A) assert all(v >= 1 for v in NBQJ.values()) def nbq_of(jb): return NBQJ.get(jb, NBQLO if jb <= 2 else NBQ) # (A): the sigma_beta block count for j_beta QOUTER = os.environ.get('QOUTER', '0') == '1' # (B) QADAPT = {} # (C): (jb, kb) -> (scaled target, split, depth) for _it in (os.environ['QADAPT'].split(';') if os.environ.get('QADAPT') else []): _cls, _tg, _sp, _dp = _it.split(':'); _jb, _kb = (int(x) for x in _cls.split(',')); _tf = Fr(_tg) QADAPT[(_jb, _kb)] = (fl(_tf.numerator * SCALE, _tf.denominator), int(_sp), int(_dp)) assert int(_sp) >= 2 and int(_dp) >= 0 assert QOUTER or not QADAPT, 'QADAPT needs QOUTER=1' def slab_br(br, t_lo, t_hi): Rb = Rk(br) return (d_trunc(Rb, iH - t_lo, iH), d_trunc(Rb, iH - t_hi, iH)) def conv_chain(kw, sl, m): cur = sl if kw >= 1: cur = conv_nn(cur, Rk(kw)) if m >= 1: cur = conv_nn(cur, Rk(m)) return cur def coefK(j, i, b, kw, ky, br, twisted): tot = 0 for i1 in range(0, i - b + 1): i2 = i - i1 s_out = rho_(j + i1 + kw + 1, twisted) tot += comb(i - b, i1) * s_out * (-1) ** (i2 + ky + 1) * (-1) ** b * (-1) ** br return tot def H_times_x(Hd, X, lo_u): """f(u) = H(u) X(1 - u) on u-cells [lo_u, iA) (X a DPL in x)""" return d_from_cells([cell_mul(d_ends(Hd, u), d_ends_rev(X, M - u - 1)) for u in range(lo_u, iA)], lo_u) def H_times_u(Hd, X, lo_u): """f(u) = H(u) X(u) on u-cells [lo_u, iA)""" return d_from_cells([cell_mul(d_ends(Hd, u), d_ends(X, u)) for u in range(lo_u, iA)], lo_u) # ---------------------------------------------------------------- refined threshold blocks (gm2/DESIGN_SO.md addendum 1): Part A exact at the block # start tau0, Part B the sawtooth rho(tau) = int_{tau0}^{tau} R_br(1/2 - t) dt times the range of g = R_kw * R_m on a window of width dtau. # BLOCKS=plain restores the narrow/wide (first-order) block bound for comparison. BLOCKMODE = os.environ.get('BLOCKS', 'refined') ALLCOMBOS = sorted(set((kw, br, k - kw - br) for k in KALL if kin(k) for kw in range(0, k) for br in range(1, k - kw + 1) if not ((br >= 2 and (0, br) not in PS) or (kw >= 2 and (0, kw) not in PS) or (k - kw - br >= 2 and (0, k - kw - br) not in PS)))) _GD = {} def g_dpl(kw, m): if (kw, m) not in _GD: if kw == 0 and m == 0: _GD[(kw, m)] = None elif kw == 0: _GD[(kw, m)] = Rk(m) elif m == 0: _GD[(kw, m)] = Rk(kw) else: _GD[(kw, m)] = conv_nn(Rk(kw), Rk(m)) return _GD[(kw, m)] def _sparse(vals, op): tab = [vals] while (1 << len(tab)) <= len(vals): h = 1 << (len(tab) - 1); a = tab[-1] tab.append([op(a[i], a[i + h]) for i in range(len(a) - h)]) def q(l, r): k = (r - l + 1).bit_length() - 1 return op(tab[k][l], tab[k][r - (1 << k) + 1]) return q _GR = {} def g_range(kw, m, dtc, kind): """(Gl, Gu) per cell n: the range of g = R_kw * R_m (>= 0) on the window of g-cells [iH-1-n-dtc, iH-1-n] (kind 'corr': argument 1/2 - y - s, s in [0, dtau], y in cell n) or [n-iH-dtc, n-iH] (kind 'conv': argument z - 1/2 - s, z in cell n); off-grid cells count as 0""" key = (kw, m, dtc, kind) if key in _GR: return _GR[key] g = g_dpl(kw, m) qlo = _sparse([min(a, b) for a, b in zip(g.la, g.lb)], min); qhi = _sparse([max(a, b) for a, b in zip(g.ua, g.ub)], max) Gl = [0] * M; Gu = [0] * M for n in range(M): c_lo, c_hi = (iH - 1 - n - dtc, iH - 1 - n) if kind == 'corr' else (n - iH - dtc, n - iH) vals_lo = []; vals_hi = [] if c_lo < 0 or c_hi >= M: vals_lo.append(0); vals_hi.append(0) a_, b_ = max(c_lo, 0), min(c_hi, M - 1) if a_ <= b_: vals_lo.append(qlo(a_, b_)); vals_hi.append(qhi(a_, b_)) Gl[n] = max(0, min(vals_lo)); Gu[n] = max(0, max(vals_hi)) _GR[key] = (Gl, Gu) return _GR[key] def rho_dpl(br, bl): """rho(tau) = int_{tau0(block)}^{tau} R_br(1/2 - t) dt on the tau-cells of the blocks bl, as a DPL (linear bounds from the min/max of R_br's enclosure on the mirrored cell; the cumulative sums are exact integrals of R_br's linear bounds)""" Rb = Rk(br); la = [0] * M; lb = [0] * M; ua = [0] * M; ub = [0] * M for (c0, c1) in bl: acc_lo = acc_hi = 0 for c in range(c0, c1): n = iH - 1 - c rl = max(0, min(Rb.la[n], Rb.lb[n])); ru = max(Rb.ua[n], Rb.ub[n]) la[c] = acc_lo; lb[c] = acc_lo + fl(rl, M); ua[c] = acc_hi; ub[c] = acc_hi + ce(ru, M) acc_lo += fl(max(Rb.la[n] + Rb.lb[n], 0), 2 * M) # audit fix 1 acc_hi += ce(Rb.ua[n] + Rb.ub[n], 2 * M) return D(la, lb, ua, ub) def W_dpl(b, br, bl): rho = rho_dpl(br, bl); S = Sst(b); lo_c, hi_c = bl[0][0], bl[-1][1] return d_from_cells([cell_mul(d_ends(S, c), d_ends(rho, c)) for c in range(lo_c, hi_c)], lo_c) def H_pm(Hd): lp = _pos_lo(Hd.la, Hd.lb); up = _pos_hi(Hd.ua, Hd.ub); ln = _neg_lo(Hd.ua, Hd.ub); un = _neg_hi(Hd.la, Hd.lb) return D(lp[0], lp[1], up[0], up[1]), D(ln[0], ln[1], un[0], un[1]) def _gI(G, Ip, Im, c): """c * [Gl I+ - Gu I-, Gu I+ - Gl I-] per cell (G constant per cell, I+- >= 0 DPLs), as a DPL""" Gl, Gu = G lo_a = [fl(gl * ipa - gu * ima, SCALE) for gl, gu, ipa, ima in zip(Gl, Gu, Ip.la, Im.ua)] lo_b = [fl(gl * ipb - gu * imb, SCALE) for gl, gu, ipb, imb in zip(Gl, Gu, Ip.lb, Im.ub)] hi_a = [ce(gu * ipa - gl * ima, SCALE) for gl, gu, ipa, ima in zip(Gl, Gu, Ip.ua, Im.la)] hi_b = [ce(gu * ipb - gl * imb, SCALE) for gl, gu, ipb, imb in zip(Gl, Gu, Ip.ub, Im.lb)] X = D(lo_a, lo_b, hi_a, hi_b) return d_scale(X, c) def slab_corr(Hd, lo_u, i, coefs, combos, at0=False): """sum_b sum_combos coef(b, kw, br, m) * int int H(u) X_tau(1 - u) S_b(tau) S_{i-b}(u - tau - sigma) dtau du for each coefficient function in coefs; DPLs in sigma (or intervals at sigma = 0 if at0). kw = m = 0 terms vanish here (argument 1 - u > 1/2 is outside the slab).""" combos = [cb for cb in combos if cb[0] + cb[2] >= 1] res = [((0, 0) if at0 else dz()) for _ in coefs] def acc_(o, X): if at0: res[o] = add_iv(res[o], X) else: res[o] = d_add(res[o], X) for b in range(1, i + 1): bl = blocks(b * iE, iA); dtc = max(c1 - c0 for c0, c1 in bl) cmb = [cb for cb in combos if any(cf(b, *cb) != 0 for cf in coefs)] if not cmb: continue for (c0, c1) in bl: Kb = d_trunc(Sst(b), c0, c1); K = Kb if b == i else conv_nn(Kb, Sst(i - b)) if BLOCKMODE == 'refined': X0 = {cb: conv_chain(cb[0], d_trunc(Rk(cb[1]), iH - c0, iH), cb[2]) for cb in cmb} for o, cf in enumerate(coefs): Q = dz() for cb, X in X0.items(): cv = cf(b, *cb) if cv: Q = d_add(Q, d_scale(X, cv)) f = H_times_x(Hd, Q, lo_u) acc_(o, val_sn(f, K, 'corr') if at0 else corr_sn(f, K)) else: ch = {} for cb in cmb: sn, sw = slab_br(cb[1], c0, c1); ch[cb] = (conv_chain(cb[0], sn, cb[2]), conv_chain(cb[0], sw, cb[2])) for o, cf in enumerate(coefs): Q = dz() for cb, (Xn, Xw) in ch.items(): cv = cf(b, *cb) if cv: Q = d_add(Q, d_scale(D(Xn.la, Xn.lb, Xw.ua, Xw.ub), cv)) # d_scale swaps the sides when cv < 0 f = H_times_x(Hd, Q, lo_u) acc_(o, val_sn(f, K, 'corr') if at0 else corr_sn(f, K)) if BLOCKMODE != 'refined': continue Hp, Hm = H_pm(Hd) for br in sorted(set(cb[1] for cb in cmb)): W = W_dpl(b, br, bl) Ip, Im = corr_nn(Hp, W), corr_nn(Hm, W) for o, cf in enumerate(coefs): PB = None for cb in cmb: if cb[1] != br: continue cv = cf(b, *cb) if not cv: continue t_ = _gI(g_range(cb[0], cb[2], dtc, 'corr'), Ip, Im, cv) PB = t_ if PB is None else d_add(PB, t_) if PB is None: continue if b < i: acc_(o, val_sn(PB, Sst(i - b), 'corr') if at0 else corr_sn(PB, Sst(i - b))) else: acc_(o, (PB.la[0], PB.ua[0]) if at0 else PB) return res def coupling(j, i, Hd): """(twisted, untwisted) C_{j,i}[H_{j+i}](sigma) as DPLs in sigma (DESIGN_GM): free + i2 = 0 part, then the i2 >= 1 slab terms""" lo_u = (j + i) * iE res = [corr_sn(H_times_x(Hd, EB(j + i, tw), lo_u), Sst(i)) for tw in (True, False)] ext = slab_corr(Hd, lo_u, i, [lambda b, kw, br, m, tw=tw: coefK(j, i, b, kw, br + m, br, tw) * wk(kw + br + m) for tw in (True, False)], ALLCOMBOS) return d_add(res[0], ext[0]), d_add(res[1], ext[1]) def coupling_at0(i, Hd): """(twisted, untwisted) C_{0,i}[H_i] at sigma = 0, as intervals""" lo_u = i * iE res = [val_sn(H_times_x(Hd, EB(i, tw), lo_u), Sst(i), 'corr') for tw in (True, False)] ext = slab_corr(Hd, lo_u, i, [lambda b, kw, br, m, tw=tw: coefK(0, i, b, kw, br + m, br, tw) * wk(kw + br + m) for tw in (True, False)], ALLCOMBOS, at0=True) return add_iv(res[0], ext[0]), add_iv(res[1], ext[1]) # ---------------------------------------------------------------- Step A: top-down forcing H = {}; untw_bad = {} for j in range(MSTAR, 0, -1): lo_c = j * iE; cells = range(lo_c, iA) At = dz(); Au = dz() for b in range(1, NBIN + 1): At = d_add(At, MACH[b].A_dpl(j, True, DB[b])); Au = d_add(Au, MACH[b].A_dpl(j, False, DB[b])) Btw, Bun = Bk(j, True), Bk(j, False) num = [d_ends_rev(At, M - u - 1) for u in cells] bad = sum(1 for u in cells if not cell_contains0(d_ends_rev(Au, M - u - 1))) + sum(1 for u in cells if not cell_contains0(d_ends_rev(Bun, M - u - 1))) wA = max(max(x[2] - x[0], x[3] - x[1]) for x in num) for i in range(1, MSTAR - j + 1): Ct, Cu = coupling(j, i, H[j + i]) bu = sum(1 for u in cells if not cell_contains0(d_ends(Cu, u))) bad += bu num = [cell_add(x, d_ends(Ct, u)) for x, u in zip(num, cells)] log(f" T{j}: coupling to H{j+i}: twisted width max {fz(d_maxwidth(Ct, lo_c, iA)):.3e}; untwisted cells not containing 0: {bu}/{len(cells)}") Bc = [d_ends_rev(Btw, M - u - 1) for u in cells] zc = [u for u, bb in zip(cells, Bc) if not ((bb[0] > 0 and bb[1] > 0) or (bb[2] < 0 and bb[3] < 0))] if zc: log(f"T{j}: B may vanish on {len(zc)} cells, sigma in [{zc[0]/M:.5f}, {zc[-1]/M:.5f}]"); sys.exit(2) Hc = [cell_neg_quot(n_, b_) for n_, b_ in zip(num, Bc)] H[j] = d_from_cells(Hc, lo_c); untw_bad[j] = bad log(f"T{j}: B in [{fz(min(min(x[0], x[1]) for x in Bc)):+.4f}, {fz(max(max(x[2], x[3]) for x in Bc)):+.4f}]; " f"H{j} in [{fz(d_minlow(H[j], lo_c, iA)):+.4f}, {fz(d_maxup(H[j], lo_c, iA)):+.4f}]; A-part width max {fz(wA):.3e}; " f"H width max {fz(d_maxwidth(H[j], lo_c, iA)):.3e}; untwisted A/B/C cell ends not enclosing 0: {bad}") # ---------------------------------------------------------------- Step B: T0 and f(1) def Omega_rK(Hd, r, kws): """Omega^K_r[H](a) as a DPL in a (DESIGN_GM addendum 2; b = r term by the per-br slab identity; 1 <= b <= r-1 terms by blocks: refined (Part A at tau0 + Part B, DESIGN_SO addendum 1) for m >= 1, narrow/wide for m = 0)""" out = dz() HS = H_times_u(Hd, Sst(r), r * iE) Hp = Hm = None for (k, wgt) in kws: if k < 1 or wgt == 0: continue for br in range(1, k + 1): m = k - br if (br >= 2 and (0, br) not in PS) or (m >= 2 and (0, m) not in PS): continue X = d_trunc(conv_sn(HS, d_trunc(Rk(br), 0, iH)), iH) if m >= 1: X = conv_sn(X, Rk(m)) out = d_add(out, d_scale(X, wgt * (-1) ** (r + k + 1) * (-1) ** r * (-1) ** br)) for b in range(1, r): sgb = wgt * (-1) ** (r + k + 1) * (-1) ** b * (-1) ** br bl = blocks(b * iE, iA); dtc = max(c1 - c0 for c0, c1 in bl) if m == 0 or BLOCKMODE != 'refined': for (c0, c1) in bl: Kk = conv_nn(d_trunc(Sst(b), c0, c1), Sst(r - b)) w = H_times_u(Hd, Kk, r * iE) sn, sw = slab_br(br, c0, c1) Xn, Xw = conv_chain(0, sn, m), conv_chain(0, sw, m) out = d_add(out, d_scale(conv_ss(w, D(Xn.la, Xn.lb, Xw.ua, Xw.ub)), sgb)) continue for (c0, c1) in bl: # Part A: the chain exact at tau0 = c0 Kk = conv_nn(d_trunc(Sst(b), c0, c1), Sst(r - b)) w = H_times_u(Hd, Kk, r * iE) out = d_add(out, d_scale(conv_sn(w, conv_chain(0, d_trunc(Rk(br), iH - c0, iH), m)), sgb)) if Hp is None: Hp, Hm = H_pm(Hd) # Part B: S_{r-b} J+- convolved with the window range of R_m W = W_dpl(b, br, bl) Jp, Jm = corr_nn(Hp, W), corr_nn(Hm, W) S_ = Sst(r - b) Ap = d_from_cells([cell_mul(d_ends(S_, y), d_ends(Jp, y)) for y in range(M)], 0) Am = d_from_cells([cell_mul(d_ends(S_, y), d_ends(Jm, y)) for y in range(M)], 0) Gl, Gu = g_range(0, m, dtc, 'conv') Gd = D(list(Gl), list(Gl), list(Gu), list(Gu)) out = d_add(out, d_scale(d_add(conv_nn(Ap, Gd), conv_nn(Am, Gd), -1), sgb)) return out KSET = KWS T0t = (0, 0); T0u = (0, 0) for _b in range(1, NBIN + 1): T0t = add_iv(T0t, MACH[_b].T0_c(DB[_b], True)); T0u = add_iv(T0u, MACH[_b].T0_c(DB[_b], False)) log(f"T0's c-part: twisted [{fz(T0t[0]):+.6f}, {fz(T0t[1]):+.6f}]; untwisted [{fz(T0u[0]):+.6f}, {fz(T0u[1]):+.6f}]") for i in range(1, MSTAR + 1): ct, cu = coupling_at0(i, H[i]) log(f"T0's H{i} part: twisted [{fz(ct[0]):+.6f}, {fz(ct[1]):+.6f}]; untwisted [{fz(cu[0]):+.6f}, {fz(cu[1]):+.6f}] (must contain 0)") T0t = add_iv(T0t, ct); T0u = add_iv(T0u, cu) log(f"untwisted T0 (actual data): [{fz(T0u[0]):+.4e}, {fz(T0u[1]):+.4e}] (must contain 0)") # ---------------------------------------------------------------- adjoint T0 (env T0ADJ=1; DESIGN_SO addendum 5): with L H = -A c (the forcing), # sum_m = -sum_m where L* lambda = kappa is solved bottom-up: lambda_m = (kappa_m - sum_{j coupling_at0(i, H) (twisted). The A-part widths then enter once, weighted by |lambda|. T0ADJ = os.environ.get('T0ADJ', '0') == '1' def val_ss(f, g): """interval of int f g, both signed DPLs: f against g+ and g- separately (each by the exact half-hat dot products of val_sn)""" gpl, gph, gml, gmh = _pos_lo(g.la, g.lb), _pos_hi(g.ua, g.ub), _neg_lo(g.ua, g.ub), _neg_hi(g.la, g.lb) a = val_sn(f, D(gpl[0], gpl[1], gph[0], gph[1]), 'corr'); b_ = val_sn(f, D(gml[0], gml[1], gmh[0], gmh[1]), 'corr') return (a[0] - b_[1], a[1] - b_[0]) def _cf_tw(j, i): return lambda b, kw, br, m: coefK(j, i, b, kw, br + m, br, True) * wk(kw + br + m) def _slab_parts(j, i, b): combos = [cb for cb in ALLCOMBOS if cb[0] + cb[2] >= 1] cf = _cf_tw(j, i) return [cb for cb in combos if cf(b, *cb) != 0], cf def kappa_dpl(i): """representer of H -> C_{0,i}[H](0) (twisted) on u-cells [i iE, iA)""" lo_u = i * iE; cells = range(lo_u, iA) acc = d_from_cells([cell_mul(d_ends_rev(EB(i, True), M - u - 1), d_ends(Sst(i), u)) for u in cells], lo_u) for b in range(1, i + 1): cmb, cf = _slab_parts(0, i, b) if not cmb: continue bl = blocks(b * iE, iA); dtc = max(c1 - c0 for c0, c1 in bl) for (c0, c1) in bl: # Part A: chain exact at the block start Kb = d_trunc(Sst(b), c0, c1); K = Kb if b == i else conv_nn(Kb, Sst(i - b)) Q = dz() for cb in cmb: Q = d_add(Q, d_scale(conv_chain(cb[0], d_trunc(Rk(cb[1]), iH - c0, iH), cb[2]), cf(b, *cb))) acc = d_add(acc, d_from_cells([cell_mul(d_ends_rev(Q, M - u - 1), d_ends(K, u)) for u in cells], lo_u)) for br in sorted(set(cb[1] for cb in cmb)): # Part B: W * (G-range x S_{i-b}), all factors >= 0 W = W_dpl(b, br, bl) for cb in cmb: if cb[1] != br: continue Gl, Gu = g_range(cb[0], cb[2], dtc, 'corr') if b == i: kB = d_from_cells([cell_mul(d_ends(W, u), (Gl[0], Gl[0], Gu[0], Gu[0])) for u in cells], lo_u) else: S_ = Sst(i - b) kB = conv_nn(W, d_from_cells([cell_mul((Gl[y], Gl[y], Gu[y], Gu[y]), d_ends(S_, y)) for y in range(M)], 0)) acc = d_add(acc, d_scale(d_trunc(kB, lo_u, iA), cf(b, *cb))) return acc def coefK_prime(i, b, kw, ky, br, twisted): """the beta-empty part of coefK(0, i, ...): i1 = 0 small parts and kw = 0 rough parts outside the large part, i.e. the prime's own fragmentations inside the d = 1 row (its sign rho(1) is the prime's -f(1))""" if kw != 0: return 0 return rho_(1, twisted) * (-1) ** (i + ky + 1) * (-1) ** b * (-1) ** br def kappa_F(i): """representer of H -> Omega_i^K[H](1), f(1)'s H_i part (F1ADJ): minus the prime's part of kappa_dpl(i). kappa_dpl's EB term has no prime part (its vectors keep all i small parts outside the large part), so only the slab terms with kw = 0 and i1 = 0 enter; Parts A and B exactly as in kappa_dpl""" lo_u = i * iE; cells = range(lo_u, iA) acc = dz() cf = lambda b, kw, br, m: coefK_prime(i, b, kw, br + m, br, True) * wk(kw + br + m) combos = [cb for cb in ALLCOMBOS if cb[0] + cb[2] >= 1] for b in range(1, i + 1): cmb = [cb for cb in combos if cf(b, *cb) != 0] if not cmb: continue bl = blocks(b * iE, iA); dtc = max(c1 - c0 for c0, c1 in bl) for (c0, c1) in bl: # Part A: chain exact at the block start Kb = d_trunc(Sst(b), c0, c1); K = Kb if b == i else conv_nn(Kb, Sst(i - b)) Q = dz() for cb in cmb: Q = d_add(Q, d_scale(conv_chain(cb[0], d_trunc(Rk(cb[1]), iH - c0, iH), cb[2]), cf(b, *cb))) acc = d_add(acc, d_from_cells([cell_mul(d_ends_rev(Q, M - u - 1), d_ends(K, u)) for u in cells], lo_u)) for br in sorted(set(cb[1] for cb in cmb)): # Part B: W * (G-range x S_{i-b}), all factors >= 0 W = W_dpl(b, br, bl) for cb in cmb: if cb[1] != br: continue Gl, Gu = g_range(cb[0], cb[2], dtc, 'corr') if b == i: kB = d_from_cells([cell_mul(d_ends(W, u), (Gl[0], Gl[0], Gu[0], Gu[0])) for u in cells], lo_u) else: S_ = Sst(i - b) kB = conv_nn(W, d_from_cells([cell_mul((Gl[y], Gl[y], Gu[y], Gu[y]), d_ends(S_, y)) for y in range(M)], 0)) acc = d_add(acc, d_scale(d_trunc(kB, lo_u, iA), cf(b, *cb))) return d_neg(acc) def adj_coupling(j, i, lam): """C*_{j,i}[lam](u) on u-cells [(j+i) iE, iA) (twisted); lam lives on sigma-cells [j iE, iA)""" lo_u = (j + i) * iE; cells = range(lo_u, iA) LS = conv_sn(lam, Sst(i)) acc = d_from_cells([cell_mul(d_ends_rev(EB(j + i, True), M - u - 1), d_ends(LS, u)) for u in cells], lo_u) for b in range(1, i + 1): cmb, cf = _slab_parts(j, i, b) if not cmb: continue bl = blocks(b * iE, iA); dtc = max(c1 - c0 for c0, c1 in bl) for (c0, c1) in bl: Kb = d_trunc(Sst(b), c0, c1); K = Kb if b == i else conv_nn(Kb, Sst(i - b)) Q = dz() for cb in cmb: Q = d_add(Q, d_scale(conv_chain(cb[0], d_trunc(Rk(cb[1]), iH - c0, iH), cb[2]), cf(b, *cb))) LK = conv_sn(lam, K) acc = d_add(acc, d_from_cells([cell_mul(d_ends_rev(Q, M - u - 1), d_ends(LK, u)) for u in cells], lo_u)) Lam = lam if b == i else conv_sn(lam, Sst(i - b)) Lp, Lm = H_pm(Lam) for br in sorted(set(cb[1] for cb in cmb)): W = W_dpl(b, br, bl) for cb in cmb: if cb[1] != br: continue Gl, Gu = g_range(cb[0], cb[2], dtc, 'corr') GLp = d_from_cells([cell_mul((Gl[y], Gl[y], Gu[y], Gu[y]), d_ends(Lp, y)) for y in range(M)], 0) GLm = d_from_cells([cell_mul((Gl[y], Gl[y], Gu[y], Gu[y]), d_ends(Lm, y)) for y in range(M)], 0) gB = d_add(conv_nn(W, GLp), conv_nn(W, GLm), -1) acc = d_add(acc, d_scale(d_trunc(gB, lo_u, iA), cf(b, *cb))) return acc _T0_forward_for_json = T0t # audit fix 6 if T0ADJ: LAM = {}; T0adj = (0, 0) for _b in range(1, NBIN + 1): T0adj = add_iv(T0adj, MACH[_b].T0_c(DB[_b], True)) for m_ in range(1, MSTAR + 1): lo_c = m_ * iE; cells = range(lo_c, iA) rhs = kappa_dpl(m_) for j_ in range(1, m_): rhs = d_add(rhs, adj_coupling(j_, m_ - j_, LAM[j_]), -1) Bc = [d_ends_rev(Bk(m_, True), M - u - 1) for u in cells] LAM[m_] = d_from_cells([cell_neg_quot(d_ends(d_neg(rhs), u), b_) for u, b_ in zip(cells, Bc)], lo_c) Am = dz() for _b in range(1, NBIN + 1): if DB[_b]: Am = d_add(Am, MACH[_b].A_dpl(m_, True, DB[_b])) As = d_from_cells([d_ends_rev(Am, M - u - 1) for u in cells], lo_c) part = val_ss(LAM[m_], As) T0adj = add_iv(T0adj, (-part[1], -part[0])) log(f" adjoint T0: lambda_{m_} in [{fz(d_minlow(LAM[m_], lo_c, iA)):+.4f}, {fz(d_maxup(LAM[m_], lo_c, iA)):+.4f}] width max {fz(d_maxwidth(LAM[m_], lo_c, iA)):.3e}; " f"- = [{fz(-part[1]):+.6f}, {fz(-part[0]):+.6f}]") log(f"T0 forward [{fz(T0t[0]):+.6f}, {fz(T0t[1]):+.6f}]; T0 adjoint [{fz(T0adj[0]):+.6f}, {fz(T0adj[1]):+.6f}]") assert T0adj[0] <= T0t[1] and T0t[0] <= T0adj[1], "forward and adjoint T0 enclosures are disjoint: an engine error" T0t = (max(T0t[0], T0adj[0]), min(T0t[1], T0adj[1])) log(f"T0 used (intersection): [{fz(T0t[0]):+.6f}, {fz(T0t[1]):+.6f}]") F1c = (0, 0) for _b in range(1, NBIN + 1): F1c = add_iv(F1c, MACH[_b].f1_c(DB[_b])) log(f"f(1)'s c-part: [{fz(F1c[0]):+.6f}, {fz(F1c[1]):+.6f}]") F1cp = F1c OMK = {} for r in range(1, MSTAR + 1): OMK[r] = Omega_rK(H[r], r, KSET) part = d_at1(OMK[r]) log(f"f(1)'s H{r} part: [{fz(part[0]):+.6f}, {fz(part[1]):+.6f}]") F1c = add_iv(F1c, part) # ---------------------------------------------------------------- adjoint f(1) (env F1ADJ=1; DESIGN_SO_DEV_addendum7.md): f(1)'s H-part is # sum_m = -sum_m with L* nu = kappaF, solved bottom-up exactly as T0ADJ's lambda (same adj_coupling, same B). F1ADJ = os.environ.get('F1ADJ', '0') == '1' F1_forward_at_c_for_json = F1c # audit fix 4 F1_identity_checks_for_json = {} if F1ADJ: NU = {}; F1adj = F1cp for m_ in range(1, MSTAR + 1): lo_c = m_ * iE; cells = range(lo_c, iA) kF = kappa_F(m_) chk = val_ss(kF, H[m_]); fwd = d_at1(OMK[m_]) F1_identity_checks_for_json[str(m_)] = { # audit fix 4 'representer_H': [str(Fr(x, SCALE)) for x in chk], 'forward_H': [str(Fr(x, SCALE)) for x in fwd], } log(f" adjoint f(1): = [{fz(chk[0]):+.6f}, {fz(chk[1]):+.6f}] vs forward [{fz(fwd[0]):+.6f}, {fz(fwd[1]):+.6f}]") assert chk[0] <= fwd[1] and fwd[0] <= chk[1], "kappa_F does not represent f(1)'s H-part: an engine error" rhs = kF for j_ in range(1, m_): rhs = d_add(rhs, adj_coupling(j_, m_ - j_, NU[j_]), -1) Bc = [d_ends_rev(Bk(m_, True), M - u - 1) for u in cells] NU[m_] = d_from_cells([cell_neg_quot(d_ends(d_neg(rhs), u), b_) for u, b_ in zip(cells, Bc)], lo_c) Am = dz() for _b in range(1, NBIN + 1): if DB[_b]: Am = d_add(Am, MACH[_b].A_dpl(m_, True, DB[_b])) As = d_from_cells([d_ends_rev(Am, M - u - 1) for u in cells], lo_c) part = val_ss(NU[m_], As) F1adj = add_iv(F1adj, (-part[1], -part[0])) log(f" adjoint f(1): nu_{m_} in [{fz(d_minlow(NU[m_], lo_c, iA)):+.4f}, {fz(d_maxup(NU[m_], lo_c, iA)):+.4f}] width max {fz(d_maxwidth(NU[m_], lo_c, iA)):.3e}; " f"- = [{fz(-part[1]):+.6f}, {fz(-part[0]):+.6f}]") log(f"f(1) at c: forward [{fz(F1c[0]):+.6f}, {fz(F1c[1]):+.6f}]; adjoint [{fz(F1adj[0]):+.6f}, {fz(F1adj[1]):+.6f}]") assert F1adj[0] <= F1c[1] and F1c[0] <= F1adj[1], "forward and adjoint f(1) enclosures are disjoint: an engine error" F1c = (max(F1c[0], F1adj[0]), min(F1c[1], F1adj[1])) log(f"f(1) at c used (intersection): [{fz(F1c[0]):+.6f}, {fz(F1c[1]):+.6f}]") T0g = (0, 0); PHIG1 = (0, 0) for _b in range(1, NBIN + 1): T0g = add_iv(T0g, MACH[_b].T0_c(DG[_b], True)); PHIG1 = add_iv(PHIG1, MACH[_b].f1_c(DG[_b])) tiv = iv_div((-T0t[1], -T0t[0]), T0g) F1 = add_iv(F1c, iv_mul(tiv, PHIG1)) log(f"T0(c) in [{fz(T0t[0]):+.6f}, {fz(T0t[1]):+.6f}]; T0(g) in [{fz(T0g[0]):+.6f}, {fz(T0g[1]):+.6f}]; t in [{fz(tiv[0]):+.6f}, {fz(tiv[1]):+.6f}]") log(f"f(1) at c: [{fz(F1c[0]):+.6f}, {fz(F1c[1]):+.6f}]; on the segment: [{fz(F1[0]):+.6f}, {fz(F1[1]):+.6f}]") # ---------------------------------------------------------------- Step C: positivity (per-cell ranges: local first-order losses only) def realisable_free(j, k): if k == 0: return j * ETA < 1 < j * ALPHA return j * ETA + k * ALPHA < 1 < j * ALPHA + Fr(k, 2) def kset_of(kb): return kws_of(kb) free_min = 0; free_arg = None for j in range(0, int(1 / ETA) + 1): for k in range(0, 6): for bb in ([0] if k == 0 else range(1, NBIN + 1)): if not realisable_bin(j, k, bb): continue v = C3.get((j, k, bb), Fr(0)); lo = fl(v.numerator * SCALE, v.denominator) if (j, k, bb) in GDICT: w_g = GDICT[(j, k, bb)]; lo += min(0, fl(w_g.numerator * tiv[0], w_g.denominator), fl(w_g.numerator * tiv[1], w_g.denominator)) cands = [lo] if 1 <= j <= MSTAR and kin(k): w_ = wk(k) hlo = fl(w_.numerator * d_minlow(H[j], j * iE, iA), w_.denominator) if w_ > 0 else fl(w_.numerator * d_maxup(H[j], j * iE, iA), w_.denominator) cands = [lo + hlo] if j == 1 else [lo + hlo, lo] for x in cands: if x < free_min: free_min, free_arg = x, (j, k, bb) log(f"free min >= {fz(free_min):+.6f} at {free_arg}") _L0K = {} def L0K(kb): ks = tuple(kset_of(kb)) if ks not in _L0K: acc = dz() for (ky, w_) in ks: if ky >= 2 and (0, ky) in P: acc = d_add(acc, d_scale(phi(0, ky), w_)) _L0K[ks] = acc return _L0K[ks] _POK = {} def POK(kb): ks = tuple(kset_of(kb)) if ks not in _POK: acc = dz() for r in range(1, MSTAR + 1): acc = d_add(acc, OMK[r] if ks == KSET else Omega_rK(H[r], r, ks)) _POK[ks] = acc return _POK[ks] def J_fnK(jb, i2, ks): """(j_b >= 1, k_b = 0) h-term with i2 >= 1 small fragments, per rough count, as a DPL in sigma_beta (slab terms via slab_corr: kw = 0)""" wd = dict(ks) combos = sorted(set((0, br, k - br) for (k, w_) in ks for br in range(1, k + 1) if not ((br >= 2 and (0, br) not in PS) or (k - br >= 2 and (0, k - br) not in PS)))) def cf(b, kw, br, m): k = br + m if kw != 0 or wd.get(k, 0) == 0: return 0 return (-1) ** (i2 + k + 1) * (-1) ** b * (-1) ** br * wd[k] return slab_corr(H[jb + i2], (jb + i2) * iE, i2, [cf], combos)[0] JK = {jb: {i2: J_fnK(jb, i2, kset_of(0)) for i2 in range(1, MSTAR - jb + 1)} for jb in range(1, MSTAR)} log("J functions ready:", {jb: list(JK[jb]) for jb in JK}) ext_min = 0; ext_arg = None; EXT = {} def upd(val, arg): global ext_min, ext_arg key = arg[:2] if key not in EXT or val[0] < EXT[key][0]: EXT[key] = (val[0], arg) if val[0] < ext_min: ext_min, ext_arg = val[0], arg def make_rq(j): lo_c = j * iE lo_ = [min(H[j].la[u], H[j].lb[u]) for u in range(lo_c, iA)]; hi_ = [max(H[j].ua[u], H[j].ub[u]) for u in range(lo_c, iA)] SPmin, SPmax = [lo_], [hi_] while (1 << len(SPmin)) <= len(lo_): kk = len(SPmin); h = 1 << (kk - 1); a_, b_ = SPmin[-1], SPmax[-1] SPmin.append([min(a_[i], a_[i + h]) for i in range(len(a_) - h)]); SPmax.append([max(b_[i], b_[i + h]) for i in range(len(b_) - h)]) def rq(l, r): out_lo = []; out_hi = [] if r >= iA: out_lo.append(0); out_hi.append(0); r = iA - 1 if l < lo_c: out_lo.append(0); out_hi.append(0); l = lo_c if l <= r: l2, r2 = l - lo_c, r - lo_c; kk = (r2 - l2 + 1).bit_length() - 1 out_lo.append(min(SPmin[kk][l2], SPmin[kk][r2 - (1 << kk) + 1])); out_hi.append(max(SPmax[kk][l2], SPmax[kk][r2 - (1 << kk) + 1])) return (min(out_lo), max(out_hi)) if out_lo else (0, 0) return rq RQ = {j: make_rq(j) for j in range(1, MSTAR + 1)} QB = {} def block_oms(jb, kb, c0, c1): """the Omega DPLs of sigma_beta block [c0, c1), i2 = 1 .. MSTAR - jb: Hsh carries H_{jb+i2}'s range over cells c0+i .. c1+i at shift cell i""" oms = [] for i2 in range(1, MSTAR - jb + 1): Hsh = d_const_cells([RQ[jb + i2](c0 + i, c1 + i) for i in range(i2 * iE, iA)], i2 * iE) oms.append(Omega_rK(Hsh, i2, kset_of(kb))) return oms def get_QB(jb, kb): key = (jb, tuple(kset_of(kb))) if key not in QB: lst = [] for (c0, c1) in blocksQ(jb * iE, iA, nbq_of(jb)): hr = RQ[jb](c0, c1 - 1) lst.append((c0, c1, hr, block_oms(jb, kb, c0, c1))) QB[key] = lst return QB[key] HBB = {}; HBG = {} for jb in range(0, int(1 / (2 * ETA)) + 1): for kb in range(0, 4): for b in range(1, NBIN + 1): HBB[(jb, kb, b)] = MACH[b].H_beta(jb, kb, DB[b]); HBG[(jb, kb, b)] = MACH[b].H_beta(jb, kb, DG[b]) def sum_d(lst): acc = None for x in lst: if x is None: continue acc = x if acc is None else d_add(acc, x) return acc POS2D = os.environ.get('POS2D', 'fast') def _sp(vals, op): tab = [vals] while (1 << len(tab)) <= len(vals): hh = 1 << (len(tab) - 1); a_ = tab[-1] tab.append([op(a_[i], a_[i + hh]) for i in range(len(a_) - hh)]) def q(l, r): k = (r - l + 1).bit_length() - 1 return op(tab[k][l], tab[k][r - (1 << k) + 1]) return q HRQ = {} for _j in range(1, MSTAR + 1): _Hj = H[_j] HRQ[_j] = (_sp([min(_Hj.la[u], _Hj.lb[u]) for u in range(M)], min), _sp([max(_Hj.ua[u], _Hj.ub[u]) for u in range(M)], max)) def min_aq(n, L0_, L1_): """lower bound of min over a in a-cell n of a * L(a), L the linear function with (scaled) end values L0_, L1_ (a > 0): the endpoint values minus the convexity slack (h^2/8) * 2 L' = (h/4) max(L1 - L0, 0)""" return min(fl(n * L0_, M), fl((n + 1) * L1_, M)) - ce(max(L1_ - L0_, 0), 4 * M) def t_ends(Hg, n): """lower bound ends of t * Hg on a-cell n, t in tiv (the min of linear functions is concave, so the chord through the end minima is below)""" if Hg is None: return (0, 0) out = [] for lo_, hi_ in ((Hg.la[n], Hg.ua[n]), (Hg.lb[n], Hg.ub[n])): out.append(min(fl(tv * hv, SCALE) for tv in tiv for hv in (lo_, hi_))) return tuple(out) def mulL(Hv, L0e, e): """lower bound of Hv * L0(a) at a-end e (Hv a constant)""" return fl(Hv * (L0e[e] if Hv >= 0 else L0e[2 + e]), SCALE) LEAVES = {} def ext_blockouter(jb, kb): """addendum 8 (B)/(C): the 'elif hasH' branch of the loop below (its prelude and its POS2D fast path) for one class (jb, kb) with sigma_beta blocks, evaluated block by block. Pass 1 stores the per-(bin, a-cell) prelude (and makes the 'noH' updates); pass 2 computes one block's Omegas at a time and commits the block's minimum (only leaves, under QADAPT). Returns [(c0, c1, depth, value or None)].""" assert POS2D == 'fast', 'QOUTER needs the POS2D fast path' L0b = L0K(kb); rqmin, rqmax = HRQ[jb]; pre = [] for bbeta in range(1, NBIN + 1): tlo, thi = class_total_range(jb, kb, bbeta) if tlo >= Fr(1, 2): continue Hb = sum_d([HBB[(jb, kb, b)] for b in range(max(bbeta, 1), NBIN + 1)]) Hg = sum_d([HBG[(jb, kb, b)] for b in range(max(bbeta, 1), NBIN + 1)]) for n in range(iH, M): tau_lo, tau_hi = 1 - Fr(n + 1, M), 1 - Fr(n, M) if tau_hi <= tlo or tau_lo >= thi: continue tg = t_ends(Hg, n) b0 = (Hb.la[n] if Hb is not None else 0) + tg[0]; b1 = (Hb.lb[n] if Hb is not None else 0) + tg[1] sg_lo = max(jb * ETA, tau_lo - kb * EDGES[bbeta]); sg_hi = min(jb * ALPHA, tau_hi - EDGES[bbeta - 1] - (kb - 1) * ALPHA) if sg_lo > sg_hi: continue if sg_hi > ALPHA: upd((min_aq(n, b0, b1),), (jb, kb, n / M, 'noH')) L0e = d_ends(L0b, n) u_lo = max(jb * iE, int(sg_lo * M) - 1); u_hi = min(iA - 1, int(min(sg_hi, ALPHA) * M)) while u_lo <= u_hi and Fr(u_lo + 1, M) < sg_lo: u_lo += 1 while u_hi >= u_lo and Fr(u_hi, M) > min(sg_hi, ALPHA): u_hi -= 1 if u_lo > u_hi: continue pre.append((bbeta, n, b0, b1, L0e, u_lo, u_hi)) if not pre: return [] def eval_block(c0, c1): if not any(max(e[5], c0) <= min(e[6], c1 - 1) for e in pre): return None # no part: the n-outer loop makes no update here either oms = block_oms(jb, kb, c0, c1); best = None for (bbeta, n, b0, b1, L0e, u_lo, u_hi) in pre: p0, p1 = max(u_lo, c0), min(u_hi, c1 - 1) if p0 > p1: continue hmn, hmx = rqmin(p0, p1), rqmax(p0, p1) L0_ = b0 + min(mulL(hmn, L0e, 0), mulL(hmx, L0e, 0)); L1_ = b1 + min(mulL(hmn, L0e, 1), mulL(hmx, L0e, 1)) for om in oms: L0_ += om.la[n]; L1_ += om.lb[n] v = min_aq(n, L0_, L1_) if best is None or v < best[0]: best = (v, (jb, kb, n / M, p0 / M, 'b%d' % bbeta)) return best ad = QADAPT.get((jb, kb)); leaves = [] roots = blocksQ(jb * iE, iA, nbq_of(jb)) assert roots[0][0] == jb * iE and roots[-1][1] == iA and all(a < b for (a, b) in roots) and \ all(roots[i][1] == roots[i + 1][0] for i in range(len(roots) - 1)), 'sigma_beta base blocks do not tile [jb iE, iA): an engine error' for (r0, r1) in roots: stack = [(r0, r1, 0)]; mine = [] while stack: c0, c1, d = stack.pop() best = eval_block(c0, c1) if ad is not None and best is not None and best[0] < ad[0] and d < ad[2] and c1 - c0 > 1: for (s0, s1) in reversed(blocksQ(c0, c1, ad[1])): stack.append((s0, s1, d + 1)) # refine: the parent is not committed continue if best is not None: upd((best[0],), best[1]) mine.append((c0, c1, d, None if best is None else best[0])) mine.sort(key=lambda x: (x[0], x[1])) # check: the committed leaves tile [r0, r1) exactly assert mine and mine[0][0] == r0 and mine[-1][1] == r1 and all(x[0] < x[1] for x in mine) and \ all(mine[i][1] == mine[i + 1][0] for i in range(len(mine) - 1)), f'leaves of block [{r0}, {r1}) do not tile it: an engine error' if ad is not None: log(f" block-outer {(jb, kb)} block [{r0}, {r1}): {len(mine)} leaves") leaves.extend(mine) return leaves for jb in range(0, int(1 / (2 * ETA)) + 1): for kb in range(0, 4): QB.pop((jb, tuple(kset_of(kb - 1))), None) if kb >= 1 else None # memory: drop the previous k_b's sigma-block Omegas (one (j_b, k_b) cached at a time) if QOUTER and 1 <= jb < MSTAR and kb >= 1 and len(kset_of(kb)) > 0: # (B)/(C): exactly the classes whose QBl is not None below LEAVES[(jb, kb)] = ext_blockouter(jb, kb); continue for bbeta in ([0] if kb == 0 else range(1, NBIN + 1)): if (jb, kb) == (0, 0): continue tlo, thi = class_total_range(jb, kb, bbeta) if tlo >= Fr(1, 2): continue Hb = sum_d([HBB[(jb, kb, b)] for b in range(max(bbeta, 1), NBIN + 1)]) Hg = sum_d([HBG[(jb, kb, b)] for b in range(max(bbeta, 1), NBIN + 1)]) L0b = L0K(kb); hasH = (jb >= 1 and jb <= MSTAR and len(kset_of(kb)) > 0) QBl = get_QB(jb, kb) if (hasH and kb >= 1 and jb < MSTAR) else None for n in range(iH, M): tau_lo, tau_hi = 1 - Fr(n + 1, M), 1 - Fr(n, M) if tau_hi <= tlo or tau_lo >= thi: continue tg = t_ends(Hg, n) b0 = (Hb.la[n] if Hb is not None else 0) + tg[0]; b1 = (Hb.lb[n] if Hb is not None else 0) + tg[1] if jb == 0: Pk = POK(kb); upd((min_aq(n, b0 + Pk.la[n], b1 + Pk.lb[n]),), (jb, kb, n / M)) elif hasH and kb == 0: ib = M - n - 1 if ib >= iA: upd((min_aq(n, b0, b1),), (jb, kb, n / M)); continue if ib < jb * iE: continue Hj = H[jb] pr = cell_mul((Hj.lb[ib], Hj.la[ib], Hj.ub[ib], Hj.ua[ib]), d_ends(L0b, n)) L0_, L1_ = b0 + pr[0], b1 + pr[1] for i2 in range(1, MSTAR - jb + 1): Jd = JK[jb][i2]; L0_ += Jd.lb[ib]; L1_ += Jd.la[ib] upd((min_aq(n, L0_, L1_),), (jb, kb, n / M, ib / M)) elif hasH: # beta's kb rough parts: the largest in bin bbeta, the others in [alpha', E[bbeta]], so their total lies in # [E[bbeta-1] + (kb-1) alpha', kb E[bbeta]]; with one bin this is the old [kb alpha', kb/2] sg_lo = max(jb * ETA, tau_lo - kb * EDGES[bbeta]); sg_hi = min(jb * ALPHA, tau_hi - EDGES[bbeta - 1] - (kb - 1) * ALPHA) if sg_lo > sg_hi: continue if sg_hi > ALPHA: upd((min_aq(n, b0, b1),), (jb, kb, n / M, 'noH')) L0e = d_ends(L0b, n); Hj = H[jb] u_lo = max(jb * iE, int(sg_lo * M) - 1); u_hi = min(iA - 1, int(min(sg_hi, ALPHA) * M)) while u_lo <= u_hi and Fr(u_lo + 1, M) < sg_lo: u_lo += 1 while u_hi >= u_lo and Fr(u_hi, M) > min(sg_hi, ALPHA): u_hi -= 1 if u_lo > u_hi: continue if POS2D == 'slow': for u in range(u_lo, u_hi + 1): Hv = (Hj.la[u], Hj.lb[u], Hj.ua[u], Hj.ub[u]) m0 = min(mulL(h_, L0e, 0) for h_ in Hv); m1 = min(mulL(h_, L0e, 1) for h_ in Hv) L0_, L1_ = b0 + m0, b1 + m1 if QBl is not None: for (c0, c1, hr, oms) in QBl: if c0 <= u < c1: for om in oms: L0_ += om.la[n]; L1_ += om.lb[n] break upd((min_aq(n, L0_, L1_),), (jb, kb, n / M, u / M, 'b%d' % bbeta)) continue # fast: per sigma-block, the H extremes over the sub-range (phi(h) = h * L0-bound is concave in h, so its minimum over a # range of H values sits at the range's extremes); the bracket is concave in sigma's H, so per a-end minima give a valid chord rqmin, rqmax = HRQ[jb] parts = [(u_lo, u_hi, ())] if QBl is None else \ [(max(u_lo, c0), min(u_hi, c1 - 1), oms) for (c0, c1, hr, oms) in QBl if max(u_lo, c0) <= min(u_hi, c1 - 1)] for (p0, p1, oms) in parts: hmn, hmx = rqmin(p0, p1), rqmax(p0, p1) L0_ = b0 + min(mulL(hmn, L0e, 0), mulL(hmx, L0e, 0)); L1_ = b1 + min(mulL(hmn, L0e, 1), mulL(hmx, L0e, 1)) for om in oms: L0_ += om.la[n]; L1_ += om.lb[n] upd((min_aq(n, L0_, L1_),), (jb, kb, n / M, p0 / M, 'b%d' % bbeta)) else: upd((min_aq(n, b0, b1),), (jb, kb, n / M)) for _key in sorted(LEAVES): # (B)/(C) _lv = LEAVES[_key]; _byd = {} for (_c0, _c1, _d, _v) in _lv: _byd[_d] = _byd.get(_d, 0) + 1 _vals = [_v for (_c0, _c1, _d, _v) in _lv if _v is not None] log(f" block-outer {_key}: {len(_lv)} leaves, by depth {dict(sorted(_byd.items()))}, min leaf " + (f"{fz(min(_vals)):+.6f}" if _vals else "none")) log(f"extended min >= {fz(ext_min):+.6f} at {ext_arg}") for key in sorted(EXT): log(f" extended min for (j_b, k_b) = {key}: {fz(EXT[key][0]):+.6f} at {EXT[key][1]}") m_ = min(free_min, ext_min) margin = 1 - Fr(-m_) / Fr(-F1[1]) if (m_ < 0 and F1[1] < 0) else Fr(-1) verdict = "PASS" if (F1[1] < 0 and m_ > F1[1] and all(v == 0 for v in untw_bad.values()) and T0u[0] <= 0 <= T0u[1]) else "FAIL" log(f"VERDICT {verdict}: min f >= {fz(m_):+.6f}, f(1)_hi = {fz(F1[1]):+.6f}, scaled margin >= {float(margin):.5f} (print rounding: quote the JSON value rounded down); untwisted bad {untw_bad}") json.dump(dict(M=M, NB=NB, tag=TAG_, engine='second-order DPL (gm2/cert2.py, development)', script_sha256=hashlib.sha256(open(__file__, 'rb').read()).hexdigest(), dpl_sha256=hashlib.sha256(open(dpl.__file__, 'rb').read()).hexdigest(), block_mode=BLOCKMODE, pos2d=POS2D, nbq=NBQ, nbqlo=NBQLO, t0adj=T0ADJ, f1adj=F1ADJ, maxs=MAXS, kprof=('all' if KPROF is None else sorted(KPROF)), kw=(None if KW is None else {str(k): str(v) for k, v in KW.items()}), eta=str(ETA), alpha=str(ALPHA), mstar=MSTAR, edges=[str(e) for e in EDGES], gcls=list(GCLS), gvec={str(k): str(v) for k, v in GDICT.items()}, pickle_sha256=PKL_SHA, c={str(kk): str(v) for kk, v in sorted(C3.items())}, T0=[str(Fr(x, SCALE)) for x in T0t], T0u=[str(Fr(x, SCALE)) for x in T0u], T0g=[str(Fr(x, SCALE)) for x in T0g], t=[str(Fr(x, SCALE)) for x in tiv], f1=[str(Fr(x, SCALE)) for x in F1], free_min=str(Fr(free_min, SCALE)), ext_min=str(Fr(ext_min, SCALE)), ext_arg=str(ext_arg), untwisted_bad=untw_bad, verdict=verdict, margin=str(margin), nbqj={str(k): v for k, v in sorted(NBQJ.items())}, qouter=QOUTER, # addendum 8 qadapt={f'{k[0]},{k[1]}': [str(Fr(v[0], SCALE)), v[1], v[2]] for k, v in sorted(QADAPT.items())}, leaves={f'{k[0]},{k[1]}': [[a, b, d, (None if v is None else str(Fr(v, SCALE)))] for (a, b, d, v) in LEAVES[k]] for k in sorted(LEAVES)}, T0_forward=[str(Fr(x, SCALE)) for x in _T0_forward_for_json], # audit fix 6 T0_adjoint=([str(Fr(x, SCALE)) for x in T0adj] if T0ADJ else None), free_bound_arg=str(free_arg), free_bound_domain='H over [j*eta,alpha] independently of class; no-H candidate for j>=2', gcls_legacy=bool(os.environ.get('GVEC')), F1_constant_part=[str(Fr(x, SCALE)) for x in F1cp], # audit fix 4 F1_forward_at_c=[str(Fr(x, SCALE)) for x in F1_forward_at_c_for_json], F1_adjoint_at_c=([str(Fr(x, SCALE)) for x in F1adj] if F1ADJ else None), F1_used_at_c=[str(Fr(x, SCALE)) for x in F1c], F1_identity_checks=F1_identity_checks_for_json), open(f'cert2_{TAG_}_M{M}.json', 'x'), indent=1) ~~~~~~~~~~~~ #### file: cert2_f3_diag2.py - kind: script - as run: SHA-256 eac46ea92ecba5de48eb3299fc64bedff955fb2076f10f22e94f0328f786abb0 - as shipped: SHA-256 f9c849dc6e9b4020fd3937b9f7b54530e7c6ed3ea7075479a951bb9f2e8c7c43 - identity: identical, after dropping docstrings and comments, to the file as run with 3 declared change(s) of a machine-specific path, which now read: `sys.path.insert(0, __import__('os').path.dirname(__import__('os').path.abspath(__file__)))`; `open(f'diag2_{TAG_}_M{M}_levels.pkl', 'wb')`; `open(f'cert2_{TAG_}_M{M}.json', 'x')` - trailing newline: yes ~~~~~~~~~~~~ python #!/usr/bin/env python # cert2_f3_diag2.py: cert2_f3.py with the JSON output written to the working directory as a new file (mode 'x') and a per-level # dump (floats) of the A-part, each twisted coupling, B and H per cell, to diag2__M_levels.pkl. A diagnostic copy. import sys, os, time, json, hashlib, pickle from fractions import Fraction as Fr from math import comb sys.path.insert(0, __import__('os').path.dirname(__import__('os').path.abspath(__file__))) import dpl T_START = time.time() def log(*a): print(f"[{time.time() - T_START:7.1f}s]", *a, flush=True) M_, NB_, PKL_, TAG_ = sys.argv[1:5] M = int(M_); NB = int(NB_) dpl.init(M) from dpl import (D, SCALE, fl, ce, fz, dz, d_add, d_neg, d_scale, d_div, d_trunc, d_cell, d_ends, d_ends_rev, d_at1, d_at0, d_from_cells, d_const_cells, d_minlow, d_maxup, d_maxwidth, d_kernel, conv_nn, conv_sn, conv_ss, corr_sn, corr_nn, cell_mul, cell_neg_quot, cell_add, cell_contains0, iv_mul, iv_div, add_iv, iv_scale, _pos_lo, _pos_hi, _neg_lo, _neg_hi) raw = open(PKL_, 'rb').read(); PKL_SHA = hashlib.sha256(raw).hexdigest(); dpk = pickle.loads(raw) ETA, ALPHA = Fr(dpk['eta']), Fr(dpk['alpha']) assert (M * ETA).denominator == 1 and (M * ALPHA).denominator == 1 and M % 2 == 0 iE, iA, iH = int(M * ETA), int(M * ALPHA), M // 2 KPROF = None if os.environ.get('KPROF', 'all') == 'all' else set(int(x) for x in os.environ['KPROF'].split(',')) assert M > 0 and NB > 0 # audit fix 2 assert 0 < ETA < ALPHA < Fr(1, 2) assert 6 * ALPHA >= 1, "rough-count cutoff 5 is insufficient" assert 20 * ETA >= ALPHA, "the forced-level search range 1..19 of MSTAR is insufficient" # addendum 9 (was 10*ETA >= ALPHA, m* <= 9) MSTAR = max(j for j in range(1, 20) if j * ETA < ALPHA) log(f"cert2 (second order): M={M} NB={NB} NBQ={os.environ.get('NBQ', NB)} pickle sha256={PKL_SHA} tag={TAG_} KPROF={'all' if KPROF is None else sorted(KPROF)} KW={os.environ.get('KW')} eta={ETA} alpha'={ALPHA} m*={MSTAR}") # ---------------------------------------------------------------- bins and constants _ed = dpk.get('edges') if isinstance(_ed, str): _ed = [Fr(t) for t in _ed.split(',') if t.strip()] # some input pickles store the interior edges as text EDGES = [Fr(e) for e in _ed] if _ed else [ALPHA, Fr(1, 2)] if EDGES[-1] != Fr(1, 2): EDGES = EDGES + [Fr(1, 2)] if EDGES[0] != ALPHA: EDGES = [ALPHA] + [e for e in EDGES if e > ALPHA] assert EDGES[-1] == Fr(1, 2), EDGES assert len(EDGES) >= 2 and EDGES[0] == ALPHA assert all(a < b for a, b in zip(EDGES, EDGES[1:])), "unordered bin edges" NBIN = len(EDGES) - 1 for q in EDGES: assert (q * M).denominator == 1, f"M does not put {q} on the grid" iEdge = [int(e * M) for e in EDGES] def class_total_range(j, k, b): if k == 0: return (j * ETA, j * ALPHA) lo_b, hi_b = EDGES[b - 1], EDGES[b] return (j * ETA + (k - 1) * ALPHA + lo_b, j * ALPHA + k * hi_b) def realisable_bin(j, k, b): lo, hi = class_total_range(j, k, b); return lo < 1 < hi TEXTS = {'v2main': [(0, 3), (0, 4), (2, 2), (2, 3), (2, 4), (3, 2), (3, 3), (4, 1), (4, 2), (5, 1), (5, 2), (6, 1), (6, 2)]} TEXT = TEXTS.get(TAG_) C3 = {} for cls, x in zip(dpk['classes'], dpk['x']): j, k, b = (int(t) for t in cls) if abs(float(x)) == 0: continue if k == 0: b = 0 if NBIN == 1 and k >= 1: b = 1 if (k == 0 and b != 0) or (k >= 1 and not 1 <= b <= NBIN): continue if TEXT is not None and (j, k) not in TEXT: continue if not realisable_bin(j, k, b): continue assert (j, k, b) not in C3 C3[(j, k, b)] = Fr(float(x)) if TEXT is not None: assert sorted((j, k) for (j, k, b) in C3) == sorted(TEXT), sorted(C3) MAXS = max(9, max(j for (j, k, b) in C3), int(os.environ.get('MAXS', '0'))) # small-count power cutoff: every constant's small count, the forced levels, and env MAXS (oracle bases) assert MAXS >= MSTAR def abel(cdict): Dd = {b: {} for b in range(1, NBIN + 1)} for (j, k, b), v in cdict.items(): if k == 0: Dd[NBIN][(j, 0)] = Dd[NBIN].get((j, 0), 0) + v else: Dd[b][(j, k)] = Dd[b].get((j, k), 0) + v if b >= 2: Dd[b - 1][(j, k)] = Dd[b - 1].get((j, k), 0) - v return {b: {jk: v for jk, v in dd.items() if v != 0} for b, dd in Dd.items()} GBIN = int(os.environ.get('GBIN', str(NBIN))) GCLS = (0, 3, GBIN if NBIN > 1 else 1) # env GVEC 'b:w,b:w' = the T0 correction direction g = sum_b w_b e_(0,3,b) (H-neutral: no small parts); default e_GCLS GDICT = {(0, 3, int(a)): Fr(w) for a, w in (t.split(':') for t in os.environ['GVEC'].split(','))} if os.environ.get('GVEC') else {GCLS: Fr(1)} for _g in GDICT: assert realisable_bin(*_g), f'g class {_g} not realisable' DB = abel(C3); DG = abel(GDICT) log(f"bins: edges {[str(e) for e in EDGES]}; {len(C3)} constants; g = {GDICT}") log("c:", {kk: round(float(v), 6) for kk, v in sorted(C3.items())}) # ---------------------------------------------------------------- kernels (global: uncut rough kernel), powers, Lemma K kernels phi S1 = d_kernel(iE, iA); R1 = d_kernel(iA, iH) def build_powers(Rker): Pd = {}; PSd = {(1, 0): S1, (0, 1): Rker} for b in range(0, MAXS + 1): for bp in range(0, 6): if b + bp < 2 or b * ETA + bp * ALPHA > 1: continue if bp >= 1: prev, ker, div = (b, bp - 1), Rker, bp else: prev, ker, div = (b - 1, 0), S1, b if prev not in PSd: continue Pd[(b, bp)] = d_div(conv_nn(PSd[prev], ker), div); PSd[(b, bp)] = Pd[(b, bp)] return Pd, PSd P, PS = build_powers(R1) log(f"powers: {len(P)}") def Rk(k): return R1 if k == 1 else PS[(0, k)] def Sst(r): return PS[(r, 0)] class Phis: def __init__(self, Pd, PSd): self.P, self.PS, self.TR, self.PHI = Pd, PSd, {}, {} def trs(self, bb): if bb not in self.TR: self.TR[bb] = None if (sum(bb) < 2 or bb not in self.P) else d_trunc(self.P[bb], iH) return self.TR[bb] def phi(self, j, k): if (j, k) in self.PHI: return self.PHI[(j, k)] acc = dz() for bs in range(0, j + 1): for br in range(0, k + 1): if bs + br < 2 or (bs, br) not in self.P: continue sign = (-1) ** (j + k + 1) * (-1) ** (bs + br); comp = (j - bs, k - br) if comp == (0, 0): X = d_trunc(self.P[(bs, br)], iH) else: if comp not in self.PS: continue X = conv_nn(self.trs((bs, br)), self.PS[comp]) acc = d_add(acc, X, sign) self.PHI[(j, k)] = d_trunc(acc, iH); return self.PHI[(j, k)] def lin_phi(self, coef): acc = dz() for jk, cv in coef.items(): if cv != 0: acc = d_add(acc, d_scale(self.phi(*jk), cv)) return acc def H_beta(self, jb, kb, dd): coef = {} for (cj, ck), cv in dd.items(): ju, ku = cj - jb, ck - kb if ju >= 0 and ku >= 0 and ju + ku >= 2: coef[(ju, ku)] = coef.get((ju, ku), 0) + cv return self.lin_phi(coef) if coef else None GPH = Phis(P, PS) phi = GPH.phi def rsum(ms): s = dz() for m in ms: if m == 1: s = d_add(s, R1) elif (0, m) in P: s = d_add(s, P[(0, m)]) return s RALL = rsum([1, 2, 3, 4, 5]); EP = rsum([2, 4]); EM = rsum([1, 3, 5]) def E_conv(X, rho_c, twisted=True, nodelta=False): """E_rho^{(c)} * X: twisted (-1)^c (delta + E'+ - E'-) * X; untwisted (delta + R_all) * X""" if twisted: out = dz() if nodelta else X out = d_add(out, conv_sn(X, EP)); out = d_add(out, conv_sn(X, EM), -1) return d_neg(out) if (rho_c or 0) % 2 == 1 else out out = dz() if nodelta else X return d_add(out, conv_sn(X, RALL)) log("kernels ready") # exact values of (f*g)(1) and int f g, without building the whole convolution (no curvature needed at a grid point) def _N_at(F, G, k, mode): a, b = F; c, d = G; tot = 0 if mode == 'conv': # n + m = k - 1 for n in range(max(0, k - M), min(M, k)): m = k - 1 - n if m < 0 or m >= M: continue an, bn, cm, dm = a[n], b[n], c[m], d[m] if an or bn: tot += (an * cm + bn * dm) + 2 * (an * dm + bn * cm) else: # corr at s = 0: n = m for n in range(M): an, bn, cm, dm = a[n], b[n], c[n], d[n] if (an or bn) and (cm or dm): tot += 2 * (an * cm + bn * dm) + (an * dm + bn * cm) return tot def val_sn(f, g, mode): """interval of (f*g)(1) (mode 'conv') or int f g (mode 'corr'), f signed, g >= 0""" S6 = 6 * M * SCALE glo = _pos_lo(g.la, g.lb); ghi = _pos_hi(g.ua, g.ub) lo = fl(_N_at(_pos_lo(f.la, f.lb), glo, M, mode), S6) - ce(_N_at(_neg_hi(f.la, f.lb), ghi, M, mode), S6) hi = ce(_N_at(_pos_hi(f.ua, f.ub), ghi, M, mode), S6) - fl(_N_at(_neg_lo(f.ua, f.ub), glo, M, mode), S6) return (lo, hi) # ---------------------------------------------------------------- one bin machine per edge (rough kernel cut at the edge) class Mach(Phis): def __init__(self, ecell): if ecell == iH: Pd, PSd = P, PS else: Pd, PSd = build_powers(d_kernel(iA, ecell)) Phis.__init__(self, Pd, PSd) if ecell == iH: self.PHI = GPH.PHI; self.TR = GPH.TR def f1_c(self, dd): acc = (0, 0) for jk, cv in dd.items(): if sum(jk) >= 2: acc = add_iv(acc, iv_scale(d_at1(self.phi(*jk)), cv)) return acc def T0_c(self, dd, twisted): rho = (lambda d: (-1) ** d) if twisted else (lambda d: 1) acc = (0, 0) if not dd: return acc for (j, k), cv in dd.items(): if (j, k) in self.P: acc = add_iv(acc, iv_scale(d_at1(self.P[(j, k)]), cv * rho(j + k))) if j + k >= 2: acc = add_iv(acc, iv_scale(d_at1(self.phi(j, k)), cv * rho(1))) jmax = max(j for j, k in dd); kmax = max(k for j, k in dd) for jb in range(0, jmax + 1): for kb in range(0, kmax + 1): if (jb, kb) == (0, 0) or (jb, kb) not in self.PS: continue Hb = self.H_beta(jb, kb, dd) if Hb is None: continue acc = add_iv(acc, iv_scale(val_sn(d_trunc(Hb, iH), self.PS[(jb, kb)], 'conv'), rho(1 + jb + kb))) return acc def A_dpl(self, base, twisted, dd): """A_base(x).c as a DPL in x = 1 - sigma (free part + extended part), DESIGN_GM""" rho = (lambda d: (-1) ** d) if twisted else (lambda d: 1) A = dz() if not dd: return A for (cj, ck), cv in dd.items(): if cj >= base: jz, kz = cj - base, ck if jz + kz >= 2 and (jz, kz) in self.P: A = d_add(A, d_scale(self.P[(jz, kz)], cv * rho(base + jz + kz))) jmax = max(j for j, k in dd); kmax = max(k for j, k in dd) for jp in range(0, jmax + 1): for kp in range(0, kmax + 1): coef = {} for (cj, ck), cv in dd.items(): ju, ku = cj - base - jp, ck - kp if ju >= 0 and ku >= 0 and ju + ku >= 2: coef[(ju, ku)] = coef.get((ju, ku), 0) + cv if not coef: continue Gk = self.lin_phi(coef) if (jp, kp) == (0, 0): X = Gk else: if (jp, kp) not in self.PS: continue X = conv_sn(d_trunc(Gk, iH), self.PS[(jp, kp)]) A = d_add(A, d_scale(X, rho(base + 1 + jp + kp))) return A MACH = {b: Mach(iEdge[b]) for b in range(1, NBIN + 1)} log(f"bin machines ready: {NBIN}") # ---------------------------------------------------------------- rough-count profile: B_{j,K}, brackets, slab chains KALL = list(range(2, 6)) # weighted rough-count profile (env KW, e.g. '2:1,3:-1'): H_j(sigma, k) = w_k h_j(sigma); KPROF = the support of KW. Unset: KPROF as before (w = 1). KW_ENV = os.environ.get('KW') KW = {int(a): Fr(b) for a, b in (t.split(':') for t in KW_ENV.split(','))} if KW_ENV else None if KW is not None: KPROF = set(k for k, v in KW.items() if v != 0) def wk(k): if KW is not None: return KW.get(k, Fr(0)) return Fr(1) if (KPROF is None or k in KPROF) else Fr(0) def kin(k): return wk(k) != 0 def kws_of(kb): """fragment rough counts ky with the weight w(kb + ky) of the completed vector's total rough count (KPROF all, no KW: ky = 1..5, w = 1)""" if KPROF is None and KW is None: return tuple((ky, Fr(1)) for ky in range(1, 6)) return tuple((k - kb, wk(k)) for k in range(1, 6) if wk(k) != 0 and k - kb >= 1) KWS = tuple((k, wk(k)) for k in KALL if wk(k) != 0) # f(1)'s Omega: rough counts >= 2, no beta def rho_(d, twisted): return (-1) ** d if twisted else 1 _RPHI = {} def RPHI(kw, ky): if (kw, ky) not in _RPHI: if (0, ky) not in P: _RPHI[(kw, ky)] = None elif kw == 0: _RPHI[(kw, ky)] = phi(0, ky) elif (0, kw) not in PS: _RPHI[(kw, ky)] = None else: _RPHI[(kw, ky)] = conv_sn(d_trunc(phi(0, ky), iH), PS[(0, kw)]) return _RPHI[(kw, ky)] _BK = {} def Bk(j, twisted): key = ('B', j % 2 if twisted else 0, twisted) if key not in _BK: acc = dz() for k in KALL: if not kin(k): continue if (0, k) in P: acc = d_add(acc, d_scale(P[(0, k)], wk(k) * rho_(j + k, twisted))) for kw in range(0, k - 1): X = RPHI(kw, k - kw) if X is None: continue acc = d_add(acc, d_scale(X, wk(k) * rho_(j + kw + 1, twisted))) _BK[key] = acc return _BK[key] def EB(c, twisted): """free-completion bracket sum_{k in K} rho(c+k) R_k + sum_{k in K} sum_{kw+ky=k, ky>=2} rho(c+1+kw) (R_kw * Phi_{0,ky}tr), as a DPL in x""" key = ('E', c % 2 if twisted else 0, twisted) if key not in _BK: acc = dz() for k in KALL: if not kin(k): continue if (0, k) in P: acc = d_add(acc, d_scale(P[(0, k)], wk(k) * rho_(c + k, twisted))) for kw in range(0, k - 1): X = RPHI(kw, k - kw) if X is None: continue acc = d_add(acc, d_scale(X, wk(k) * rho_(c + 1 + kw, twisted))) _BK[key] = acc return _BK[key] def blocks(lo_c, hi_c): n_c = hi_c - lo_c; bl = max(1, -(-n_c // NB)) return [(c0, min(c0 + bl, hi_c)) for c0 in range(lo_c, hi_c, bl)] NBQ = int(os.environ.get('NBQ', str(NB))) # sigma_beta blocks in positivity's (j_b >= 1, k_b >= 1) Omega terms (first order there) NBQLO = int(os.environ.get('NBQLO', str(NBQ))) # finer sigma_beta blocks for j_b <= 2 (the binding (1,1), (2,1) classes at 1/4) def blocksQ(lo_c, hi_c, nbq=None): n_c = hi_c - lo_c; bl = max(1, -(-n_c // (nbq or NBQ))) return [(c0, min(c0 + bl, hi_c)) for c0 in range(lo_c, hi_c, bl)] NBQJ = {int(a): int(b) for a, b in (u.split(':') for u in os.environ['NBQJ'].split(','))} if os.environ.get('NBQJ') else {} # (A) assert all(v >= 1 for v in NBQJ.values()) def nbq_of(jb): return NBQJ.get(jb, NBQLO if jb <= 2 else NBQ) # (A): the sigma_beta block count for j_beta QOUTER = os.environ.get('QOUTER', '0') == '1' # (B) QADAPT = {} # (C): (jb, kb) -> (scaled target, split, depth) for _it in (os.environ['QADAPT'].split(';') if os.environ.get('QADAPT') else []): _cls, _tg, _sp, _dp = _it.split(':'); _jb, _kb = (int(x) for x in _cls.split(',')); _tf = Fr(_tg) QADAPT[(_jb, _kb)] = (fl(_tf.numerator * SCALE, _tf.denominator), int(_sp), int(_dp)) assert int(_sp) >= 2 and int(_dp) >= 0 assert QOUTER or not QADAPT, 'QADAPT needs QOUTER=1' def slab_br(br, t_lo, t_hi): Rb = Rk(br) return (d_trunc(Rb, iH - t_lo, iH), d_trunc(Rb, iH - t_hi, iH)) def conv_chain(kw, sl, m): cur = sl if kw >= 1: cur = conv_nn(cur, Rk(kw)) if m >= 1: cur = conv_nn(cur, Rk(m)) return cur def coefK(j, i, b, kw, ky, br, twisted): tot = 0 for i1 in range(0, i - b + 1): i2 = i - i1 s_out = rho_(j + i1 + kw + 1, twisted) tot += comb(i - b, i1) * s_out * (-1) ** (i2 + ky + 1) * (-1) ** b * (-1) ** br return tot def H_times_x(Hd, X, lo_u): """f(u) = H(u) X(1 - u) on u-cells [lo_u, iA) (X a DPL in x)""" return d_from_cells([cell_mul(d_ends(Hd, u), d_ends_rev(X, M - u - 1)) for u in range(lo_u, iA)], lo_u) def H_times_u(Hd, X, lo_u): """f(u) = H(u) X(u) on u-cells [lo_u, iA)""" return d_from_cells([cell_mul(d_ends(Hd, u), d_ends(X, u)) for u in range(lo_u, iA)], lo_u) # ---------------------------------------------------------------- refined threshold blocks (gm2/DESIGN_SO.md addendum 1): Part A exact at the block # start tau0, Part B the sawtooth rho(tau) = int_{tau0}^{tau} R_br(1/2 - t) dt times the range of g = R_kw * R_m on a window of width dtau. # BLOCKS=plain restores the narrow/wide (first-order) block bound for comparison. BLOCKMODE = os.environ.get('BLOCKS', 'refined') ALLCOMBOS = sorted(set((kw, br, k - kw - br) for k in KALL if kin(k) for kw in range(0, k) for br in range(1, k - kw + 1) if not ((br >= 2 and (0, br) not in PS) or (kw >= 2 and (0, kw) not in PS) or (k - kw - br >= 2 and (0, k - kw - br) not in PS)))) _GD = {} def g_dpl(kw, m): if (kw, m) not in _GD: if kw == 0 and m == 0: _GD[(kw, m)] = None elif kw == 0: _GD[(kw, m)] = Rk(m) elif m == 0: _GD[(kw, m)] = Rk(kw) else: _GD[(kw, m)] = conv_nn(Rk(kw), Rk(m)) return _GD[(kw, m)] def _sparse(vals, op): tab = [vals] while (1 << len(tab)) <= len(vals): h = 1 << (len(tab) - 1); a = tab[-1] tab.append([op(a[i], a[i + h]) for i in range(len(a) - h)]) def q(l, r): k = (r - l + 1).bit_length() - 1 return op(tab[k][l], tab[k][r - (1 << k) + 1]) return q _GR = {} def g_range(kw, m, dtc, kind): """(Gl, Gu) per cell n: the range of g = R_kw * R_m (>= 0) on the window of g-cells [iH-1-n-dtc, iH-1-n] (kind 'corr': argument 1/2 - y - s, s in [0, dtau], y in cell n) or [n-iH-dtc, n-iH] (kind 'conv': argument z - 1/2 - s, z in cell n); off-grid cells count as 0""" key = (kw, m, dtc, kind) if key in _GR: return _GR[key] g = g_dpl(kw, m) qlo = _sparse([min(a, b) for a, b in zip(g.la, g.lb)], min); qhi = _sparse([max(a, b) for a, b in zip(g.ua, g.ub)], max) Gl = [0] * M; Gu = [0] * M for n in range(M): c_lo, c_hi = (iH - 1 - n - dtc, iH - 1 - n) if kind == 'corr' else (n - iH - dtc, n - iH) vals_lo = []; vals_hi = [] if c_lo < 0 or c_hi >= M: vals_lo.append(0); vals_hi.append(0) a_, b_ = max(c_lo, 0), min(c_hi, M - 1) if a_ <= b_: vals_lo.append(qlo(a_, b_)); vals_hi.append(qhi(a_, b_)) Gl[n] = max(0, min(vals_lo)); Gu[n] = max(0, max(vals_hi)) _GR[key] = (Gl, Gu) return _GR[key] def rho_dpl(br, bl): """rho(tau) = int_{tau0(block)}^{tau} R_br(1/2 - t) dt on the tau-cells of the blocks bl, as a DPL (linear bounds from the min/max of R_br's enclosure on the mirrored cell; the cumulative sums are exact integrals of R_br's linear bounds)""" Rb = Rk(br); la = [0] * M; lb = [0] * M; ua = [0] * M; ub = [0] * M for (c0, c1) in bl: acc_lo = acc_hi = 0 for c in range(c0, c1): n = iH - 1 - c rl = max(0, min(Rb.la[n], Rb.lb[n])); ru = max(Rb.ua[n], Rb.ub[n]) la[c] = acc_lo; lb[c] = acc_lo + fl(rl, M); ua[c] = acc_hi; ub[c] = acc_hi + ce(ru, M) acc_lo += fl(max(Rb.la[n] + Rb.lb[n], 0), 2 * M) # audit fix 1 acc_hi += ce(Rb.ua[n] + Rb.ub[n], 2 * M) return D(la, lb, ua, ub) def W_dpl(b, br, bl): rho = rho_dpl(br, bl); S = Sst(b); lo_c, hi_c = bl[0][0], bl[-1][1] return d_from_cells([cell_mul(d_ends(S, c), d_ends(rho, c)) for c in range(lo_c, hi_c)], lo_c) def H_pm(Hd): lp = _pos_lo(Hd.la, Hd.lb); up = _pos_hi(Hd.ua, Hd.ub); ln = _neg_lo(Hd.ua, Hd.ub); un = _neg_hi(Hd.la, Hd.lb) return D(lp[0], lp[1], up[0], up[1]), D(ln[0], ln[1], un[0], un[1]) def _gI(G, Ip, Im, c): """c * [Gl I+ - Gu I-, Gu I+ - Gl I-] per cell (G constant per cell, I+- >= 0 DPLs), as a DPL""" Gl, Gu = G lo_a = [fl(gl * ipa - gu * ima, SCALE) for gl, gu, ipa, ima in zip(Gl, Gu, Ip.la, Im.ua)] lo_b = [fl(gl * ipb - gu * imb, SCALE) for gl, gu, ipb, imb in zip(Gl, Gu, Ip.lb, Im.ub)] hi_a = [ce(gu * ipa - gl * ima, SCALE) for gl, gu, ipa, ima in zip(Gl, Gu, Ip.ua, Im.la)] hi_b = [ce(gu * ipb - gl * imb, SCALE) for gl, gu, ipb, imb in zip(Gl, Gu, Ip.ub, Im.lb)] X = D(lo_a, lo_b, hi_a, hi_b) return d_scale(X, c) def slab_corr(Hd, lo_u, i, coefs, combos, at0=False): """sum_b sum_combos coef(b, kw, br, m) * int int H(u) X_tau(1 - u) S_b(tau) S_{i-b}(u - tau - sigma) dtau du for each coefficient function in coefs; DPLs in sigma (or intervals at sigma = 0 if at0). kw = m = 0 terms vanish here (argument 1 - u > 1/2 is outside the slab).""" combos = [cb for cb in combos if cb[0] + cb[2] >= 1] res = [((0, 0) if at0 else dz()) for _ in coefs] def acc_(o, X): if at0: res[o] = add_iv(res[o], X) else: res[o] = d_add(res[o], X) for b in range(1, i + 1): bl = blocks(b * iE, iA); dtc = max(c1 - c0 for c0, c1 in bl) cmb = [cb for cb in combos if any(cf(b, *cb) != 0 for cf in coefs)] if not cmb: continue for (c0, c1) in bl: Kb = d_trunc(Sst(b), c0, c1); K = Kb if b == i else conv_nn(Kb, Sst(i - b)) if BLOCKMODE == 'refined': X0 = {cb: conv_chain(cb[0], d_trunc(Rk(cb[1]), iH - c0, iH), cb[2]) for cb in cmb} for o, cf in enumerate(coefs): Q = dz() for cb, X in X0.items(): cv = cf(b, *cb) if cv: Q = d_add(Q, d_scale(X, cv)) f = H_times_x(Hd, Q, lo_u) acc_(o, val_sn(f, K, 'corr') if at0 else corr_sn(f, K)) else: ch = {} for cb in cmb: sn, sw = slab_br(cb[1], c0, c1); ch[cb] = (conv_chain(cb[0], sn, cb[2]), conv_chain(cb[0], sw, cb[2])) for o, cf in enumerate(coefs): Q = dz() for cb, (Xn, Xw) in ch.items(): cv = cf(b, *cb) if cv: Q = d_add(Q, d_scale(D(Xn.la, Xn.lb, Xw.ua, Xw.ub), cv)) # d_scale swaps the sides when cv < 0 f = H_times_x(Hd, Q, lo_u) acc_(o, val_sn(f, K, 'corr') if at0 else corr_sn(f, K)) if BLOCKMODE != 'refined': continue Hp, Hm = H_pm(Hd) for br in sorted(set(cb[1] for cb in cmb)): W = W_dpl(b, br, bl) Ip, Im = corr_nn(Hp, W), corr_nn(Hm, W) for o, cf in enumerate(coefs): PB = None for cb in cmb: if cb[1] != br: continue cv = cf(b, *cb) if not cv: continue t_ = _gI(g_range(cb[0], cb[2], dtc, 'corr'), Ip, Im, cv) PB = t_ if PB is None else d_add(PB, t_) if PB is None: continue if b < i: acc_(o, val_sn(PB, Sst(i - b), 'corr') if at0 else corr_sn(PB, Sst(i - b))) else: acc_(o, (PB.la[0], PB.ua[0]) if at0 else PB) return res def coupling(j, i, Hd): """(twisted, untwisted) C_{j,i}[H_{j+i}](sigma) as DPLs in sigma (DESIGN_GM): free + i2 = 0 part, then the i2 >= 1 slab terms""" lo_u = (j + i) * iE res = [corr_sn(H_times_x(Hd, EB(j + i, tw), lo_u), Sst(i)) for tw in (True, False)] ext = slab_corr(Hd, lo_u, i, [lambda b, kw, br, m, tw=tw: coefK(j, i, b, kw, br + m, br, tw) * wk(kw + br + m) for tw in (True, False)], ALLCOMBOS) return d_add(res[0], ext[0]), d_add(res[1], ext[1]) def coupling_at0(i, Hd): """(twisted, untwisted) C_{0,i}[H_i] at sigma = 0, as intervals""" lo_u = i * iE res = [val_sn(H_times_x(Hd, EB(i, tw), lo_u), Sst(i), 'corr') for tw in (True, False)] ext = slab_corr(Hd, lo_u, i, [lambda b, kw, br, m, tw=tw: coefK(0, i, b, kw, br + m, br, tw) * wk(kw + br + m) for tw in (True, False)], ALLCOMBOS, at0=True) return add_iv(res[0], ext[0]), add_iv(res[1], ext[1]) # ---------------------------------------------------------------- Step A: top-down forcing H = {}; untw_bad = {} _DGALL = {} for j in range(MSTAR, 0, -1): lo_c = j * iE; cells = range(lo_c, iA) At = dz(); Au = dz() for b in range(1, NBIN + 1): At = d_add(At, MACH[b].A_dpl(j, True, DB[b])); Au = d_add(Au, MACH[b].A_dpl(j, False, DB[b])) Btw, Bun = Bk(j, True), Bk(j, False) num = [d_ends_rev(At, M - u - 1) for u in cells] bad = sum(1 for u in cells if not cell_contains0(d_ends_rev(Au, M - u - 1))) + sum(1 for u in cells if not cell_contains0(d_ends_rev(Bun, M - u - 1))) wA = max(max(x[2] - x[0], x[3] - x[1]) for x in num) _DG = {'A': [tuple(fz(v) for v in x) for x in num], 'C': {}} for i in range(1, MSTAR - j + 1): Ct, Cu = coupling(j, i, H[j + i]) bu = sum(1 for u in cells if not cell_contains0(d_ends(Cu, u))) bad += bu _DG['C'][i] = [tuple(fz(v) for v in d_ends(Ct, u)) for u in cells] num = [cell_add(x, d_ends(Ct, u)) for x, u in zip(num, cells)] log(f" T{j}: coupling to H{j+i}: twisted width max {fz(d_maxwidth(Ct, lo_c, iA)):.3e}; untwisted cells not containing 0: {bu}/{len(cells)}") Bc = [d_ends_rev(Btw, M - u - 1) for u in cells] zc = [u for u, bb in zip(cells, Bc) if not ((bb[0] > 0 and bb[1] > 0) or (bb[2] < 0 and bb[3] < 0))] if zc: log(f"T{j}: B may vanish on {len(zc)} cells, sigma in [{zc[0]/M:.5f}, {zc[-1]/M:.5f}]"); sys.exit(2) Hc = [cell_neg_quot(n_, b_) for n_, b_ in zip(num, Bc)] H[j] = d_from_cells(Hc, lo_c); untw_bad[j] = bad _DG['B'] = [tuple(fz(v) for v in x) for x in Bc]; _DG['H'] = [tuple(fz(v) for v in x) for x in Hc]; _DG['lo_c'] = lo_c; _DGALL[j] = _DG pickle.dump(dict(M=M, NB=NB, iE=iE, iA=iA, levels=_DGALL), open(f'diag2_{TAG_}_M{M}_levels.pkl', 'wb')) log(f"T{j}: B in [{fz(min(min(x[0], x[1]) for x in Bc)):+.4f}, {fz(max(max(x[2], x[3]) for x in Bc)):+.4f}]; " f"H{j} in [{fz(d_minlow(H[j], lo_c, iA)):+.4f}, {fz(d_maxup(H[j], lo_c, iA)):+.4f}]; A-part width max {fz(wA):.3e}; " f"H width max {fz(d_maxwidth(H[j], lo_c, iA)):.3e}; untwisted A/B/C cell ends not enclosing 0: {bad}") # ---------------------------------------------------------------- Step B: T0 and f(1) def Omega_rK(Hd, r, kws): """Omega^K_r[H](a) as a DPL in a (DESIGN_GM addendum 2; b = r term by the per-br slab identity; 1 <= b <= r-1 terms by blocks: refined (Part A at tau0 + Part B, DESIGN_SO addendum 1) for m >= 1, narrow/wide for m = 0)""" out = dz() HS = H_times_u(Hd, Sst(r), r * iE) Hp = Hm = None for (k, wgt) in kws: if k < 1 or wgt == 0: continue for br in range(1, k + 1): m = k - br if (br >= 2 and (0, br) not in PS) or (m >= 2 and (0, m) not in PS): continue X = d_trunc(conv_sn(HS, d_trunc(Rk(br), 0, iH)), iH) if m >= 1: X = conv_sn(X, Rk(m)) out = d_add(out, d_scale(X, wgt * (-1) ** (r + k + 1) * (-1) ** r * (-1) ** br)) for b in range(1, r): sgb = wgt * (-1) ** (r + k + 1) * (-1) ** b * (-1) ** br bl = blocks(b * iE, iA); dtc = max(c1 - c0 for c0, c1 in bl) if m == 0 or BLOCKMODE != 'refined': for (c0, c1) in bl: Kk = conv_nn(d_trunc(Sst(b), c0, c1), Sst(r - b)) w = H_times_u(Hd, Kk, r * iE) sn, sw = slab_br(br, c0, c1) Xn, Xw = conv_chain(0, sn, m), conv_chain(0, sw, m) out = d_add(out, d_scale(conv_ss(w, D(Xn.la, Xn.lb, Xw.ua, Xw.ub)), sgb)) continue for (c0, c1) in bl: # Part A: the chain exact at tau0 = c0 Kk = conv_nn(d_trunc(Sst(b), c0, c1), Sst(r - b)) w = H_times_u(Hd, Kk, r * iE) out = d_add(out, d_scale(conv_sn(w, conv_chain(0, d_trunc(Rk(br), iH - c0, iH), m)), sgb)) if Hp is None: Hp, Hm = H_pm(Hd) # Part B: S_{r-b} J+- convolved with the window range of R_m W = W_dpl(b, br, bl) Jp, Jm = corr_nn(Hp, W), corr_nn(Hm, W) S_ = Sst(r - b) Ap = d_from_cells([cell_mul(d_ends(S_, y), d_ends(Jp, y)) for y in range(M)], 0) Am = d_from_cells([cell_mul(d_ends(S_, y), d_ends(Jm, y)) for y in range(M)], 0) Gl, Gu = g_range(0, m, dtc, 'conv') Gd = D(list(Gl), list(Gl), list(Gu), list(Gu)) out = d_add(out, d_scale(d_add(conv_nn(Ap, Gd), conv_nn(Am, Gd), -1), sgb)) return out KSET = KWS T0t = (0, 0); T0u = (0, 0) for _b in range(1, NBIN + 1): T0t = add_iv(T0t, MACH[_b].T0_c(DB[_b], True)); T0u = add_iv(T0u, MACH[_b].T0_c(DB[_b], False)) log(f"T0's c-part: twisted [{fz(T0t[0]):+.6f}, {fz(T0t[1]):+.6f}]; untwisted [{fz(T0u[0]):+.6f}, {fz(T0u[1]):+.6f}]") for i in range(1, MSTAR + 1): ct, cu = coupling_at0(i, H[i]) log(f"T0's H{i} part: twisted [{fz(ct[0]):+.6f}, {fz(ct[1]):+.6f}]; untwisted [{fz(cu[0]):+.6f}, {fz(cu[1]):+.6f}] (must contain 0)") T0t = add_iv(T0t, ct); T0u = add_iv(T0u, cu) log(f"untwisted T0 (actual data): [{fz(T0u[0]):+.4e}, {fz(T0u[1]):+.4e}] (must contain 0)") # ---------------------------------------------------------------- adjoint T0 (env T0ADJ=1; DESIGN_SO addendum 5): with L H = -A c (the forcing), # sum_m = -sum_m where L* lambda = kappa is solved bottom-up: lambda_m = (kappa_m - sum_{j coupling_at0(i, H) (twisted). The A-part widths then enter once, weighted by |lambda|. T0ADJ = os.environ.get('T0ADJ', '0') == '1' def val_ss(f, g): """interval of int f g, both signed DPLs: f against g+ and g- separately (each by the exact half-hat dot products of val_sn)""" gpl, gph, gml, gmh = _pos_lo(g.la, g.lb), _pos_hi(g.ua, g.ub), _neg_lo(g.ua, g.ub), _neg_hi(g.la, g.lb) a = val_sn(f, D(gpl[0], gpl[1], gph[0], gph[1]), 'corr'); b_ = val_sn(f, D(gml[0], gml[1], gmh[0], gmh[1]), 'corr') return (a[0] - b_[1], a[1] - b_[0]) def _cf_tw(j, i): return lambda b, kw, br, m: coefK(j, i, b, kw, br + m, br, True) * wk(kw + br + m) def _slab_parts(j, i, b): combos = [cb for cb in ALLCOMBOS if cb[0] + cb[2] >= 1] cf = _cf_tw(j, i) return [cb for cb in combos if cf(b, *cb) != 0], cf def kappa_dpl(i): """representer of H -> C_{0,i}[H](0) (twisted) on u-cells [i iE, iA)""" lo_u = i * iE; cells = range(lo_u, iA) acc = d_from_cells([cell_mul(d_ends_rev(EB(i, True), M - u - 1), d_ends(Sst(i), u)) for u in cells], lo_u) for b in range(1, i + 1): cmb, cf = _slab_parts(0, i, b) if not cmb: continue bl = blocks(b * iE, iA); dtc = max(c1 - c0 for c0, c1 in bl) for (c0, c1) in bl: # Part A: chain exact at the block start Kb = d_trunc(Sst(b), c0, c1); K = Kb if b == i else conv_nn(Kb, Sst(i - b)) Q = dz() for cb in cmb: Q = d_add(Q, d_scale(conv_chain(cb[0], d_trunc(Rk(cb[1]), iH - c0, iH), cb[2]), cf(b, *cb))) acc = d_add(acc, d_from_cells([cell_mul(d_ends_rev(Q, M - u - 1), d_ends(K, u)) for u in cells], lo_u)) for br in sorted(set(cb[1] for cb in cmb)): # Part B: W * (G-range x S_{i-b}), all factors >= 0 W = W_dpl(b, br, bl) for cb in cmb: if cb[1] != br: continue Gl, Gu = g_range(cb[0], cb[2], dtc, 'corr') if b == i: kB = d_from_cells([cell_mul(d_ends(W, u), (Gl[0], Gl[0], Gu[0], Gu[0])) for u in cells], lo_u) else: S_ = Sst(i - b) kB = conv_nn(W, d_from_cells([cell_mul((Gl[y], Gl[y], Gu[y], Gu[y]), d_ends(S_, y)) for y in range(M)], 0)) acc = d_add(acc, d_scale(d_trunc(kB, lo_u, iA), cf(b, *cb))) return acc def coefK_prime(i, b, kw, ky, br, twisted): """the beta-empty part of coefK(0, i, ...): i1 = 0 small parts and kw = 0 rough parts outside the large part, i.e. the prime's own fragmentations inside the d = 1 row (its sign rho(1) is the prime's -f(1))""" if kw != 0: return 0 return rho_(1, twisted) * (-1) ** (i + ky + 1) * (-1) ** b * (-1) ** br def kappa_F(i): """representer of H -> Omega_i^K[H](1), f(1)'s H_i part (F1ADJ): minus the prime's part of kappa_dpl(i). kappa_dpl's EB term has no prime part (its vectors keep all i small parts outside the large part), so only the slab terms with kw = 0 and i1 = 0 enter; Parts A and B exactly as in kappa_dpl""" lo_u = i * iE; cells = range(lo_u, iA) acc = dz() cf = lambda b, kw, br, m: coefK_prime(i, b, kw, br + m, br, True) * wk(kw + br + m) combos = [cb for cb in ALLCOMBOS if cb[0] + cb[2] >= 1] for b in range(1, i + 1): cmb = [cb for cb in combos if cf(b, *cb) != 0] if not cmb: continue bl = blocks(b * iE, iA); dtc = max(c1 - c0 for c0, c1 in bl) for (c0, c1) in bl: # Part A: chain exact at the block start Kb = d_trunc(Sst(b), c0, c1); K = Kb if b == i else conv_nn(Kb, Sst(i - b)) Q = dz() for cb in cmb: Q = d_add(Q, d_scale(conv_chain(cb[0], d_trunc(Rk(cb[1]), iH - c0, iH), cb[2]), cf(b, *cb))) acc = d_add(acc, d_from_cells([cell_mul(d_ends_rev(Q, M - u - 1), d_ends(K, u)) for u in cells], lo_u)) for br in sorted(set(cb[1] for cb in cmb)): # Part B: W * (G-range x S_{i-b}), all factors >= 0 W = W_dpl(b, br, bl) for cb in cmb: if cb[1] != br: continue Gl, Gu = g_range(cb[0], cb[2], dtc, 'corr') if b == i: kB = d_from_cells([cell_mul(d_ends(W, u), (Gl[0], Gl[0], Gu[0], Gu[0])) for u in cells], lo_u) else: S_ = Sst(i - b) kB = conv_nn(W, d_from_cells([cell_mul((Gl[y], Gl[y], Gu[y], Gu[y]), d_ends(S_, y)) for y in range(M)], 0)) acc = d_add(acc, d_scale(d_trunc(kB, lo_u, iA), cf(b, *cb))) return d_neg(acc) def adj_coupling(j, i, lam): """C*_{j,i}[lam](u) on u-cells [(j+i) iE, iA) (twisted); lam lives on sigma-cells [j iE, iA)""" lo_u = (j + i) * iE; cells = range(lo_u, iA) LS = conv_sn(lam, Sst(i)) acc = d_from_cells([cell_mul(d_ends_rev(EB(j + i, True), M - u - 1), d_ends(LS, u)) for u in cells], lo_u) for b in range(1, i + 1): cmb, cf = _slab_parts(j, i, b) if not cmb: continue bl = blocks(b * iE, iA); dtc = max(c1 - c0 for c0, c1 in bl) for (c0, c1) in bl: Kb = d_trunc(Sst(b), c0, c1); K = Kb if b == i else conv_nn(Kb, Sst(i - b)) Q = dz() for cb in cmb: Q = d_add(Q, d_scale(conv_chain(cb[0], d_trunc(Rk(cb[1]), iH - c0, iH), cb[2]), cf(b, *cb))) LK = conv_sn(lam, K) acc = d_add(acc, d_from_cells([cell_mul(d_ends_rev(Q, M - u - 1), d_ends(LK, u)) for u in cells], lo_u)) Lam = lam if b == i else conv_sn(lam, Sst(i - b)) Lp, Lm = H_pm(Lam) for br in sorted(set(cb[1] for cb in cmb)): W = W_dpl(b, br, bl) for cb in cmb: if cb[1] != br: continue Gl, Gu = g_range(cb[0], cb[2], dtc, 'corr') GLp = d_from_cells([cell_mul((Gl[y], Gl[y], Gu[y], Gu[y]), d_ends(Lp, y)) for y in range(M)], 0) GLm = d_from_cells([cell_mul((Gl[y], Gl[y], Gu[y], Gu[y]), d_ends(Lm, y)) for y in range(M)], 0) gB = d_add(conv_nn(W, GLp), conv_nn(W, GLm), -1) acc = d_add(acc, d_scale(d_trunc(gB, lo_u, iA), cf(b, *cb))) return acc _T0_forward_for_json = T0t # audit fix 6 if T0ADJ: LAM = {}; T0adj = (0, 0) for _b in range(1, NBIN + 1): T0adj = add_iv(T0adj, MACH[_b].T0_c(DB[_b], True)) for m_ in range(1, MSTAR + 1): lo_c = m_ * iE; cells = range(lo_c, iA) rhs = kappa_dpl(m_) for j_ in range(1, m_): rhs = d_add(rhs, adj_coupling(j_, m_ - j_, LAM[j_]), -1) Bc = [d_ends_rev(Bk(m_, True), M - u - 1) for u in cells] LAM[m_] = d_from_cells([cell_neg_quot(d_ends(d_neg(rhs), u), b_) for u, b_ in zip(cells, Bc)], lo_c) Am = dz() for _b in range(1, NBIN + 1): if DB[_b]: Am = d_add(Am, MACH[_b].A_dpl(m_, True, DB[_b])) As = d_from_cells([d_ends_rev(Am, M - u - 1) for u in cells], lo_c) part = val_ss(LAM[m_], As) T0adj = add_iv(T0adj, (-part[1], -part[0])) log(f" adjoint T0: lambda_{m_} in [{fz(d_minlow(LAM[m_], lo_c, iA)):+.4f}, {fz(d_maxup(LAM[m_], lo_c, iA)):+.4f}] width max {fz(d_maxwidth(LAM[m_], lo_c, iA)):.3e}; " f"- = [{fz(-part[1]):+.6f}, {fz(-part[0]):+.6f}]") log(f"T0 forward [{fz(T0t[0]):+.6f}, {fz(T0t[1]):+.6f}]; T0 adjoint [{fz(T0adj[0]):+.6f}, {fz(T0adj[1]):+.6f}]") assert T0adj[0] <= T0t[1] and T0t[0] <= T0adj[1], "forward and adjoint T0 enclosures are disjoint: an engine error" T0t = (max(T0t[0], T0adj[0]), min(T0t[1], T0adj[1])) log(f"T0 used (intersection): [{fz(T0t[0]):+.6f}, {fz(T0t[1]):+.6f}]") F1c = (0, 0) for _b in range(1, NBIN + 1): F1c = add_iv(F1c, MACH[_b].f1_c(DB[_b])) log(f"f(1)'s c-part: [{fz(F1c[0]):+.6f}, {fz(F1c[1]):+.6f}]") F1cp = F1c OMK = {} for r in range(1, MSTAR + 1): OMK[r] = Omega_rK(H[r], r, KSET) part = d_at1(OMK[r]) log(f"f(1)'s H{r} part: [{fz(part[0]):+.6f}, {fz(part[1]):+.6f}]") F1c = add_iv(F1c, part) # ---------------------------------------------------------------- adjoint f(1) (env F1ADJ=1; DESIGN_SO_DEV_addendum7.md): f(1)'s H-part is # sum_m = -sum_m with L* nu = kappaF, solved bottom-up exactly as T0ADJ's lambda (same adj_coupling, same B). F1ADJ = os.environ.get('F1ADJ', '0') == '1' F1_forward_at_c_for_json = F1c # audit fix 4 F1_identity_checks_for_json = {} if F1ADJ: NU = {}; F1adj = F1cp for m_ in range(1, MSTAR + 1): lo_c = m_ * iE; cells = range(lo_c, iA) kF = kappa_F(m_) chk = val_ss(kF, H[m_]); fwd = d_at1(OMK[m_]) F1_identity_checks_for_json[str(m_)] = { # audit fix 4 'representer_H': [str(Fr(x, SCALE)) for x in chk], 'forward_H': [str(Fr(x, SCALE)) for x in fwd], } log(f" adjoint f(1): = [{fz(chk[0]):+.6f}, {fz(chk[1]):+.6f}] vs forward [{fz(fwd[0]):+.6f}, {fz(fwd[1]):+.6f}]") assert chk[0] <= fwd[1] and fwd[0] <= chk[1], "kappa_F does not represent f(1)'s H-part: an engine error" rhs = kF for j_ in range(1, m_): rhs = d_add(rhs, adj_coupling(j_, m_ - j_, NU[j_]), -1) Bc = [d_ends_rev(Bk(m_, True), M - u - 1) for u in cells] NU[m_] = d_from_cells([cell_neg_quot(d_ends(d_neg(rhs), u), b_) for u, b_ in zip(cells, Bc)], lo_c) Am = dz() for _b in range(1, NBIN + 1): if DB[_b]: Am = d_add(Am, MACH[_b].A_dpl(m_, True, DB[_b])) As = d_from_cells([d_ends_rev(Am, M - u - 1) for u in cells], lo_c) part = val_ss(NU[m_], As) F1adj = add_iv(F1adj, (-part[1], -part[0])) log(f" adjoint f(1): nu_{m_} in [{fz(d_minlow(NU[m_], lo_c, iA)):+.4f}, {fz(d_maxup(NU[m_], lo_c, iA)):+.4f}] width max {fz(d_maxwidth(NU[m_], lo_c, iA)):.3e}; " f"- = [{fz(-part[1]):+.6f}, {fz(-part[0]):+.6f}]") log(f"f(1) at c: forward [{fz(F1c[0]):+.6f}, {fz(F1c[1]):+.6f}]; adjoint [{fz(F1adj[0]):+.6f}, {fz(F1adj[1]):+.6f}]") assert F1adj[0] <= F1c[1] and F1c[0] <= F1adj[1], "forward and adjoint f(1) enclosures are disjoint: an engine error" F1c = (max(F1c[0], F1adj[0]), min(F1c[1], F1adj[1])) log(f"f(1) at c used (intersection): [{fz(F1c[0]):+.6f}, {fz(F1c[1]):+.6f}]") T0g = (0, 0); PHIG1 = (0, 0) for _b in range(1, NBIN + 1): T0g = add_iv(T0g, MACH[_b].T0_c(DG[_b], True)); PHIG1 = add_iv(PHIG1, MACH[_b].f1_c(DG[_b])) tiv = iv_div((-T0t[1], -T0t[0]), T0g) F1 = add_iv(F1c, iv_mul(tiv, PHIG1)) log(f"T0(c) in [{fz(T0t[0]):+.6f}, {fz(T0t[1]):+.6f}]; T0(g) in [{fz(T0g[0]):+.6f}, {fz(T0g[1]):+.6f}]; t in [{fz(tiv[0]):+.6f}, {fz(tiv[1]):+.6f}]") log(f"f(1) at c: [{fz(F1c[0]):+.6f}, {fz(F1c[1]):+.6f}]; on the segment: [{fz(F1[0]):+.6f}, {fz(F1[1]):+.6f}]") # ---------------------------------------------------------------- Step C: positivity (per-cell ranges: local first-order losses only) def realisable_free(j, k): if k == 0: return j * ETA < 1 < j * ALPHA return j * ETA + k * ALPHA < 1 < j * ALPHA + Fr(k, 2) def kset_of(kb): return kws_of(kb) free_min = 0; free_arg = None for j in range(0, int(1 / ETA) + 1): for k in range(0, 6): for bb in ([0] if k == 0 else range(1, NBIN + 1)): if not realisable_bin(j, k, bb): continue v = C3.get((j, k, bb), Fr(0)); lo = fl(v.numerator * SCALE, v.denominator) if (j, k, bb) in GDICT: w_g = GDICT[(j, k, bb)]; lo += min(0, fl(w_g.numerator * tiv[0], w_g.denominator), fl(w_g.numerator * tiv[1], w_g.denominator)) cands = [lo] if 1 <= j <= MSTAR and kin(k): w_ = wk(k) hlo = fl(w_.numerator * d_minlow(H[j], j * iE, iA), w_.denominator) if w_ > 0 else fl(w_.numerator * d_maxup(H[j], j * iE, iA), w_.denominator) cands = [lo + hlo] if j == 1 else [lo + hlo, lo] for x in cands: if x < free_min: free_min, free_arg = x, (j, k, bb) log(f"free min >= {fz(free_min):+.6f} at {free_arg}") _L0K = {} def L0K(kb): ks = tuple(kset_of(kb)) if ks not in _L0K: acc = dz() for (ky, w_) in ks: if ky >= 2 and (0, ky) in P: acc = d_add(acc, d_scale(phi(0, ky), w_)) _L0K[ks] = acc return _L0K[ks] _POK = {} def POK(kb): ks = tuple(kset_of(kb)) if ks not in _POK: acc = dz() for r in range(1, MSTAR + 1): acc = d_add(acc, OMK[r] if ks == KSET else Omega_rK(H[r], r, ks)) _POK[ks] = acc return _POK[ks] def J_fnK(jb, i2, ks): """(j_b >= 1, k_b = 0) h-term with i2 >= 1 small fragments, per rough count, as a DPL in sigma_beta (slab terms via slab_corr: kw = 0)""" wd = dict(ks) combos = sorted(set((0, br, k - br) for (k, w_) in ks for br in range(1, k + 1) if not ((br >= 2 and (0, br) not in PS) or (k - br >= 2 and (0, k - br) not in PS)))) def cf(b, kw, br, m): k = br + m if kw != 0 or wd.get(k, 0) == 0: return 0 return (-1) ** (i2 + k + 1) * (-1) ** b * (-1) ** br * wd[k] return slab_corr(H[jb + i2], (jb + i2) * iE, i2, [cf], combos)[0] JK = {jb: {i2: J_fnK(jb, i2, kset_of(0)) for i2 in range(1, MSTAR - jb + 1)} for jb in range(1, MSTAR)} log("J functions ready:", {jb: list(JK[jb]) for jb in JK}) ext_min = 0; ext_arg = None; EXT = {} def upd(val, arg): global ext_min, ext_arg key = arg[:2] if key not in EXT or val[0] < EXT[key][0]: EXT[key] = (val[0], arg) if val[0] < ext_min: ext_min, ext_arg = val[0], arg def make_rq(j): lo_c = j * iE lo_ = [min(H[j].la[u], H[j].lb[u]) for u in range(lo_c, iA)]; hi_ = [max(H[j].ua[u], H[j].ub[u]) for u in range(lo_c, iA)] SPmin, SPmax = [lo_], [hi_] while (1 << len(SPmin)) <= len(lo_): kk = len(SPmin); h = 1 << (kk - 1); a_, b_ = SPmin[-1], SPmax[-1] SPmin.append([min(a_[i], a_[i + h]) for i in range(len(a_) - h)]); SPmax.append([max(b_[i], b_[i + h]) for i in range(len(b_) - h)]) def rq(l, r): out_lo = []; out_hi = [] if r >= iA: out_lo.append(0); out_hi.append(0); r = iA - 1 if l < lo_c: out_lo.append(0); out_hi.append(0); l = lo_c if l <= r: l2, r2 = l - lo_c, r - lo_c; kk = (r2 - l2 + 1).bit_length() - 1 out_lo.append(min(SPmin[kk][l2], SPmin[kk][r2 - (1 << kk) + 1])); out_hi.append(max(SPmax[kk][l2], SPmax[kk][r2 - (1 << kk) + 1])) return (min(out_lo), max(out_hi)) if out_lo else (0, 0) return rq RQ = {j: make_rq(j) for j in range(1, MSTAR + 1)} QB = {} def block_oms(jb, kb, c0, c1): """the Omega DPLs of sigma_beta block [c0, c1), i2 = 1 .. MSTAR - jb: Hsh carries H_{jb+i2}'s range over cells c0+i .. c1+i at shift cell i""" oms = [] for i2 in range(1, MSTAR - jb + 1): Hsh = d_const_cells([RQ[jb + i2](c0 + i, c1 + i) for i in range(i2 * iE, iA)], i2 * iE) oms.append(Omega_rK(Hsh, i2, kset_of(kb))) return oms def get_QB(jb, kb): key = (jb, tuple(kset_of(kb))) if key not in QB: lst = [] for (c0, c1) in blocksQ(jb * iE, iA, nbq_of(jb)): hr = RQ[jb](c0, c1 - 1) lst.append((c0, c1, hr, block_oms(jb, kb, c0, c1))) QB[key] = lst return QB[key] HBB = {}; HBG = {} for jb in range(0, int(1 / (2 * ETA)) + 1): for kb in range(0, 4): for b in range(1, NBIN + 1): HBB[(jb, kb, b)] = MACH[b].H_beta(jb, kb, DB[b]); HBG[(jb, kb, b)] = MACH[b].H_beta(jb, kb, DG[b]) def sum_d(lst): acc = None for x in lst: if x is None: continue acc = x if acc is None else d_add(acc, x) return acc POS2D = os.environ.get('POS2D', 'fast') def _sp(vals, op): tab = [vals] while (1 << len(tab)) <= len(vals): hh = 1 << (len(tab) - 1); a_ = tab[-1] tab.append([op(a_[i], a_[i + hh]) for i in range(len(a_) - hh)]) def q(l, r): k = (r - l + 1).bit_length() - 1 return op(tab[k][l], tab[k][r - (1 << k) + 1]) return q HRQ = {} for _j in range(1, MSTAR + 1): _Hj = H[_j] HRQ[_j] = (_sp([min(_Hj.la[u], _Hj.lb[u]) for u in range(M)], min), _sp([max(_Hj.ua[u], _Hj.ub[u]) for u in range(M)], max)) def min_aq(n, L0_, L1_): """lower bound of min over a in a-cell n of a * L(a), L the linear function with (scaled) end values L0_, L1_ (a > 0): the endpoint values minus the convexity slack (h^2/8) * 2 L' = (h/4) max(L1 - L0, 0)""" return min(fl(n * L0_, M), fl((n + 1) * L1_, M)) - ce(max(L1_ - L0_, 0), 4 * M) def t_ends(Hg, n): """lower bound ends of t * Hg on a-cell n, t in tiv (the min of linear functions is concave, so the chord through the end minima is below)""" if Hg is None: return (0, 0) out = [] for lo_, hi_ in ((Hg.la[n], Hg.ua[n]), (Hg.lb[n], Hg.ub[n])): out.append(min(fl(tv * hv, SCALE) for tv in tiv for hv in (lo_, hi_))) return tuple(out) def mulL(Hv, L0e, e): """lower bound of Hv * L0(a) at a-end e (Hv a constant)""" return fl(Hv * (L0e[e] if Hv >= 0 else L0e[2 + e]), SCALE) LEAVES = {} def ext_blockouter(jb, kb): """addendum 8 (B)/(C): the 'elif hasH' branch of the loop below (its prelude and its POS2D fast path) for one class (jb, kb) with sigma_beta blocks, evaluated block by block. Pass 1 stores the per-(bin, a-cell) prelude (and makes the 'noH' updates); pass 2 computes one block's Omegas at a time and commits the block's minimum (only leaves, under QADAPT). Returns [(c0, c1, depth, value or None)].""" assert POS2D == 'fast', 'QOUTER needs the POS2D fast path' L0b = L0K(kb); rqmin, rqmax = HRQ[jb]; pre = [] for bbeta in range(1, NBIN + 1): tlo, thi = class_total_range(jb, kb, bbeta) if tlo >= Fr(1, 2): continue Hb = sum_d([HBB[(jb, kb, b)] for b in range(max(bbeta, 1), NBIN + 1)]) Hg = sum_d([HBG[(jb, kb, b)] for b in range(max(bbeta, 1), NBIN + 1)]) for n in range(iH, M): tau_lo, tau_hi = 1 - Fr(n + 1, M), 1 - Fr(n, M) if tau_hi <= tlo or tau_lo >= thi: continue tg = t_ends(Hg, n) b0 = (Hb.la[n] if Hb is not None else 0) + tg[0]; b1 = (Hb.lb[n] if Hb is not None else 0) + tg[1] sg_lo = max(jb * ETA, tau_lo - kb * EDGES[bbeta]); sg_hi = min(jb * ALPHA, tau_hi - EDGES[bbeta - 1] - (kb - 1) * ALPHA) if sg_lo > sg_hi: continue if sg_hi > ALPHA: upd((min_aq(n, b0, b1),), (jb, kb, n / M, 'noH')) L0e = d_ends(L0b, n) u_lo = max(jb * iE, int(sg_lo * M) - 1); u_hi = min(iA - 1, int(min(sg_hi, ALPHA) * M)) while u_lo <= u_hi and Fr(u_lo + 1, M) < sg_lo: u_lo += 1 while u_hi >= u_lo and Fr(u_hi, M) > min(sg_hi, ALPHA): u_hi -= 1 if u_lo > u_hi: continue pre.append((bbeta, n, b0, b1, L0e, u_lo, u_hi)) if not pre: return [] def eval_block(c0, c1): if not any(max(e[5], c0) <= min(e[6], c1 - 1) for e in pre): return None # no part: the n-outer loop makes no update here either oms = block_oms(jb, kb, c0, c1); best = None for (bbeta, n, b0, b1, L0e, u_lo, u_hi) in pre: p0, p1 = max(u_lo, c0), min(u_hi, c1 - 1) if p0 > p1: continue hmn, hmx = rqmin(p0, p1), rqmax(p0, p1) L0_ = b0 + min(mulL(hmn, L0e, 0), mulL(hmx, L0e, 0)); L1_ = b1 + min(mulL(hmn, L0e, 1), mulL(hmx, L0e, 1)) for om in oms: L0_ += om.la[n]; L1_ += om.lb[n] v = min_aq(n, L0_, L1_) if best is None or v < best[0]: best = (v, (jb, kb, n / M, p0 / M, 'b%d' % bbeta)) return best ad = QADAPT.get((jb, kb)); leaves = [] roots = blocksQ(jb * iE, iA, nbq_of(jb)) assert roots[0][0] == jb * iE and roots[-1][1] == iA and all(a < b for (a, b) in roots) and \ all(roots[i][1] == roots[i + 1][0] for i in range(len(roots) - 1)), 'sigma_beta base blocks do not tile [jb iE, iA): an engine error' for (r0, r1) in roots: stack = [(r0, r1, 0)]; mine = [] while stack: c0, c1, d = stack.pop() best = eval_block(c0, c1) if ad is not None and best is not None and best[0] < ad[0] and d < ad[2] and c1 - c0 > 1: for (s0, s1) in reversed(blocksQ(c0, c1, ad[1])): stack.append((s0, s1, d + 1)) # refine: the parent is not committed continue if best is not None: upd((best[0],), best[1]) mine.append((c0, c1, d, None if best is None else best[0])) mine.sort(key=lambda x: (x[0], x[1])) # check: the committed leaves tile [r0, r1) exactly assert mine and mine[0][0] == r0 and mine[-1][1] == r1 and all(x[0] < x[1] for x in mine) and \ all(mine[i][1] == mine[i + 1][0] for i in range(len(mine) - 1)), f'leaves of block [{r0}, {r1}) do not tile it: an engine error' if ad is not None: log(f" block-outer {(jb, kb)} block [{r0}, {r1}): {len(mine)} leaves") leaves.extend(mine) return leaves for jb in range(0, int(1 / (2 * ETA)) + 1): for kb in range(0, 4): QB.pop((jb, tuple(kset_of(kb - 1))), None) if kb >= 1 else None # memory: drop the previous k_b's sigma-block Omegas (one (j_b, k_b) cached at a time) if QOUTER and 1 <= jb < MSTAR and kb >= 1 and len(kset_of(kb)) > 0: # (B)/(C): exactly the classes whose QBl is not None below LEAVES[(jb, kb)] = ext_blockouter(jb, kb); continue for bbeta in ([0] if kb == 0 else range(1, NBIN + 1)): if (jb, kb) == (0, 0): continue tlo, thi = class_total_range(jb, kb, bbeta) if tlo >= Fr(1, 2): continue Hb = sum_d([HBB[(jb, kb, b)] for b in range(max(bbeta, 1), NBIN + 1)]) Hg = sum_d([HBG[(jb, kb, b)] for b in range(max(bbeta, 1), NBIN + 1)]) L0b = L0K(kb); hasH = (jb >= 1 and jb <= MSTAR and len(kset_of(kb)) > 0) QBl = get_QB(jb, kb) if (hasH and kb >= 1 and jb < MSTAR) else None for n in range(iH, M): tau_lo, tau_hi = 1 - Fr(n + 1, M), 1 - Fr(n, M) if tau_hi <= tlo or tau_lo >= thi: continue tg = t_ends(Hg, n) b0 = (Hb.la[n] if Hb is not None else 0) + tg[0]; b1 = (Hb.lb[n] if Hb is not None else 0) + tg[1] if jb == 0: Pk = POK(kb); upd((min_aq(n, b0 + Pk.la[n], b1 + Pk.lb[n]),), (jb, kb, n / M)) elif hasH and kb == 0: ib = M - n - 1 if ib >= iA: upd((min_aq(n, b0, b1),), (jb, kb, n / M)); continue if ib < jb * iE: continue Hj = H[jb] pr = cell_mul((Hj.lb[ib], Hj.la[ib], Hj.ub[ib], Hj.ua[ib]), d_ends(L0b, n)) L0_, L1_ = b0 + pr[0], b1 + pr[1] for i2 in range(1, MSTAR - jb + 1): Jd = JK[jb][i2]; L0_ += Jd.lb[ib]; L1_ += Jd.la[ib] upd((min_aq(n, L0_, L1_),), (jb, kb, n / M, ib / M)) elif hasH: # beta's kb rough parts: the largest in bin bbeta, the others in [alpha', E[bbeta]], so their total lies in # [E[bbeta-1] + (kb-1) alpha', kb E[bbeta]]; with one bin this is the old [kb alpha', kb/2] sg_lo = max(jb * ETA, tau_lo - kb * EDGES[bbeta]); sg_hi = min(jb * ALPHA, tau_hi - EDGES[bbeta - 1] - (kb - 1) * ALPHA) if sg_lo > sg_hi: continue if sg_hi > ALPHA: upd((min_aq(n, b0, b1),), (jb, kb, n / M, 'noH')) L0e = d_ends(L0b, n); Hj = H[jb] u_lo = max(jb * iE, int(sg_lo * M) - 1); u_hi = min(iA - 1, int(min(sg_hi, ALPHA) * M)) while u_lo <= u_hi and Fr(u_lo + 1, M) < sg_lo: u_lo += 1 while u_hi >= u_lo and Fr(u_hi, M) > min(sg_hi, ALPHA): u_hi -= 1 if u_lo > u_hi: continue if POS2D == 'slow': for u in range(u_lo, u_hi + 1): Hv = (Hj.la[u], Hj.lb[u], Hj.ua[u], Hj.ub[u]) m0 = min(mulL(h_, L0e, 0) for h_ in Hv); m1 = min(mulL(h_, L0e, 1) for h_ in Hv) L0_, L1_ = b0 + m0, b1 + m1 if QBl is not None: for (c0, c1, hr, oms) in QBl: if c0 <= u < c1: for om in oms: L0_ += om.la[n]; L1_ += om.lb[n] break upd((min_aq(n, L0_, L1_),), (jb, kb, n / M, u / M, 'b%d' % bbeta)) continue # fast: per sigma-block, the H extremes over the sub-range (phi(h) = h * L0-bound is concave in h, so its minimum over a # range of H values sits at the range's extremes); the bracket is concave in sigma's H, so per a-end minima give a valid chord rqmin, rqmax = HRQ[jb] parts = [(u_lo, u_hi, ())] if QBl is None else \ [(max(u_lo, c0), min(u_hi, c1 - 1), oms) for (c0, c1, hr, oms) in QBl if max(u_lo, c0) <= min(u_hi, c1 - 1)] for (p0, p1, oms) in parts: hmn, hmx = rqmin(p0, p1), rqmax(p0, p1) L0_ = b0 + min(mulL(hmn, L0e, 0), mulL(hmx, L0e, 0)); L1_ = b1 + min(mulL(hmn, L0e, 1), mulL(hmx, L0e, 1)) for om in oms: L0_ += om.la[n]; L1_ += om.lb[n] upd((min_aq(n, L0_, L1_),), (jb, kb, n / M, p0 / M, 'b%d' % bbeta)) else: upd((min_aq(n, b0, b1),), (jb, kb, n / M)) for _key in sorted(LEAVES): # (B)/(C) _lv = LEAVES[_key]; _byd = {} for (_c0, _c1, _d, _v) in _lv: _byd[_d] = _byd.get(_d, 0) + 1 _vals = [_v for (_c0, _c1, _d, _v) in _lv if _v is not None] log(f" block-outer {_key}: {len(_lv)} leaves, by depth {dict(sorted(_byd.items()))}, min leaf " + (f"{fz(min(_vals)):+.6f}" if _vals else "none")) log(f"extended min >= {fz(ext_min):+.6f} at {ext_arg}") for key in sorted(EXT): log(f" extended min for (j_b, k_b) = {key}: {fz(EXT[key][0]):+.6f} at {EXT[key][1]}") m_ = min(free_min, ext_min) margin = 1 - Fr(-m_) / Fr(-F1[1]) if (m_ < 0 and F1[1] < 0) else Fr(-1) verdict = "PASS" if (F1[1] < 0 and m_ > F1[1] and all(v == 0 for v in untw_bad.values()) and T0u[0] <= 0 <= T0u[1]) else "FAIL" log(f"VERDICT {verdict}: min f >= {fz(m_):+.6f}, f(1)_hi = {fz(F1[1]):+.6f}, scaled margin >= {float(margin):.5f} (print rounding: quote the JSON value rounded down); untwisted bad {untw_bad}") json.dump(dict(M=M, NB=NB, tag=TAG_, engine='second-order DPL (gm2/cert2.py, development)', script_sha256=hashlib.sha256(open(__file__, 'rb').read()).hexdigest(), dpl_sha256=hashlib.sha256(open(dpl.__file__, 'rb').read()).hexdigest(), block_mode=BLOCKMODE, pos2d=POS2D, nbq=NBQ, nbqlo=NBQLO, t0adj=T0ADJ, f1adj=F1ADJ, maxs=MAXS, kprof=('all' if KPROF is None else sorted(KPROF)), kw=(None if KW is None else {str(k): str(v) for k, v in KW.items()}), eta=str(ETA), alpha=str(ALPHA), mstar=MSTAR, edges=[str(e) for e in EDGES], gcls=list(GCLS), gvec={str(k): str(v) for k, v in GDICT.items()}, pickle_sha256=PKL_SHA, c={str(kk): str(v) for kk, v in sorted(C3.items())}, T0=[str(Fr(x, SCALE)) for x in T0t], T0u=[str(Fr(x, SCALE)) for x in T0u], T0g=[str(Fr(x, SCALE)) for x in T0g], t=[str(Fr(x, SCALE)) for x in tiv], f1=[str(Fr(x, SCALE)) for x in F1], free_min=str(Fr(free_min, SCALE)), ext_min=str(Fr(ext_min, SCALE)), ext_arg=str(ext_arg), untwisted_bad=untw_bad, verdict=verdict, margin=str(margin), nbqj={str(k): v for k, v in sorted(NBQJ.items())}, qouter=QOUTER, # addendum 8 qadapt={f'{k[0]},{k[1]}': [str(Fr(v[0], SCALE)), v[1], v[2]] for k, v in sorted(QADAPT.items())}, leaves={f'{k[0]},{k[1]}': [[a, b, d, (None if v is None else str(Fr(v, SCALE)))] for (a, b, d, v) in LEAVES[k]] for k in sorted(LEAVES)}, T0_forward=[str(Fr(x, SCALE)) for x in _T0_forward_for_json], # audit fix 6 T0_adjoint=([str(Fr(x, SCALE)) for x in T0adj] if T0ADJ else None), free_bound_arg=str(free_arg), free_bound_domain='H over [j*eta,alpha] independently of class; no-H candidate for j>=2', gcls_legacy=bool(os.environ.get('GVEC')), F1_constant_part=[str(Fr(x, SCALE)) for x in F1cp], # audit fix 4 F1_forward_at_c=[str(Fr(x, SCALE)) for x in F1_forward_at_c_for_json], F1_adjoint_at_c=([str(Fr(x, SCALE)) for x in F1adj] if F1ADJ else None), F1_used_at_c=[str(Fr(x, SCALE)) for x in F1c], F1_identity_checks=F1_identity_checks_for_json), open(f'cert2_{TAG_}_M{M}.json', 'x'), indent=1) ~~~~~~~~~~~~ #### file: analyse_diag2.py - kind: script - as run: SHA-256 6f6e8a4706261f03a925309f27140282a6caee4886e02bc9e5a2ff2f55be870e - as shipped: SHA-256 e254d7ca03ac3223332c0defe412609c87b9501eedfb4a81d1e219799d27c7a6 - identity: identical, after dropping docstrings and comments, to the file as run - trailing newline: yes ~~~~~~~~~~~~ python #!/usr/bin/env python3 # analyse_diag2.py PKL: per-level width profile from cert2_f3_diag2.py's dump: for each level j, where the width of H is largest, # the extreme losses that the free bound sees, and the split of the width between the A-part and each coupling at the cell of largest # width. Floats; a diagnostic. import sys, pickle d = pickle.load(open(sys.argv[1], 'rb')); M = d['M']; iE = d['iE']; iA = d['iA'] def w(x): return max(x[2] - x[0], x[3] - x[1]) def mid(x): return ((x[0] + x[2]) / 2, (x[1] + x[3]) / 2) print(f"M = {M}, NB = {d['NB']}, levels {sorted(d['levels'])}") for j in sorted(d['levels'], reverse=True): L = d['levels'][j]; lo = L['lo_c']; H = L['H']; n = len(H) ws = [w(x) for x in H]; k = max(range(n), key=lambda i: ws[i]) lowmin = min(min(x[0], x[1]) for x in H); midmin = min(min(mid(x)) for x in H) upmax = max(max(x[2], x[3]) for x in H); midmax = max(max(mid(x)) for x in H) kl = min(range(n), key=lambda i: min(H[i][0], H[i][1])); ku = max(range(n), key=lambda i: max(H[i][2], H[i][3])) q = [0, n // 20, n // 5, n // 2, n] reg = [max(ws[q[a]:q[a + 1]]) for a in range(4)] print(f"T{j}: cells {n} (tau {lo/M:.4f}..{iA/M:.4f}); H width max {ws[k]:.3e} at tau {(lo+k)/M:.5f}; region max widths " f"[first 5%, 5-20%, 20-50%, 50-100%] = " + ", ".join(f"{r:.2e}" for r in reg)) print(f" free-bound view: min lower {lowmin:+.5f} (mid {midmin:+.5f}, loss {midmin-lowmin:.2e}) at tau {(lo+kl)/M:.5f}; " f"max upper {upmax:+.5f} (mid {midmax:+.5f}, loss {upmax-midmax:.2e}) at tau {(lo+ku)/M:.5f}") parts = [('A', L['A'][k])] + [(f'C{i}', L['C'][i][k]) for i in sorted(L['C'])] print(" at the max-width cell: " + "; ".join(f"{nm} {w(x):.2e}" for nm, x in parts) + f"; B {min(L['B'][k]):+.4f}..{max(L['B'][k]):+.4f}") for nm, arr in [('A', L['A'])] + [(f'C{i}', L['C'][i]) for i in sorted(L['C'])]: wa = [w(x) for x in arr]; ka = max(range(n), key=lambda i: wa[i]); avg = sum(wa) / n print(f" {nm}: width max {wa[ka]:.2e} at tau {(lo+ka)/M:.5f}; mean {avg:.2e}") ~~~~~~~~~~~~ #### file: cont_hms_rec6.py - kind: script - as run: SHA-256 0f7cf5c18aa03aa407958652b3b26a885ae6d61afd066559e3bfaad7d0aeddf0 - as shipped: SHA-256 48952db2ce40420cdf6382f3d31049e0fae1c35c7296a276a425df877b945be4 - identity: identical, after dropping docstrings and comments, to the file as run - trailing newline: yes ~~~~~~~~~~~~ python #!/usr/bin/env python3 # cont_hms_rec6.py: the programme of record for the families of section 8; see its docstring and the experiments # README's "Conventions of the family programmes". """cont_hms_rec6.py: the programme of record for the families of section 8. Free data f(u) = c[j, k, b] + 1[tau <= alpha'] H_j(tau, m) + 1[k = 1] G_j(tau, m): with CONT_NOGREC=1 and CONT_TIEM=1 the forced family, with CONT_TIEM=1 alone the one-rough (G) family, and with neither the family whose data also see the largest small component m. The programme minimises f(1) subject to the twisted rows and to f >= -1 at the generated grid points, free and extended (strides ASUB, SSUB and CONT_MSUB; no upper positivity condition when CONT_NOUPPER is set), with the constants in [-1, U] and the added functions bounded by C (CONT_GCAP); CONT_LBC is refused. The grid conventions are listed in the experiments README under "Conventions of the family programmes"; in particular there is no positivity row at E = (1/2, 1/2), and the H columns on the interval of Lemma 8.1 are free, bounded by C. Before solving, the program checks the untwisted (Type I) rows as a hard gate and stops unless they hold, with an exactness re-check under closed weights at 1/2 in a second run of itself (CONT_HALFCLOSE, CONT_GATEONLY); its GATE line states the tolerances. CONT_NOGATE disables the stop, and CONT_GATE_SELFTEST emulates an earlier kernel defect as a positive control of the gate. With CONT_NOX, after the solve it prints FREEMIN_CLOSED, the minimum of f over every free grid node. Floats; not a certificate. Usage: N eta_num eta_den alpha_num alpha_den EDGES U ASUB SSUB [--check], with U the upper bound of the class constants. Environment: CONT_NOUPPER, CONT_GCAP, CONT_TIEM, CONT_NOGREC, CONT_MSUB, CONT_DEADEPS (the rows of Lemma 8.1 imposed with absolute residual at most this value), CONT_DEADSCALE (those rows and their right-hand sides multiplied by this factor; 1024 in the recipes), CONT_TINY (the coefficient cutoff, default 1e-9), CONT_NOX (scipy's bundled HiGHS, interior point without crossover), CONT_PRESOLVE, CONT_PTOL, CONT_OTOL, CONT_TLIM, CONT_DISP, CONT_SAVEX, and the gate options above. With CONT_NOX, for each solve it prints the status, the objective on the matrices after and before the cutoff, the residuals and positivity violations on both, an acceptance flag (status Optimal; equality residuals at most 1e-7, the rows of Lemma 8.1 within their bound plus 1e-10, positivity and box violations at most 1e-7) and binary64 weak-duality lower bounds (not certificates).""" import sys, time, os, pickle from fractions import Fraction as Fr import numpy as np from scipy.signal import fftconvolve from scipy.optimize import linprog from scipy.sparse import csr_matrix t0 = time.time() N = int(sys.argv[1]); eta = Fr(int(sys.argv[2]), int(sys.argv[3])); alpha = Fr(int(sys.argv[4]), int(sys.argv[5])) EDG = [Fr(e) for e in sys.argv[6].split(',')] if sys.argv[6] else [] PHIMAX = float(sys.argv[7]); ASUB = int(sys.argv[8]); SSUB = int(sys.argv[9]) assert eta <= alpha < Fr(1, 2) h = 1.0 / N def idx(x): v = x * N; assert v.denominator == 1, f"{x} not on grid N={N}"; return int(v) ie, ia, ihf = idx(eta), idx(alpha), idx(Fr(1, 2)) edges = [alpha] + EDG + [Fr(1, 2)]; iedges = [idx(e) for e in edges]; NB = len(edges) - 1 assert all(edges[i_] < edges[i_ + 1] for i_ in range(len(edges) - 1)), 'bin edges must be strictly increasing between alpha and 1/2' inv = np.zeros(N + 1); inv[1:] = 1.0 / (np.arange(1, N + 1) * h) def conv(F, G): return fftconvolve(F, G)[:N + 1] * h delta = np.zeros(N + 1); delta[0] = 1.0 / h HC = os.environ.get('CONT_HALFCLOSE', 'trap'); HCF = {'trap': 0.5, 'closed': 1.0}[HC] # rec5: weight of a boundary at exactly 1/2 SELFTEST = bool(os.environ.get('CONT_GATE_SELFTEST')) # rec5: positive control of the gate (emulates the old a = 1/2 defect) def band(lo, hi): F = np.zeros(N + 1); F[lo:hi + 1] = inv[lo:hi + 1]; F[lo] *= 0.5; F[hi] *= 0.5; return F def trunc_ge(F, i0): G = F.copy(); G[:max(i0, 0)] = 0.0 if 0 <= i0 <= N: G[i0] *= HCF return G def shift(F, i): G = np.zeros(N + 1); G[i:] = F[:N + 1 - i]; return G s_den = band(ie, ia) JMAX = int(1 / eta) + 1; KMAX = int(1 / alpha) + 1 S = [delta] for j in range(1, JMAX + 1): S.append(conv(S[-1], s_den) / j) S2 = S[2] Rcap = {} for e in iedges[1:]: r = band(ia, e) if HC == 'closed' and e == ihf: r[ihf] = 0.0 # rec5 closed: a component exactly 1/2 is large P = [delta] for k in range(1, KMAX + 1): P.append(conv(P[-1], r) / k) Rcap[e] = P def Rbin(k, b): if k == 0: return delta hi = iedges[b] return Rcap[hi][k] if b == 1 else Rcap[hi][k] - Rcap[iedges[b - 1]][k] Rall = Rcap[ihf] bins_k = lambda k: [0] if k == 0 else list(range(1, NB + 1)) etaf, alf, ef = float(eta), float(alpha), [float(e) for e in edges] def support(j, k, b): if k == 0: return (j * etaf, j * alf) return (j * etaf + (k - 1) * alf + ef[b - 1], j * alf + k * ef[b]) def in_sup(sup, t): return sup[1] > sup[0] and sup[0] - 1e-12 <= t <= sup[1] + 1e-12 print(f"[{time.time()-t0:.1f}s] N={N} eta={eta} alpha={alpha} edges={[str(e) for e in edges]}", flush=True) # ---- Lemma K kernels (c-part) ---- PS = {} for e in iedges[1:]: for bs in range(JMAX + 1): for br in range(KMAX + 1): PS[(e, bs, br)] = conv(S[bs], Rcap[e][br]) if br else S[bs] def Lam_cap(jy, ky, e): K = jy + ky; L = -PS[(e, jy, ky)].copy(); acc = np.zeros(N + 1) for bs in range(jy + 1): for br in range(ky + 1): if (bs, br) in ((0, 0), (jy, ky)): continue acc += (-1) ** (bs + br) * conv(trunc_ge(PS[(e, bs, br)], ihf), PS[(e, jy - bs, ky - br)]) return L + (-1) ** (K + 1) * acc Lam = {} for jy in range(JMAX + 1): for ky in range(KMAX + 1): if jy + ky < 2: continue if ky == 0: Lam[(jy, 0, 0)] = Lam_cap(jy, 0, ihf); continue prev = None for b in range(1, NB + 1): cur = Lam_cap(jy, ky, iedges[b]); Lam[(jy, ky, b)] = cur if prev is None else cur - prev; prev = cur Lam0 = sum(Lam_cap(0, ky, ihf) for ky in range(2, KMAX + 1)) del PS # ---- XS(t; theta) for theta = 1/2 - x, x in [0, ia] ---- XS = {} for x in range(0, ia + 1): acc = np.zeros(N + 1) for k in range(1, KMAX + 1): for br in range(1, k + 1): acc += (-1) ** (k + br) * conv(trunc_ge(Rall[br], ihf - x), Rall[k - br]) XS[x] = acc # ======================= cont_hms ======================= MS = ia // ie while MS * ie > ia: MS -= 1 Hgrid = {j: list(range(j * ie, ia + 1)) for j in range(1, MS + 1)} Rall = Rcap[ihf] Rsgn = sum(((-1) ** k) * Rall[k] for k in range(0, KMAX + 1)); Runs = sum(Rall[k] for k in range(0, KMAX + 1)) na = ia + 1; ng = N - ia + 1 JG = min(JMAX, (N - ia) // ie) R1 = Rall[1] XSmat = np.array([XS[x] for x in range(0, ia + 1)]) XS1 = np.zeros((ng, N + 1)) for sg_ in range(0, min(ng, ihf)): XS1[sg_] = trunc_ge(R1, ihf - sg_) if ihf < ng: XS1[ihf] = (1.0 - HCF) * R1 # small sub-sum exactly 1/2: S and S + r cancel in the weight HCF tr_mask = (np.arange(N + 1) >= ihf).astype(float); tr_mask[ihf] = HCF Lam0Tr = trunc_ge(Lam0, ihf) K0sgn = conv(Rsgn, Lam0Tr); K0uns = conv(Runs, Lam0Tr) mgrid = list(range(ie, ia + 1)) JS = max(JMAX, JG) def restricted(m): """kernels with every EXTRA small part <= m (grid index)""" sm = s_den.copy(); sm[m + 1:] = 0.0 Sm = [delta] for j in range(1, JS + 1): Sm.append(conv(Sm[-1], sm) / j) w = h * sm # M_j over sigma, tau in [0, ng) Mj = [None] * (JG + 1); Mj[0] = np.zeros((ng, ng)); Mj[0][0, 0] = 1.0 / h for j in range(1, JG + 1): acc = np.zeros((ng, ng)) for t in range(ie, m + 1): if w[t] == 0.0: continue acc[:, t:] += w[t] * Mj[j - 1][:, :ng - t] acc[t:, t:] -= w[t] * Mj[j - 1][:ng - t, :ng - t] Mj[j] = acc / j LJ = {}; Ks = {}; Ku = {} for j in range(1, MS + 1): A = (Mj[j][:na, :na].T @ XSmat) * ((-1) ** (j - 1)) L = np.zeros((na, N + 1)) for tau in Hgrid[j]: L[tau, tau:] = A[tau, :N + 1 - tau] if HC == 'trap': L[:, ihf] *= 2.0 # rec4 (1): right-limit value at a = 1/2, as Lam_cap if SELFTEST: L[:, ihf] *= 0.5 LJ[j] = L LT = L * tr_mask[None, :] Ks[j] = np.array([conv(Rsgn, LT[t]) for t in range(na)]); Ku[j] = np.array([conv(Runs, LT[t]) for t in range(na)]) LG = {}; LGT = {} for j in range(1, JG + 1): A = (Mj[j].T @ XS1) * ((-1) ** (j - 1)) L = np.zeros((ng, N + 1)) for tau in range(j * ie, ng): L[tau, tau:] = A[tau, :N + 1 - tau] if HC == 'trap': L[:, ihf] *= 2.0 # rec4 (1), G kernels if SELFTEST: L[:, ihf] *= 0.5 LG[j] = L; LGT[j] = L * tr_mask[None, :] LAS = {} for jy in range(2, JG + 1): acc = np.zeros(N + 1) for bs in range(1, jy): acc += (-1) ** bs * conv(trunc_ge(Sm[bs], ihf), Sm[jy - bs]) LAS[jy] = -Sm[jy] + ((-1) ** (jy + 1)) * acc # all-small Lemma K kernel, parts <= m (Lam_cap(jy, 0, .) with S restricted) LAST = {jy: trunc_ge(L, ihf) for jy, L in LAS.items()} return dict(S=Sm, LJ=LJ, Ks=Ks, Ku=Ku, LG=LG, LGT=LGT, LAS=LAS, LAST=LAST) RK = {} for m in mgrid: RK[m] = restricted(m) EMPTY = None print(f"[{time.time()-t0:.1f}s] restricted kernels for {len(mgrid)} maxima (MS {MS}, JG {JG})", flush=True) if '--kcheck' in sys.argv: top = RK[ia] print(" kcheck S_j(<= ia) vs S_j:", max(np.abs(top['S'][j] - S[j]).max() for j in range(1, min(JS, JMAX) + 1))) print(" kcheck LAS(<= ia) vs Lam[(jy,0,0)]:", max(np.abs(top['LAS'][jy] - Lam[(jy, 0, 0)]).max() for jy in top['LAS'] if (jy, 0, 0) in Lam)) def Sd(m, j, x): # density of j extra parts with total x and maximum EXACTLY m (vectorised in x) a = RK[m]['S'][j][x] return a - RK[m - 1]['S'][j][x] if m - 1 >= ie else a # ---- variables ---- classes = [(j, k, b) for j in range(JMAX + 1) for k in range(KMAX + 1) if j + k >= 1 for b in bins_k(k)] D = {cl: (conv(S[cl[0]], Rbin(cl[1], cl[2])) if cl[1] else S[cl[0]]) for cl in classes}; D[(0, 0, 0)] = delta cid = {cl: n for n, cl in enumerate(classes)}; nc = len(classes) def feas_max(j, tau): # maxima m with j parts of total tau and maximum exactly m: exact combinatorial test (17:5x; testing if j == 1: return [tau] if ie <= tau <= ia else [] # Sd > 0 admitted FFT round-off, giving impossible (tau, m) rows and variables) return [m for m in mgrid if m <= tau - (j - 1) * ie and j * m >= tau] hid = {}; gid = {}; nv = nc for j in range(1, MS + 1): for tau in Hgrid[j]: for m in feas_max(j, tau): hid[(j, tau, m)] = nv; nv += 1 GBAND = bool(os.environ.get('CONT_GBAND')) # restricted family: G resolved by m only on the dead band tau - m in [1-2a, a] glump = {} for j in range(2, JG + 1): for tg in range(ihf + 1, N - ia + 1): if S[j][tg] <= 0.0: continue for m in feas_max(j, tg): if GBAND and not (N - 2 * ia <= tg - m <= ia): if (j, tg) not in glump: glump[(j, tg)] = nv; nv += 1 gid[(j, tg, m)] = glump[(j, tg)] else: gid[(j, tg, m)] = nv; nv += 1 print(f" variables: {nc} constants, {len(hid)} H(j, tau, m), {len(gid)} G(j, tau, m) keys in {len(set(gid.values()))} columns" f"{' (GBAND: %d lumped)' % len(glump) if GBAND else ''}", flush=True) def new(): return np.zeros(nv) LamTr = {k_: trunc_ge(L, ihf) for k_, L in Lam.items()} w_classes = [(0, 0, 0)] + [cl for cl in classes if cl[0] + cl[1] >= 1 and in_sup(support(*cl), support(*cl)[0]) and support(*cl)[0] <= 0.5 + 1e-12] _cc = {} def conv_w_lam(wc, key): k_ = (wc, key) if k_ not in _cc: _cc[k_] = LamTr[key] if wc == (0, 0, 0) else conv(D[wc], LamTr[key]) return _cc[k_] # ---- f(1) ---- f1 = new() for key, L in Lam.items(): if (cl := cid.get(key)) is not None: f1[cl] += L[N] def dmax(m, key, j, idx): # RK[m][key][j][idx] - RK[m-1][key][j][idx] a = RK[m][key][j][idx] return a - RK[m - 1][key][j][idx] if m - 1 >= ie else a for (j, tau, m), col in hid.items(): f1[col] += h * dmax(m, 'LJ', j, (tau, N)) for (j, tg, m), col in gid.items(): f1[col] += h * dmax(m, 'LG', j, (tg, N)) # ---- rows (jD, sigma, mD) ---- row_list = [(0, 0, 0)] for jD in range(1, MS + 1): for s_ in Hgrid[jD]: for mD in feas_max(jD, s_): row_list.append((jD, s_, mD)) rid = {r: n for n, r in enumerate(row_list)}; nr_ = len(row_list) T = np.zeros((nr_, nv)); I = np.zeros((nr_, nv)); DIAG = np.zeros((nr_, nv), dtype=bool) # c-part (independent of mD): computed per (jD, sigma) as in cont_hm, copied to every mD cpart = {} for jD in range(0, MS + 1): sig = [0] if jD == 0 else Hgrid[jD]; tt = np.array([N - s_ for s_ in sig]) Tc = np.zeros((len(sig), nc)); Ic = np.zeros((len(sig), nc)) for cl in classes: jp, kp, bp = cl; tgt = cid.get((jD + jp, kp, bp)) if tgt is None: continue vals = D[cl][tt]; sg = (-1) ** (jD + jp + kp); Tc[:, tgt] += sg * vals; Ic[:, tgt] += vals for wc in w_classes: jw, kw, bw = wc; sgn = (-1) ** (1 + jD + jw + kw) for key in Lam: jy, ky, byy = key; tgt = cid.get((jD + jw + jy, kw + ky, max(bw, byy))) if tgt is None: continue v = conv_w_lam(wc, key)[tt]; Tc[:, tgt] += sgn * v; Ic[:, tgt] += v for n_, s_ in enumerate(sig): cpart[(jD, s_)] = (Tc[n_], Ic[n_]) for r, (jD, s_, mD) in enumerate(row_list): T[r, :nc] = cpart[(jD, s_)][0]; I[r, :nc] = cpart[(jD, s_)][1] print(f"[{time.time()-t0:.1f}s] c-part of {nr_} rows", flush=True) # extra-parts kernels per maximum m: KH[m][D][sigma, tau] (signed, jD-independent part) and KG[m][D][sigma, tg]; unsigned KHu, KGu def build_K(m): R = RK[m]; Sm = R['S'] KH = {}; KHu = {}; KG = {}; KGu = {} sig_all = np.arange(na) for Dl in range(1, MS + 1): Ht = np.zeros((na, na)); Hu = np.zeros((na, na)) for s_ in range(na): taus = np.arange(s_, na); x = taus - s_ v = h * Sm[Dl][x] * Rsgn[N - taus]; Ht[s_, taus] += ((-1) ** Dl) * v; Hu[s_, taus] += h * Sm[Dl][x] * Runs[N - taus] for jw in range(0, Dl + 1): jy = Dl - jw; sg = (-1) ** (1 + jw) if jw == 0: vt = h * R['Ks'][Dl][x, N - s_]; vu = h * R['Ku'][Dl][x, N - s_] elif jy == 0: vt = h * Sm[jw][x] * K0sgn[N - taus]; vu = h * Sm[jw][x] * K0uns[N - taus] else: vt = np.zeros(len(taus)); vu = np.zeros(len(taus)) for i_, t_ in enumerate(taus): vv = np.arange(s_, t_ + 1) vt[i_] = h * h * np.dot(Sm[jw][vv - s_], R['Ks'][jy][t_ - vv, N - vv]) vu[i_] = h * h * np.dot(Sm[jw][vv - s_], R['Ku'][jy][t_ - vv, N - vv]) Ht[s_, taus] += sg * vt; Hu[s_, taus] += vu KH[Dl] = Ht; KHu[Dl] = Hu gt = np.arange(ihf + 1, ng) for Dl in range(1, JG + 1): Gt = np.zeros((na, ng)); Gu = np.zeros((na, ng)) for s_ in range(na): x = gt - s_ v = h * Sm[Dl][x] * R1[N - gt]; Gt[s_, gt] += ((-1) ** (Dl + 1)) * v; Gu[s_, gt] += v for jw in range(0, Dl + 1): jy = Dl - jw if jy >= 1: if jw == 0: vt = h * R['LGT'][jy][x, N - s_] else: vt = np.zeros(len(gt)) for i_, t_ in enumerate(gt): vv = np.arange(s_, t_ + 1) vt[i_] = h * h * np.dot(Sm[jw][vv - s_], R['LGT'][jy][t_ - vv, N - vv]) Gt[s_, gt] += ((-1) ** (1 + jw)) * vt; Gu[s_, gt] += vt if jy >= 2: base = R['LAST'][jy] if jw == 0 else conv(Sm[jw], R['LAST'][jy]) vt = h * R1[N - gt] * base[x] Gt[s_, gt] += ((-1) ** jw) * vt; Gu[s_, gt] += vt KG[Dl] = Gt; KGu[Dl] = Gu return KH, KHu, KG, KGu KK = {m: build_K(m) for m in mgrid} print(f"[{time.time()-t0:.1f}s] extra-part kernels built", flush=True) def Kexact(m, which, Dl): # kernel with the extra parts' maximum EXACTLY m: K(<= m) - K(<= m-1) a = KK[m][which][Dl] return a - KK[m - 1][which][Dl] if m - 1 >= ie else a # assemble the H- and G-parts of every row for r, (jD, s_, mD) in enumerate(row_list): sgD = (-1) ** jD if jD >= 1: # diagonal: H_jD(sigma, mD), all-rough completions (free and extended) col = hid[(jD, s_, mD)] T[r, col] += sgD * (Rsgn[N - s_] - K0sgn[N - s_]); I[r, col] += Runs[N - s_] + K0uns[N - s_]; DIAG[r, col] = True for (J, tau, M), col in hid.items(): if J <= jD or tau < s_: continue Dl = J - jD if M < mD: continue if jD == 0: kt = Kexact(M, 0, Dl)[0, tau]; ku = Kexact(M, 1, Dl)[0, tau] elif M == mD: kt = KK[mD][0][Dl][s_, tau]; ku = KK[mD][1][Dl][s_, tau] else: kt = Kexact(M, 0, Dl)[s_, tau]; ku = Kexact(M, 1, Dl)[s_, tau] T[r, col] += sgD * kt; I[r, col] += ku for (J, tg, M), col in gid.items(): if J <= jD: continue Dl = J - jD if Dl > JG or M < mD: continue if jD == 0: kt = Kexact(M, 2, Dl)[0, tg]; ku = Kexact(M, 3, Dl)[0, tg] elif M == mD: kt = KK[mD][2][Dl][s_, tg]; ku = KK[mD][3][Dl][s_, tg] else: kt = Kexact(M, 2, Dl)[s_, tg]; ku = Kexact(M, 3, Dl)[s_, tg] T[r, col] += sgD * kt; I[r, col] += ku print(f"[{time.time()-t0:.1f}s] rows assembled", flush=True) Hc = np.array(sorted(set(hid.values()))); Gc = np.array(sorted(set(gid.values())), dtype=int) offH = np.where(DIAG[:, Hc], 0.0, I[:, Hc]) print(f" Type I check: c-block {np.abs(I[:, :nc]).max():.3e}; H diagonal {np.abs(I[DIAG]).max():.3e}; H integrated/h {np.abs(offH).max()/h:.3e};" f" G/h {np.abs(I[:, Gc]).max()/h if len(Gc) else 0:.3e}; T0 row: c {np.abs(I[0, :nc]).max():.3e}, H/h {np.abs(I[0, Hc]).max()/h:.3e}," f" G/h {np.abs(I[0, Gc]).max()/h if len(Gc) else 0:.3e}", flush=True) _ti = dict(c=float(np.abs(I[:, :nc]).max()), Hd=float(np.abs(I[DIAG]).max()), Hi_h=float(np.abs(offH).max() / h), G_h=float(np.abs(I[:, Gc]).max() / h) if len(Gc) else 0.0) if HC == 'closed': _fail = [k_ for k_ in ('c', 'Hd', 'Hi_h', 'G_h') if not _ti[k_] <= 1e-9] print(f" GATE (closed-1/2 exactness): {'PASS' if not _fail else 'FAIL ' + str(_fail)} {_ti}", flush=True) if os.environ.get('CONT_GATEONLY'): sys.exit(3 if _fail else 0) else: _fail = [k_ for k_, ok_ in (('G_h', _ti['G_h'] <= 1e-9), ('c', _ti['c'] <= 10 * h), ('Hd', _ti['Hd'] <= 2 * h)) if not ok_] import subprocess as _sp _env = dict(os.environ, CONT_HALFCLOSE='closed', CONT_GATEONLY='1') _r = _sp.run([sys.executable, '-B', os.path.abspath(__file__)] + sys.argv[1:], env=_env, capture_output=True, text=True) _gl = [l_ for l_ in _r.stdout.splitlines() if 'GATE' in l_] if _r.returncode != 0: _fail.append('closed-1/2 exactness (exit %d): %s' % (_r.returncode, _gl[-1].strip() if _gl else _r.stderr[-300:])) print(f" GATE (trapezoid: G/h <= 1e-9, c <= 10h, H diag <= 2h; closed-1/2 exactness <= 1e-9): {'PASS' if not _fail else 'FAIL ' + str(_fail)}; " f"{_ti}; closed: {_gl[-1].strip() if _gl else 'no output'}", flush=True) if _fail and not os.environ.get('CONT_NOGATE'): sys.exit(f'cont_hms_rec5: GATE FAILED {_fail}') if os.environ.get('CONT_GATEONLY'): sys.exit(0) if '--check' in sys.argv: sys.exit() # ---- optional diagnostics ---- TDEAD = None if os.environ.get('DEAD_SVD'): # diagnostic: spectrum of the dead rows' G-block (and H-block size), then exit dead_ = [r for r, (jD_, s_, m_) in enumerate(row_list) if jD_ >= 1 and s_ >= N - 2 * ia + 2] gc_ = np.array(sorted(set(gid.values())), dtype=int) hc_ = np.array(sorted(set(hid.values())), dtype=int) Bd = T[np.ix_(dead_, gc_)] U_, sv, _ = np.linalg.svd(Bd, full_matrices=False) rk = int((sv > 1e-9 * sv[0]).sum()) NL = U_[:, rk:] # left null space of the dead G-block (dead-row combinations G cannot touch) Cd = T[np.ix_(dead_, np.arange(nc))] if NL.shape[1]: Mc = NL.T @ Cd; svc = np.linalg.svd(Mc, compute_uv=False) touched = int((np.abs(Cd).max(axis=0) > 1e-12).sum()) print(f" DEAD_NULL: G-rank {rk} of {len(dead_)} dead rows -> {NL.shape[1]} dead-row combinations G cannot touch; on the class constants they have rank " f"{(svc > 1e-9 * svc[0]).sum() if svc[0] > 0 else 0} (largest sv {svc[0]:.2e}); constants touched by dead rows {touched} of {nc}", flush=True) else: print(f" DEAD_NULL: G-block has full row rank {rk} = {len(dead_)}: no G-free dead-row combination", flush=True) ks = [k for k in (1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024) if k < len(sv)] print(f" DEAD_SVD s_max: dead rows {len(dead_)} x G cols {len(gc_)}; sv0 {sv[0]:.3e}; sv/sv0 at k={ks}: " + ", ".join(f"{sv[k]/sv[0]:.1e}" for k in ks) + f"; #(>1e-3 sv0) {(sv > 1e-3*sv[0]).sum()}, #(>1e-6) {(sv > 1e-6*sv[0]).sum()}, #(>1e-9) {(sv > 1e-9*sv[0]).sum()}; H-block max {np.abs(T[np.ix_(dead_, hc_)]).max():.2e}", flush=True) sys.exit(0) if os.environ.get('CONT_DEADEPS'): # relax the dead rows to |T_dead x| <= eps (they make the equality LP numerically degenerate) dead = [r for r, (jD_, s_, m_) in enumerate(row_list) if jD_ >= 1 and s_ >= N - 2 * ia] live = [r for r in range(nr_) if r not in set(dead)] TDEAD = T[dead].copy(); T = T[live]; nr_ = len(live) print(f" CONT_DEADEPS = {os.environ['CONT_DEADEPS']}: {len(dead)} dead rows relaxed to |T x| <= eps", flush=True) if os.environ.get('CONT_DROPDEAD'): keep = [r for r, (jD_, s_, m_) in enumerate(row_list) if jD_ == 0 or s_ < N - 2 * ia] print(f" CONT_DROPDEAD: keeping {len(keep)} of {nr_} rows", flush=True) T = T[keep]; nr_ = len(keep) # ---- positivity (sparse) ---- from scipy.sparse import coo_matrix, csr_matrix as _csr GCAP = float(os.environ.get('CONT_GCAP', '50')); MSUB = int(os.environ.get('CONT_MSUB', '1')) lb = np.full(nv, -np.inf); ub = np.full(nv, np.inf); lb[:nc] = -1.0; ub[:nc] = PHIMAX nvac = 0 for (j, tau, m), col in hid.items(): lb[col] = -GCAP; ub[col] = GCAP # rec6 (2): the H columns of the interval of Lemma 8.1 are free too for col in gid.values(): lb[col] = -GCAP; ub[col] = GCAP if os.environ.get('CONT_NOGREC'): for col in gid.values(): lb[col] = 0.0; ub[col] = 0.0 print(f" CONT_NOGREC: {len(set(gid.values()))} G columns fixed to 0", flush=True) if os.environ.get('CONT_LBC'): sys.exit('cont_hms_rec4: CONT_LBC is not supported (focused read M1)') LBC = os.environ.get('CONT_LBC'); low_cls = [] if LBC is not None: LBC = float(LBC) for cl_ in classes: if cl_[0] >= 1 and (cl_[1] >= 2 or (cl_[1] == 1 and cl_[0] >= 2 and not os.environ.get('CONT_NOGREC'))): lb[cid[cl_]] = LBC; low_cls.append(cl_) hid_arr = -np.ones((MS + 1, na, na), dtype=int) for (j, tau, m), col in hid.items(): hid_arr[j, tau, m] = col gid_arr = -np.ones((JG + 1, N + 1, na), dtype=int) for (j, tg, m), col in gid.items(): gid_arr[j, tg, m] = col # cumulative kernels as arrays over the maximum: LJc[jy][m, tau_y, a], LGc[jy][m, tau_y, a], LASc[jy][m, a] mi = np.array(mgrid); NM = len(mgrid) LJc = {jy: np.array([RK[m]['LJ'][jy] for m in mgrid]) for jy in range(1, MS + 1)} LGc = {jy: np.array([RK[m]['LG'][jy] for m in mgrid]) for jy in range(1, JG + 1)} LASc = {jy: np.array([RK[m]['LAS'][jy] for m in mgrid]) for jy in range(2, JG + 1)} def exact_from(cum, mb): """rows over M = mb..ia of the 'maximum exactly M, or <= mb at M = mb' kernel: cum has a leading axis over mgrid (18:0x: callers now pass cum already sliced at the a-index, so this copies small arrays only)""" k0 = mb - ie out = cum[k0:].copy() out[1:] -= cum[k0:-1] return out # out[0] = cum(<= mb); out[i] = cum(<= mb+i) - cum(<= mb+i-1) LJa = {jy: np.ascontiguousarray(np.moveaxis(LJc[jy], 2, 0)) for jy in LJc} # [a][m, tau_y] LGa = {jy: np.ascontiguousarray(np.moveaxis(LGc[jy], 2, 0)) for jy in LGc} LASa = {jy: np.ascontiguousarray(LASc[jy].T) for jy in LASc} # [a][m] from array import array as _arr Rr, Rc, Rv = _arr('q'), _arr('q'), _arr('d'); nrow = 0; desc = [] def emit(entries): global nrow for c_, v_ in entries.items(): if v_ != 0.0: Rr.append(nrow); Rc.append(c_); Rv.append(v_) nrow += 1 def emit_arrays(base_dict, cols, vals): d = dict(base_dict) for c_, v_ in zip(cols, vals): d[c_] = d.get(c_, 0.0) + v_ emit(d) # free rows for cl in classes: j, k, b = cl if not in_sup(support(j, k, b), 1.0) or j == 0 or k == 0: continue supR = support(0, k, b) for (jj, tau, m), col in hid.items(): if jj == j and in_sup(supR, 1 - tau * h) and lb[col] != ub[col]: emit({cid[cl]: 1.0, col: 1.0}); desc.append(('free', cl)) if k == 1: seen = set() for (jj, tg, m), col in gid.items(): if jj == j and in_sup(supR, 1 - tg * h) and col not in seen: seen.add(col); emit({cid[cl]: 1.0, col: 1.0}); desc.append(('freeG', cl)) # every free grid shape; classes with a shape carrying no active added term keep c >= -1 _fs_cls, _fs_h, _fs_g = [], [], [] def _parts(total, lower, prefix): if total == 0: yield prefix; return for p_ in range(lower, min(ihf - 1, total) + 1): if total == p_ or total - p_ >= p_: yield from _parts(total - p_, p_, prefix + (p_,)) _pure = set() for _sh in _parts(N, ie, ()): _sm = [q for q in _sh if q <= ia]; _ro = [q for q in _sh if q > ia] _j, _k, _tau = len(_sm), len(_ro), sum(_sm) _b = 0 if not _ro else next(i_ for i_ in range(1, len(iedges)) if max(_ro) <= iedges[i_]) _cl = (_j, _k, _b) if _cl not in cid: continue _hc = hid.get((_j, _tau, max(_sm))) if (_sm and _tau <= ia) else None _gc = gid.get((_j, _tau, max(_sm))) if (_sm and _k == 1) else None _act = [cc_ for cc_ in (_hc, _gc) if cc_ is not None and not (lb[cc_] == 0 and ub[cc_] == 0)] if not _act: _pure.add(_cl) _fs_cls.append(cid[_cl]); _fs_h.append(_hc if _hc is not None else -1); _fs_g.append(_gc if _gc is not None else -1) _fs_cls = np.array(_fs_cls); _fs_h = np.array(_fs_h); _fs_g = np.array(_fs_g) _restored = [] for _cl in sorted(_pure): if lb[cid[_cl]] < -1: lb[cid[_cl]] = -1.0; _restored.append(_cl) _cn_cls, _cn_h, _cn_g, _cn_bare = [], [], [], [] # rec4 (3): closed-interval free nodes, as band() weights them for _cl in classes: _j, _k, _b = _cl if _k == 0: _taus = [N] if (_j >= 1 and _j * ie <= N <= _j * ia) else [] else: _sR = support(0, _k, _b); _taus = [t_ for t_ in range(_j * ie, min(_j * ia, N) + 1) if in_sup(_sR, 1 - t_ * h)] if _j >= 1 else ([0] if in_sup(_sR, 1.0) else []) for _t in _taus: for _m in (feas_max(_j, _t) if _j >= 1 else [None]): _hc = hid.get((_j, _t, _m)) if (_j >= 1 and _t <= ia) else None _gc = gid.get((_j, _t, _m)) if (_j >= 1 and _k == 1) else None _hc = _hc if (_hc is not None and not (lb[_hc] == 0 and ub[_hc] == 0)) else None _gc = _gc if (_gc is not None and not (lb[_gc] == 0 and ub[_gc] == 0)) else None _cn_cls.append(cid[_cl]); _cn_h.append(-1 if _hc is None else _hc); _cn_g.append(-1 if _gc is None else _gc) _cn_bare.append(_hc is None and _gc is None) _cn_cls = np.array(_cn_cls); _cn_h = np.array(_cn_h); _cn_g = np.array(_cn_g); _cn_bare = np.array(_cn_bare) print(f" REC4: {len(_cn_cls)} closed-interval free nodes (class, tau, m), {int(_cn_bare.sum())} bare (constant only); constants lower bound min {lb[:nc].min():g}", flush=True) print(f" REC3: {len(_fs_cls)} free grid shapes; classes with a pure-constant free shape: {len(_pure)}; lower bound restored to -1 on {len(_restored)}: {_restored}", flush=True) npure = 0 if LBC is not None: for cl_ in low_cls: j_, k_, b_ = cl_ if k_ != 1: continue missing = False for rho in range(max(iedges[b_ - 1], ia + 1), min(iedges[b_], ihf - 1) + 1): tg = N - rho if not (j_ * ie <= tg <= j_ * ia): continue for m_ in feas_max(j_, tg): if (j_, tg, m_) not in gid: missing = True; break if missing: break if missing: emit({cid[cl_]: 1.0}); desc.append(('pure_c', cl_)); npure += 1 print(f" CONT_LBC = {LBC}: lowered {len(low_cls)} class constants; restored c >= -1 rows: {npure}", flush=True) nfree = nrow alist = list(range(ihf, N, ASUB)) for bc in w_classes: jb, kb, bb = bc if bc == (0, 0, 0): continue supR = support(0, kb, bb) for ai in alist: a = ai * h; tb = N - ai if not in_sup(support(jb, kb, bb), tb * h): continue base = {} for key, L in Lam.items(): jy, ky, bp = key; tgt = cid.get((jb + jy, kb + ky, max(bb, bp))) if tgt is not None: base[tgt] = base.get(tgt, 0.0) + a * L[ai] if jb == 0: cols = []; vals = [] for jy in range(1, MS + 1): # fragments carry the whole small part: H(jy, tau_y, M), M = their maximum ex = exact_from(LJa[jy][ai], ie) # (NM, na): maximum exactly M for k0, M in enumerate(mgrid): cc = hid_arr[jy, :, M]; ok = cc >= 0 if ok.any(): cols += list(cc[ok]); vals += list(h * a * ex[k0][ok]) if kb == 1: for jy in range(2, JG + 1): ex = exact_from(LASa[jy][ai], ie) for k0, M in enumerate(mgrid): cg = gid_arr[jy, ai, M] if ai <= N else -1 if cg >= 0: cols.append(cg); vals.append(a * ex[k0]) if bc == (0, 1, NB) and ai == ihf: continue # rec6 (1): no positivity row at E = (1/2, 1/2) emit_arrays(base, cols, vals); desc.append(('ext', bc)); continue if kb: cands = [t_ for t_ in range(jb * ie, min(jb * ia, tb) + 1, SSUB) if in_sup(supR, (tb - t_) * h)] else: cands = [tb] if jb * ie <= tb <= jb * ia else [] done_const = False for tbeta in cands: mbs = feas_max(jb, tbeta) if tbeta <= ia else [None] if tbeta > ia and kb >= 2: if not done_const: emit(dict(base)); desc.append(('ext', bc)); done_const = True continue if tbeta > ia: mbs = feas_max(jb, tbeta) for mb in mbs[::MSUB]: cols = []; vals = [] if tbeta <= ia: c0 = hid_arr[jb, tbeta, mb] if jb <= MS else -1 if c0 >= 0: cols.append(c0); vals.append(a * Lam0[ai]) for jy in range(1, MS - jb + 1): ex = exact_from(LJa[jy][ai], mb) # (ia - mb + 1, na) for k0 in range(ex.shape[0]): M = mb + k0; taus_y = np.arange(jy * ie, ia - tbeta + 1) if len(taus_y) == 0: continue cc = hid_arr[jb + jy, tbeta + taus_y, M]; ok = cc >= 0 if ok.any(): cols += list(cc[ok]); vals += list(h * a * ex[k0][taus_y[ok]]) if kb == 0: for jy in range(1, JG - jb + 1): ex = exact_from(LGa[jy][ai], mb) taus_y = np.arange(jy * ie, ng); tt_ = tbeta + taus_y; okt = tt_ <= N taus_y, tt_ = taus_y[okt], tt_[okt] for k0 in range(ex.shape[0]): M = mb + k0; cc = gid_arr[jb + jy, tt_, M]; ok = cc >= 0 if ok.any(): cols += list(cc[ok]); vals += list(h * a * ex[k0][taus_y[ok]]) elif kb == 1: for jy in range(2, JG - jb + 1): ex = exact_from(LASa[jy][ai], mb) tt_ = tbeta + ai if tt_ > N: continue for k0 in range(len(ex)): cg = gid_arr[jb + jy, tt_, mb + k0] if cg >= 0: cols.append(cg); vals.append(a * ex[k0]) emit_arrays(base, cols, vals); desc.append(('ext', bc)) Gm = coo_matrix((Rv, (Rr, Rc)), shape=(nrow, nv)).tocsr() print(f"[{time.time()-t0:.1f}s] positivity rows: free {nfree}, extended {nrow - nfree}; nnz {Gm.nnz}; vacuous H fixed {nvac}; |L| <= {GCAP}", flush=True) # ---- control: tie L(j, tau, m) across m (must reproduce cont_hmg) ---- Aeq = _csr(T) if os.environ.get('CONT_TIEM'): tr, tc, tv = [], [], []; nt = 0 for store in (hid, gid): groups = {} for (j, tau, m), col in store.items(): groups.setdefault((j, tau), []).append(col) for cols in groups.values(): cols = sorted(set(cols)) for c1, c2 in zip(cols[:-1], cols[1:]): tr += [nt, nt]; tc += [c1, c2]; tv += [1.0, -1.0]; nt += 1 Aeq = _csr(np.vstack([T, coo_matrix((tv, (tr, tc)), shape=(nt, nv)).toarray()])) if nt else Aeq print(f" CONT_TIEM: {nt} tie rows", flush=True) NOUP = bool(os.environ.get('CONT_NOUPPER')) from scipy.sparse import vstack as _svs Aub = -Gm if NOUP else _svs([-Gm, Gm]).tocsr(); bub = np.ones(nrow) if NOUP else np.concatenate([np.ones(nrow), np.full(nrow, PHIMAX)]) _RAW = dict(T=T.copy(), f1=f1.copy(), Gm=Gm.copy(), TDEAD=(TDEAD.copy() if TDEAD is not None else None)) TINY = float(os.environ.get('CONT_TINY', '1e-9')) # remove coefficients below CONT_TINY from T, the dead rows, the positivity rows and the objective; this perturbs the rows and the objective, and the residuals reported below refer to the modified matrices T[np.abs(T) < TINY] = 0.0; f1[np.abs(f1) < TINY] = 0.0 Gm.data[np.abs(Gm.data) < TINY] = 0.0; Gm.eliminate_zeros() Aeq = _csr(T) if not os.environ.get('CONT_TIEM') else Aeq if os.environ.get('CONT_TIEM'): Aeq = Aeq.tocsr(); Aeq.data[np.abs(Aeq.data) < TINY] = 0.0; Aeq.eliminate_zeros() Aub = -Gm if NOUP else _svs([-Gm, Gm]).tocsr() if TDEAD is not None: TDEAD[np.abs(TDEAD) < TINY] = 0.0; epsd = float(os.environ['CONT_DEADEPS']) DS = float(os.environ.get('CONT_DEADSCALE', '1')) Aub = _svs([Aub, _csr(DS * TDEAD), _csr(-DS * TDEAD)]).tocsr(); bub = np.concatenate([bub, np.full(2 * TDEAD.shape[0], DS * epsd)]) print(f" sanity: finite T {np.isfinite(T).all()}, f1 {np.isfinite(f1).all()}, G {np.isfinite(Gm.data).all()}; max|T| {np.abs(T).max():.3e}, max|G| {np.abs(Gm.data).max():.3e}, max|f1| {np.abs(f1).max():.3e}; min|nonzero T| {np.abs(T[T != 0]).min():.3e}", flush=True) if os.environ.get('CONT_FEASCHECK'): r0 = linprog(np.zeros(nv), A_ub=Aub, b_ub=bub, A_eq=Aeq, b_eq=np.zeros(Aeq.shape[0]), bounds=np.column_stack([lb, ub]), method='highs', options={'presolve': True}) print(f" feasibility LP (zero objective): status {r0.status} ({r0.message})", flush=True) if os.environ.get('CONT_NOX'): # bundled HiGHS, IPM, run_crossover off; primal value + residuals + a float dual bound from scipy.optimize._highspy import _core as _hc from scipy.sparse import vstack as _vs2 from types import SimpleNamespace as _NS hh_ = _hc._Highs() for k_, v_ in (('output_flag', bool(os.environ.get('CONT_DISP'))), ('solver', 'ipm'), ('run_crossover', 'off'), ('presolve', 'on' if int(os.environ.get('CONT_PRESOLVE', '0')) else 'off'), ('ipm_optimality_tolerance', float(os.environ.get('CONT_OTOL', '1e-8'))), ('primal_feasibility_tolerance', float(os.environ.get('CONT_PTOL', '1e-7'))), ('dual_feasibility_tolerance', float(os.environ.get('CONT_PTOL', '1e-7'))), ('time_limit', float(os.environ.get('CONT_TLIM', '1e9')))): hh_.setOptionValue(k_, v_) INF_ = hh_.getInfinity() A_ = _vs2([Aeq, Aub]).tocsc(); A_.sort_indices(); neq_ = Aeq.shape[0]; m_ = A_.shape[0] rl_ = np.concatenate([np.zeros(neq_), np.full(Aub.shape[0], -INF_)]); ru_ = np.concatenate([np.zeros(neq_), np.asarray(bub, dtype=np.float64)]) hh_.addRows(m_, rl_, ru_, 0, np.zeros(m_, dtype=np.int32), np.zeros(0, dtype=np.int32), np.zeros(0)) lbh_ = np.where(np.isinf(lb), -INF_, lb); ubh_ = np.where(np.isinf(ub), INF_, ub) hh_.addCols(nv, np.asarray(f1, dtype=np.float64), lbh_, ubh_, A_.nnz, A_.indptr[:-1].astype(np.int32), A_.indices.astype(np.int32), A_.data.astype(np.float64)) print(f"[{time.time()-t0:.1f}s] NOX: model {hh_.getNumRow()} rows x {hh_.getNumCol()} cols, nnz {hh_.getNumNz()}; running IPM (no crossover)", flush=True) hh_.run() st_ = hh_.modelStatusToString(hh_.getModelStatus()); info_ = hh_.getInfo(); sol_ = hh_.getSolution() xx_ = np.asarray(sol_.col_value, dtype=np.float64); yy_ = np.asarray(sol_.row_dual, dtype=np.float64) pv_ = float(f1 @ xx_) if len(xx_) == nv else float('nan') if len(xx_) == nv: eqr_ = float(np.abs(Aeq @ xx_).max()) if neq_ else 0.0; ubv_ = float(np.maximum(Aub @ xx_ - bub, 0).max()) bdv_ = float(max(np.maximum(lb - xx_, 0).max(), np.maximum(xx_ - ub, 0).max())) else: eqr_ = ubv_ = bdv_ = float('nan') # float weak-duality bound: y free on equality rows, y <= 0 on the <= rows (clipped), d = c - A^T y, every column boxed db_ = float('nan') if len(yy_) == m_: y_ = yy_.copy(); y_[neq_:] = np.minimum(y_[neq_:], 0.0) d_ = np.asarray(f1, dtype=np.float64) - A_.T @ y_ if np.isfinite(lb).all() and np.isfinite(ub).all(): db_ = float(y_[neq_:] @ np.asarray(bub, dtype=np.float64) + np.sum(np.where(d_ >= 0, d_ * lb, d_ * ub))) print(f" NOX: status {st_}; ipm iters {info_.ipm_iteration_count}, crossover iters {info_.crossover_iteration_count}; HiGHS objective " f"{info_.objective_function_value:.9f}; primal value f1.x = {pv_:.9f} (max |Aeq x| {eqr_:.1e}, max ub violation {ubv_:.1e}, " f"max bound violation {bdv_:.1e}); float dual bound {db_:.9f}; HiGHS max primal/dual infeas {info_.max_primal_infeasibility:.1e} / " f"{info_.max_dual_infeasibility:.1e}", flush=True) if os.environ.get('CONT_SAVEX'): np.save(os.environ['CONT_SAVEX'], xx_) if len(xx_) == nv and len(yy_) == m_: # residuals, acceptance, net-multiplier dual bounds (pruned and raw) DS_ = float(os.environ.get('CONT_DEADSCALE', '1')); epsd_ = float(os.environ.get('CONT_DEADEPS', '0')) nd_ = TDEAD.shape[0] if TDEAD is not None else 0; npos_ = Gm.shape[0] live_p = float(np.abs(Aeq @ xx_).max()) if neq_ else 0.0 dead_p = float(np.abs(TDEAD @ xx_).max()) if nd_ else 0.0 pos_p = float((Gm @ xx_).min()); boxv = float(max(np.maximum(lb - xx_, 0).max(), np.maximum(xx_ - ub, 0).max())) live_r = float(max(np.abs(_RAW['T'] @ xx_).max(), (np.abs(Aeq[_RAW['T'].shape[0]:] @ xx_).max() if neq_ > _RAW['T'].shape[0] else 0.0))) if neq_ else 0.0 dead_r = float(np.abs(_RAW['TDEAD'] @ xx_).max()) if nd_ else 0.0 pos_r = float((_RAW['Gm'] @ xx_).min()); obj_r = float(_RAW['f1'] @ xx_) acc_ = (st_ == 'Optimal') and (live_p <= 1e-7) and (dead_p <= epsd_ + 1e-10) and (pos_p >= -1 - 1e-7) and (boxv <= 1e-7) y_ = yy_.copy(); y_[neq_:neq_ + npos_] = np.minimum(y_[neq_:neq_ + npos_], 0.0) if nd_: yp_, ym_ = y_[neq_ + npos_:neq_ + npos_ + nd_].copy(), y_[neq_ + npos_ + nd_:].copy(); lam_ = yp_ - ym_ y_[neq_ + npos_:neq_ + npos_ + nd_] = np.minimum(lam_, 0.0); y_[neq_ + npos_ + nd_:] = np.minimum(-lam_, 0.0) bfull_ = np.concatenate([np.zeros(neq_), np.asarray(bub, dtype=np.float64)]) def _bound(Amat, cvec): d__ = np.asarray(cvec, dtype=np.float64) - Amat.T @ y_ terms = np.where(d__ >= 0, d__ * lb, d__ * ub); val = float(bfull_ @ y_ + terms.sum()) allow = 1e-13 * (float(np.abs(bfull_ * y_).sum()) + float(np.abs(terms).sum())) return val - allow Lp_ = _bound(A_, f1) nT_ = _RAW['T'].shape[0] parts = [_csr(_RAW['T'])] + ([Aeq[nT_:]] if neq_ > nT_ else []) + [-_RAW['Gm']] + ([_csr(DS_ * _RAW['TDEAD']), _csr(-DS_ * _RAW['TDEAD'])] if nd_ else []) Lr_ = _bound(_vs2(parts).tocsr(), _RAW['f1']) _fv = xx_[_fs_cls] + np.where(_fs_h >= 0, xx_[np.maximum(_fs_h, 0)], 0.0) + np.where(_fs_g >= 0, xx_[np.maximum(_fs_g, 0)], 0.0) _fvc = xx_[_cn_cls] + np.where(_cn_h >= 0, xx_[np.maximum(_cn_h, 0)], 0.0) + np.where(_cn_g >= 0, xx_[np.maximum(_cn_g, 0)], 0.0) print(f" FREEMIN_CLOSED: min f over {len(_fvc)} closed-interval free nodes = {float(_fvc.min()):.12f}; over bare nodes = {float(_fvc[_cn_bare].min()) if _cn_bare.any() else float('nan'):.12f}", flush=True) print(f" FREEMIN: min f over all {len(_fs_cls)} free grid shapes = {float(_fv.min()):.12f}", flush=True) print(f" REC: status {st_}; objective pruned {pv_:.9f} raw {obj_r:.9f}; float dual lower bound pruned {Lp_:.9f} raw {Lr_:.9f}; " f"pruned residuals live {live_p:.2e} dead {dead_p:.2e} (eps {epsd_:g}, scale {DS_:g}) min positivity {pos_p:.12f} box viol {boxv:.1e}; " f"raw residuals live {live_r:.2e} dead {dead_r:.2e} min positivity {pos_r:.12f}; ACCEPT {acc_}; rows: live {neq_}, positivity {npos_}, dead {nd_}", flush=True) if os.environ.get('CONT_SAVEX'): np.savez_compressed(os.environ['CONT_SAVEX'] + '.rec.npz', x=xx_, y=yy_, lb=lb, ub=ub) res = _NS(status=0 if st_ == 'Optimal' else 1, message=f'NOX {st_}', x=xx_ if st_ == 'Optimal' else None, fun=pv_ if st_ == 'Optimal' else None, nit=info_.ipm_iteration_count) else: res = linprog(f1, A_ub=Aub, b_ub=bub, A_eq=Aeq, b_eq=np.zeros(Aeq.shape[0]), bounds=np.column_stack([lb, ub]), method=os.environ.get('CONT_METHOD', 'highs-ipm'), options=dict(presolve=bool(int(os.environ.get('CONT_PRESOLVE', '0'))), **({'primal_feasibility_tolerance': float(os.environ['CONT_PTOL']), 'dual_feasibility_tolerance': float(os.environ['CONT_PTOL'])} if os.environ.get('CONT_PTOL') else {}), **({'ipm_optimality_tolerance': float(os.environ['CONT_OTOL'])} if os.environ.get('CONT_OTOL') else {}), **({'time_limit': float(os.environ['CONT_TLIM'])} if os.environ.get('CONT_TLIM') else {}), **({'disp': True} if os.environ.get('CONT_DISP') else {}))) print(f"[{time.time()-t0:.1f}s] LP status {res.status} ({res.message})", flush=True) print(f" result: fun {getattr(res, 'fun', None)}; nit {getattr(res, 'nit', None)}", flush=True) # night2 if res.status != 0 and getattr(res, 'x', None) is not None: # report the returned nonoptimal point and its residuals; feasibility is not asserted xx_ = np.asarray(res.x); print(f" NONOPT POINT: f1.x = {float(f1 @ xx_):.6f}; max eq residual {float(np.abs(Aeq @ xx_).max()):.2e}; max ub violation {float(np.maximum(Aub @ xx_ - bub, 0).max()):.2e}", flush=True) np.save(os.environ.get('CONT_SAVEX', 'nonopt_x.npy'), xx_) if res.status == 0: x = res.x; z = f1 @ x print(f" min f(1) = {z:.6f} -> {'PRIME-FREE after rescaling, margin %.4f' % (1 - 1/abs(z)) if z < -1 else 'NOT prime-free, C- >= %.6f' % (1 + z)}") hv = x[Hc]; gv = x[Gc] if len(Gc) else np.zeros(1) print(f" H in [{hv.min():.3f}, {hv.max():.3f}] (at cap {int((np.abs(hv) > GCAP - 1e-6).sum())}); G in [{gv.min():.3f}, {gv.max():.3f}] (at cap {int((np.abs(gv) > GCAP - 1e-6).sum())}); |T x| {np.abs(T @ x).max():.1e}") ~~~~~~~~~~~~ #### file: rec6_table.py - kind: script - as run: SHA-256 ee5623cac3a02e50dc0620bc67c933a6dd9e730e88bc439341717e9c03475706 - as shipped: SHA-256 82ad4d54663c8383b60951f613015853968df5134f23439706426020043b78d9 - identity: identical, after dropping docstrings and comments, to the file as run with 1 declared change(s) of a machine-specific path, which now read: `R = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'outputs', '')` - trailing newline: yes ~~~~~~~~~~~~ python #!/usr/bin/env python3 """rec6_table.py: reads the raw logs of the runs of record (R6*.raw, in outputs/ beside this script after unpacking), keeps the accepted solve of each point, requires the gate's PASS line, and prints one table line per point (value, raw dual bound, raw residuals, raw minimum of the positivity rows, FREEMIN_CLOSED, the raw file's SHA-256 prefix); it does not itself test the residuals or FREEMIN_CLOSED. Writes rec6_table.json beside itself. Exits non-zero if a requested point has no accepted solve.""" import re, sys, glob, hashlib, json, os R = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'outputs', '') PAT = re.compile(r'REC: status (\w[\w ]*); objective pruned ([-0-9.]+) raw ([-0-9.]+); float dual lower bound pruned ([-0-9.]+) raw ([-0-9.]+); ' r'pruned residuals live ([0-9.e+-]+) dead ([0-9.e+-]+) \(eps ([0-9.e+-]+), scale [0-9.]+\) min positivity ([-0-9.]+) box viol ([0-9.e+-]+); ' r'raw residuals live ([0-9.e+-]+) dead ([0-9.e+-]+) min positivity ([-0-9.]+); ACCEPT (\w+)') FMC = re.compile(r'FREEMIN_CLOSED: min f over (\d+) closed-interval free nodes = ([-0-9.]+); over bare nodes = ([-0-9.na]+)') SHA = re.compile(r'cont_hms_rec4') def point(stem): """stem without _p0/_p1; returns the accepted solve's record""" out = None for p in (0, 1): f = R + f'{stem}_p{p}.raw' if not os.path.exists(f): continue t = open(f).read(); m = PAT.search(t) if not m or m.group(14) != 'True': continue fm = FMC.search(t) assert 'GATE (trapezoid' in t and ': PASS' in t.split('GATE (trapezoid', 1)[1][:200], f + ': no gate PASS line' out = dict(file=os.path.basename(f), sha=hashlib.sha256(t.encode()).hexdigest()[:8], status=m.group(1), value=float(m.group(2)), value_raw=float(m.group(3)), bound_pruned=float(m.group(4)), bound_raw=float(m.group(5)), live_raw=float(m.group(11)), dead_raw=float(m.group(12)), pos_raw=float(m.group(13)), freemin_closed=float(fm.group(2)) if fm else None, nodes=int(fm.group(1)) if fm else None) break return out STEMS = { 'M72_24_u4': 'R6M_n72_e36_a24_u4_lnone_g50_eps1e-9', 'M72_25_u4': 'R6M_n72_e36_a25_u4_lnone_g50_eps1e-9', 'M72_25_u50': 'R6M_n72_e36_a25_u50_lnone_g50_eps1e-9', 'M72_25_u1000': 'R6M_n72_e36_a25_u1000_lnone_g1000_eps1e-9', 'M72_24_u1000': 'R6M_n72_e36_a24_u1000_lnone_g1000_eps1e-9', 'M72_26_u2000': 'R6M_n72_e36_a26_u2000_lnone_g2000_eps1e-9', 'M72_27_u2000': 'R6M_n72_e36_a27_u2000_lnone_g2000_eps1e-9', 'M90_31': 'R6M_n90_e45_a31_u4_lnone_g50_eps1e-9', 'M90_32': 'R6M_n90_e45_a32_u4_lnone_g50_eps1e-9', 'M96_34': 'R6M_n96_e48_a34_u4_lnone_g50_eps1e-9', 'M96_35': 'R6M_n96_e48_a35_u4_lnone_g50_eps1e-9', 'M108_36': 'R6M_n108_e36_a36_u4_lnone_g50_eps1e-9', 'M108_37': 'R6M_n108_e36_a37_u4_lnone_g50_eps1e-9', 'M90e30_30': 'R6M_n90_e30_a30_u4_lnone_g50_eps1e-9', 'M108e27_36': 'R6M_n108_e27_a36_u4_lnone_g50_eps1e-9', 'F23_u4': 'R6F_n72_e36_a23_u4_lnone_g10000_eps1e-9', 'F24_u4': 'R6F_n72_e36_a24_u4_lnone_g10000_eps1e-9', 'F25_u4': 'R6F_n72_e36_a25_u4_lnone_g10000_eps1e-9', 'F26_u4': 'R6F_n72_e36_a26_u4_lnone_g10000_eps1e-9', 'F23_u1000': 'R6F_n72_e36_a23_u1000_lnone_g10000_eps1e-9', 'F24_u1000': 'R6F_n72_e36_a24_u1000_lnone_g10000_eps1e-9', 'F25_u1000': 'R6F_n72_e36_a25_u1000_lnone_g10000_eps1e-9', 'F26_u1000': 'R6F_n72_e36_a26_u1000_lnone_g10000_eps1e-9', 'Fdrop25': 'R6Fdrop_n72_e36_a25_u4_lnone_g10000_eps1e9', 'Fdrop26': 'R6Fdrop_n72_e36_a26_u4_lnone_g10000_eps1e9', 'Fbin25_u4': 'R6Fbin1_n72_e36_a25_u4_lnone_g10000_eps1e-9', 'Fbin26_u4': 'R6Fbin1_n72_e36_a26_u4_lnone_g10000_eps1e-9', 'Fbin25_u1000': 'R6Fbin1_n72_e36_a25_u1000_lnone_g10000_eps1e-9', 'Fbin26_u1000': 'R6Fbin1_n72_e36_a26_u1000_lnone_g10000_eps1e-9', 'G25_u4': 'R6G_n72_e36_a25_u4_lnone_g50_eps1e-9', 'G25_u1000': 'R6G_n72_e36_a25_u1000_lnone_g1000_eps1e-9', } if __name__ == '__main__': T = {}; missing = [] for k, st in STEMS.items(): p = point(st) if p is None: missing.append(k); print(f'{k:14s} NOT ACCEPTED / NOT RUN ({st})'); continue T[k] = p print(f"{k:14s} {p['value']:+.9f} bound raw {p['bound_raw']:+.9f} | raw live {p['live_raw']:.1e} dead {p['dead_raw']:.1e} pos {p['pos_raw']:.9f} | " f"FMC {p['freemin_closed']} | {p['file']} {p['sha']}") json.dump(T, open(os.path.dirname(os.path.abspath(__file__)) + '/rec6_table.json', 'w'), indent=1) if missing: print('MISSING:', missing); sys.exit(1) ~~~~~~~~~~~~ #### file: cont_hms_rec6x.py - kind: script - as run: SHA-256 7c7c581a29cca882f2a4bae12cebd8c780514890bdf7baef24286bda85116cba - as shipped: SHA-256 a3ac65f614926e6119f4f5a6969a7d38d5cfa3d752bd66f960cec3ce1421dda9 - identity: identical, after dropping docstrings and comments, to the file as run - trailing newline: yes ~~~~~~~~~~~~ python #!/usr/bin/env python3 # cont_hms_rec6x.py: a diagnostic copy of cont_hms_rec6.py with the switches CONT_KEEP_E and CONT_FIX_VAC; see its docstring. """cont_hms_rec6x.py: a diagnostic copy of cont_hms_rec6.py with two switches that restore its two changed defaults singly: CONT_KEEP_E=1 keeps a positivity row at E = (1/2, 1/2), read as a rough 1/2 plus a large 1/2, and CONT_FIX_VAC=1 fixes the H columns on the interval of Lemma 8.1 at 0. Otherwise identical to cont_hms_rec6.py (see its docstring). Used only for the two diagnostic solves keepE and fixvac in gsweep.out; not a source of the paper's values. Floats; not a certificate.""" import sys, time, os, pickle from fractions import Fraction as Fr import numpy as np from scipy.signal import fftconvolve from scipy.optimize import linprog from scipy.sparse import csr_matrix t0 = time.time() N = int(sys.argv[1]); eta = Fr(int(sys.argv[2]), int(sys.argv[3])); alpha = Fr(int(sys.argv[4]), int(sys.argv[5])) EDG = [Fr(e) for e in sys.argv[6].split(',')] if sys.argv[6] else [] PHIMAX = float(sys.argv[7]); ASUB = int(sys.argv[8]); SSUB = int(sys.argv[9]) assert eta <= alpha < Fr(1, 2) h = 1.0 / N def idx(x): v = x * N; assert v.denominator == 1, f"{x} not on grid N={N}"; return int(v) ie, ia, ihf = idx(eta), idx(alpha), idx(Fr(1, 2)) edges = [alpha] + EDG + [Fr(1, 2)]; iedges = [idx(e) for e in edges]; NB = len(edges) - 1 assert all(edges[i_] < edges[i_ + 1] for i_ in range(len(edges) - 1)), 'bin edges must be strictly increasing between alpha and 1/2' inv = np.zeros(N + 1); inv[1:] = 1.0 / (np.arange(1, N + 1) * h) def conv(F, G): return fftconvolve(F, G)[:N + 1] * h delta = np.zeros(N + 1); delta[0] = 1.0 / h HC = os.environ.get('CONT_HALFCLOSE', 'trap'); HCF = {'trap': 0.5, 'closed': 1.0}[HC] # rec5: weight of a boundary at exactly 1/2 SELFTEST = bool(os.environ.get('CONT_GATE_SELFTEST')) # rec5: positive control of the gate (emulates the old a = 1/2 defect) def band(lo, hi): F = np.zeros(N + 1); F[lo:hi + 1] = inv[lo:hi + 1]; F[lo] *= 0.5; F[hi] *= 0.5; return F def trunc_ge(F, i0): G = F.copy(); G[:max(i0, 0)] = 0.0 if 0 <= i0 <= N: G[i0] *= HCF return G def shift(F, i): G = np.zeros(N + 1); G[i:] = F[:N + 1 - i]; return G s_den = band(ie, ia) JMAX = int(1 / eta) + 1; KMAX = int(1 / alpha) + 1 S = [delta] for j in range(1, JMAX + 1): S.append(conv(S[-1], s_den) / j) S2 = S[2] Rcap = {} for e in iedges[1:]: r = band(ia, e) if HC == 'closed' and e == ihf: r[ihf] = 0.0 # rec5 closed: a component exactly 1/2 is large P = [delta] for k in range(1, KMAX + 1): P.append(conv(P[-1], r) / k) Rcap[e] = P def Rbin(k, b): if k == 0: return delta hi = iedges[b] return Rcap[hi][k] if b == 1 else Rcap[hi][k] - Rcap[iedges[b - 1]][k] Rall = Rcap[ihf] bins_k = lambda k: [0] if k == 0 else list(range(1, NB + 1)) etaf, alf, ef = float(eta), float(alpha), [float(e) for e in edges] def support(j, k, b): if k == 0: return (j * etaf, j * alf) return (j * etaf + (k - 1) * alf + ef[b - 1], j * alf + k * ef[b]) def in_sup(sup, t): return sup[1] > sup[0] and sup[0] - 1e-12 <= t <= sup[1] + 1e-12 print(f"[{time.time()-t0:.1f}s] N={N} eta={eta} alpha={alpha} edges={[str(e) for e in edges]}", flush=True) # ---- Lemma K kernels (c-part) ---- PS = {} for e in iedges[1:]: for bs in range(JMAX + 1): for br in range(KMAX + 1): PS[(e, bs, br)] = conv(S[bs], Rcap[e][br]) if br else S[bs] def Lam_cap(jy, ky, e): K = jy + ky; L = -PS[(e, jy, ky)].copy(); acc = np.zeros(N + 1) for bs in range(jy + 1): for br in range(ky + 1): if (bs, br) in ((0, 0), (jy, ky)): continue acc += (-1) ** (bs + br) * conv(trunc_ge(PS[(e, bs, br)], ihf), PS[(e, jy - bs, ky - br)]) return L + (-1) ** (K + 1) * acc Lam = {} for jy in range(JMAX + 1): for ky in range(KMAX + 1): if jy + ky < 2: continue if ky == 0: Lam[(jy, 0, 0)] = Lam_cap(jy, 0, ihf); continue prev = None for b in range(1, NB + 1): cur = Lam_cap(jy, ky, iedges[b]); Lam[(jy, ky, b)] = cur if prev is None else cur - prev; prev = cur Lam0 = sum(Lam_cap(0, ky, ihf) for ky in range(2, KMAX + 1)) del PS # ---- XS(t; theta) for theta = 1/2 - x, x in [0, ia] ---- XS = {} for x in range(0, ia + 1): acc = np.zeros(N + 1) for k in range(1, KMAX + 1): for br in range(1, k + 1): acc += (-1) ** (k + br) * conv(trunc_ge(Rall[br], ihf - x), Rall[k - br]) XS[x] = acc # ======================= cont_hms ======================= MS = ia // ie while MS * ie > ia: MS -= 1 Hgrid = {j: list(range(j * ie, ia + 1)) for j in range(1, MS + 1)} Rall = Rcap[ihf] Rsgn = sum(((-1) ** k) * Rall[k] for k in range(0, KMAX + 1)); Runs = sum(Rall[k] for k in range(0, KMAX + 1)) na = ia + 1; ng = N - ia + 1 JG = min(JMAX, (N - ia) // ie) R1 = Rall[1] XSmat = np.array([XS[x] for x in range(0, ia + 1)]) XS1 = np.zeros((ng, N + 1)) for sg_ in range(0, min(ng, ihf)): XS1[sg_] = trunc_ge(R1, ihf - sg_) if ihf < ng: XS1[ihf] = (1.0 - HCF) * R1 # small sub-sum exactly 1/2: S and S + r cancel in the weight HCF tr_mask = (np.arange(N + 1) >= ihf).astype(float); tr_mask[ihf] = HCF Lam0Tr = trunc_ge(Lam0, ihf) K0sgn = conv(Rsgn, Lam0Tr); K0uns = conv(Runs, Lam0Tr) mgrid = list(range(ie, ia + 1)) JS = max(JMAX, JG) def restricted(m): """kernels with every EXTRA small part <= m (grid index)""" sm = s_den.copy(); sm[m + 1:] = 0.0 Sm = [delta] for j in range(1, JS + 1): Sm.append(conv(Sm[-1], sm) / j) w = h * sm # M_j over sigma, tau in [0, ng) Mj = [None] * (JG + 1); Mj[0] = np.zeros((ng, ng)); Mj[0][0, 0] = 1.0 / h for j in range(1, JG + 1): acc = np.zeros((ng, ng)) for t in range(ie, m + 1): if w[t] == 0.0: continue acc[:, t:] += w[t] * Mj[j - 1][:, :ng - t] acc[t:, t:] -= w[t] * Mj[j - 1][:ng - t, :ng - t] Mj[j] = acc / j LJ = {}; Ks = {}; Ku = {} for j in range(1, MS + 1): A = (Mj[j][:na, :na].T @ XSmat) * ((-1) ** (j - 1)) L = np.zeros((na, N + 1)) for tau in Hgrid[j]: L[tau, tau:] = A[tau, :N + 1 - tau] if HC == 'trap': L[:, ihf] *= 2.0 # rec4 (1): right-limit value at a = 1/2, as Lam_cap if SELFTEST: L[:, ihf] *= 0.5 LJ[j] = L LT = L * tr_mask[None, :] Ks[j] = np.array([conv(Rsgn, LT[t]) for t in range(na)]); Ku[j] = np.array([conv(Runs, LT[t]) for t in range(na)]) LG = {}; LGT = {} for j in range(1, JG + 1): A = (Mj[j].T @ XS1) * ((-1) ** (j - 1)) L = np.zeros((ng, N + 1)) for tau in range(j * ie, ng): L[tau, tau:] = A[tau, :N + 1 - tau] if HC == 'trap': L[:, ihf] *= 2.0 # rec4 (1), G kernels if SELFTEST: L[:, ihf] *= 0.5 LG[j] = L; LGT[j] = L * tr_mask[None, :] LAS = {} for jy in range(2, JG + 1): acc = np.zeros(N + 1) for bs in range(1, jy): acc += (-1) ** bs * conv(trunc_ge(Sm[bs], ihf), Sm[jy - bs]) LAS[jy] = -Sm[jy] + ((-1) ** (jy + 1)) * acc # all-small Lemma K kernel, parts <= m (Lam_cap(jy, 0, .) with S restricted) LAST = {jy: trunc_ge(L, ihf) for jy, L in LAS.items()} return dict(S=Sm, LJ=LJ, Ks=Ks, Ku=Ku, LG=LG, LGT=LGT, LAS=LAS, LAST=LAST) RK = {} for m in mgrid: RK[m] = restricted(m) EMPTY = None print(f"[{time.time()-t0:.1f}s] restricted kernels for {len(mgrid)} maxima (MS {MS}, JG {JG})", flush=True) if '--kcheck' in sys.argv: top = RK[ia] print(" kcheck S_j(<= ia) vs S_j:", max(np.abs(top['S'][j] - S[j]).max() for j in range(1, min(JS, JMAX) + 1))) print(" kcheck LAS(<= ia) vs Lam[(jy,0,0)]:", max(np.abs(top['LAS'][jy] - Lam[(jy, 0, 0)]).max() for jy in top['LAS'] if (jy, 0, 0) in Lam)) def Sd(m, j, x): # density of j extra parts with total x and maximum EXACTLY m (vectorised in x) a = RK[m]['S'][j][x] return a - RK[m - 1]['S'][j][x] if m - 1 >= ie else a # ---- variables ---- classes = [(j, k, b) for j in range(JMAX + 1) for k in range(KMAX + 1) if j + k >= 1 for b in bins_k(k)] D = {cl: (conv(S[cl[0]], Rbin(cl[1], cl[2])) if cl[1] else S[cl[0]]) for cl in classes}; D[(0, 0, 0)] = delta cid = {cl: n for n, cl in enumerate(classes)}; nc = len(classes) def feas_max(j, tau): # maxima m with j parts of total tau and maximum exactly m: exact combinatorial test (17:5x; testing if j == 1: return [tau] if ie <= tau <= ia else [] # Sd > 0 admitted FFT round-off, giving impossible (tau, m) rows and variables) return [m for m in mgrid if m <= tau - (j - 1) * ie and j * m >= tau] hid = {}; gid = {}; nv = nc for j in range(1, MS + 1): for tau in Hgrid[j]: for m in feas_max(j, tau): hid[(j, tau, m)] = nv; nv += 1 GBAND = bool(os.environ.get('CONT_GBAND')) # restricted family: G resolved by m only on the dead band tau - m in [1-2a, a] glump = {} for j in range(2, JG + 1): for tg in range(ihf + 1, N - ia + 1): if S[j][tg] <= 0.0: continue for m in feas_max(j, tg): if GBAND and not (N - 2 * ia <= tg - m <= ia): if (j, tg) not in glump: glump[(j, tg)] = nv; nv += 1 gid[(j, tg, m)] = glump[(j, tg)] else: gid[(j, tg, m)] = nv; nv += 1 print(f" variables: {nc} constants, {len(hid)} H(j, tau, m), {len(gid)} G(j, tau, m) keys in {len(set(gid.values()))} columns" f"{' (GBAND: %d lumped)' % len(glump) if GBAND else ''}", flush=True) def new(): return np.zeros(nv) LamTr = {k_: trunc_ge(L, ihf) for k_, L in Lam.items()} w_classes = [(0, 0, 0)] + [cl for cl in classes if cl[0] + cl[1] >= 1 and in_sup(support(*cl), support(*cl)[0]) and support(*cl)[0] <= 0.5 + 1e-12] _cc = {} def conv_w_lam(wc, key): k_ = (wc, key) if k_ not in _cc: _cc[k_] = LamTr[key] if wc == (0, 0, 0) else conv(D[wc], LamTr[key]) return _cc[k_] # ---- f(1) ---- f1 = new() for key, L in Lam.items(): if (cl := cid.get(key)) is not None: f1[cl] += L[N] def dmax(m, key, j, idx): # RK[m][key][j][idx] - RK[m-1][key][j][idx] a = RK[m][key][j][idx] return a - RK[m - 1][key][j][idx] if m - 1 >= ie else a for (j, tau, m), col in hid.items(): f1[col] += h * dmax(m, 'LJ', j, (tau, N)) for (j, tg, m), col in gid.items(): f1[col] += h * dmax(m, 'LG', j, (tg, N)) # ---- rows (jD, sigma, mD) ---- row_list = [(0, 0, 0)] for jD in range(1, MS + 1): for s_ in Hgrid[jD]: for mD in feas_max(jD, s_): row_list.append((jD, s_, mD)) rid = {r: n for n, r in enumerate(row_list)}; nr_ = len(row_list) T = np.zeros((nr_, nv)); I = np.zeros((nr_, nv)); DIAG = np.zeros((nr_, nv), dtype=bool) # c-part (independent of mD): computed per (jD, sigma) as in cont_hm, copied to every mD cpart = {} for jD in range(0, MS + 1): sig = [0] if jD == 0 else Hgrid[jD]; tt = np.array([N - s_ for s_ in sig]) Tc = np.zeros((len(sig), nc)); Ic = np.zeros((len(sig), nc)) for cl in classes: jp, kp, bp = cl; tgt = cid.get((jD + jp, kp, bp)) if tgt is None: continue vals = D[cl][tt]; sg = (-1) ** (jD + jp + kp); Tc[:, tgt] += sg * vals; Ic[:, tgt] += vals for wc in w_classes: jw, kw, bw = wc; sgn = (-1) ** (1 + jD + jw + kw) for key in Lam: jy, ky, byy = key; tgt = cid.get((jD + jw + jy, kw + ky, max(bw, byy))) if tgt is None: continue v = conv_w_lam(wc, key)[tt]; Tc[:, tgt] += sgn * v; Ic[:, tgt] += v for n_, s_ in enumerate(sig): cpart[(jD, s_)] = (Tc[n_], Ic[n_]) for r, (jD, s_, mD) in enumerate(row_list): T[r, :nc] = cpart[(jD, s_)][0]; I[r, :nc] = cpart[(jD, s_)][1] print(f"[{time.time()-t0:.1f}s] c-part of {nr_} rows", flush=True) # extra-parts kernels per maximum m: KH[m][D][sigma, tau] (signed, jD-independent part) and KG[m][D][sigma, tg]; unsigned KHu, KGu def build_K(m): R = RK[m]; Sm = R['S'] KH = {}; KHu = {}; KG = {}; KGu = {} sig_all = np.arange(na) for Dl in range(1, MS + 1): Ht = np.zeros((na, na)); Hu = np.zeros((na, na)) for s_ in range(na): taus = np.arange(s_, na); x = taus - s_ v = h * Sm[Dl][x] * Rsgn[N - taus]; Ht[s_, taus] += ((-1) ** Dl) * v; Hu[s_, taus] += h * Sm[Dl][x] * Runs[N - taus] for jw in range(0, Dl + 1): jy = Dl - jw; sg = (-1) ** (1 + jw) if jw == 0: vt = h * R['Ks'][Dl][x, N - s_]; vu = h * R['Ku'][Dl][x, N - s_] elif jy == 0: vt = h * Sm[jw][x] * K0sgn[N - taus]; vu = h * Sm[jw][x] * K0uns[N - taus] else: vt = np.zeros(len(taus)); vu = np.zeros(len(taus)) for i_, t_ in enumerate(taus): vv = np.arange(s_, t_ + 1) vt[i_] = h * h * np.dot(Sm[jw][vv - s_], R['Ks'][jy][t_ - vv, N - vv]) vu[i_] = h * h * np.dot(Sm[jw][vv - s_], R['Ku'][jy][t_ - vv, N - vv]) Ht[s_, taus] += sg * vt; Hu[s_, taus] += vu KH[Dl] = Ht; KHu[Dl] = Hu gt = np.arange(ihf + 1, ng) for Dl in range(1, JG + 1): Gt = np.zeros((na, ng)); Gu = np.zeros((na, ng)) for s_ in range(na): x = gt - s_ v = h * Sm[Dl][x] * R1[N - gt]; Gt[s_, gt] += ((-1) ** (Dl + 1)) * v; Gu[s_, gt] += v for jw in range(0, Dl + 1): jy = Dl - jw if jy >= 1: if jw == 0: vt = h * R['LGT'][jy][x, N - s_] else: vt = np.zeros(len(gt)) for i_, t_ in enumerate(gt): vv = np.arange(s_, t_ + 1) vt[i_] = h * h * np.dot(Sm[jw][vv - s_], R['LGT'][jy][t_ - vv, N - vv]) Gt[s_, gt] += ((-1) ** (1 + jw)) * vt; Gu[s_, gt] += vt if jy >= 2: base = R['LAST'][jy] if jw == 0 else conv(Sm[jw], R['LAST'][jy]) vt = h * R1[N - gt] * base[x] Gt[s_, gt] += ((-1) ** jw) * vt; Gu[s_, gt] += vt KG[Dl] = Gt; KGu[Dl] = Gu return KH, KHu, KG, KGu KK = {m: build_K(m) for m in mgrid} print(f"[{time.time()-t0:.1f}s] extra-part kernels built", flush=True) def Kexact(m, which, Dl): # kernel with the extra parts' maximum EXACTLY m: K(<= m) - K(<= m-1) a = KK[m][which][Dl] return a - KK[m - 1][which][Dl] if m - 1 >= ie else a # assemble the H- and G-parts of every row for r, (jD, s_, mD) in enumerate(row_list): sgD = (-1) ** jD if jD >= 1: # diagonal: H_jD(sigma, mD), all-rough completions (free and extended) col = hid[(jD, s_, mD)] T[r, col] += sgD * (Rsgn[N - s_] - K0sgn[N - s_]); I[r, col] += Runs[N - s_] + K0uns[N - s_]; DIAG[r, col] = True for (J, tau, M), col in hid.items(): if J <= jD or tau < s_: continue Dl = J - jD if M < mD: continue if jD == 0: kt = Kexact(M, 0, Dl)[0, tau]; ku = Kexact(M, 1, Dl)[0, tau] elif M == mD: kt = KK[mD][0][Dl][s_, tau]; ku = KK[mD][1][Dl][s_, tau] else: kt = Kexact(M, 0, Dl)[s_, tau]; ku = Kexact(M, 1, Dl)[s_, tau] T[r, col] += sgD * kt; I[r, col] += ku for (J, tg, M), col in gid.items(): if J <= jD: continue Dl = J - jD if Dl > JG or M < mD: continue if jD == 0: kt = Kexact(M, 2, Dl)[0, tg]; ku = Kexact(M, 3, Dl)[0, tg] elif M == mD: kt = KK[mD][2][Dl][s_, tg]; ku = KK[mD][3][Dl][s_, tg] else: kt = Kexact(M, 2, Dl)[s_, tg]; ku = Kexact(M, 3, Dl)[s_, tg] T[r, col] += sgD * kt; I[r, col] += ku print(f"[{time.time()-t0:.1f}s] rows assembled", flush=True) Hc = np.array(sorted(set(hid.values()))); Gc = np.array(sorted(set(gid.values())), dtype=int) offH = np.where(DIAG[:, Hc], 0.0, I[:, Hc]) print(f" Type I check: c-block {np.abs(I[:, :nc]).max():.3e}; H diagonal {np.abs(I[DIAG]).max():.3e}; H integrated/h {np.abs(offH).max()/h:.3e};" f" G/h {np.abs(I[:, Gc]).max()/h if len(Gc) else 0:.3e}; T0 row: c {np.abs(I[0, :nc]).max():.3e}, H/h {np.abs(I[0, Hc]).max()/h:.3e}," f" G/h {np.abs(I[0, Gc]).max()/h if len(Gc) else 0:.3e}", flush=True) _ti = dict(c=float(np.abs(I[:, :nc]).max()), Hd=float(np.abs(I[DIAG]).max()), Hi_h=float(np.abs(offH).max() / h), G_h=float(np.abs(I[:, Gc]).max() / h) if len(Gc) else 0.0) if HC == 'closed': _fail = [k_ for k_ in ('c', 'Hd', 'Hi_h', 'G_h') if not _ti[k_] <= 1e-9] print(f" GATE (closed-1/2 exactness): {'PASS' if not _fail else 'FAIL ' + str(_fail)} {_ti}", flush=True) if os.environ.get('CONT_GATEONLY'): sys.exit(3 if _fail else 0) else: _fail = [k_ for k_, ok_ in (('G_h', _ti['G_h'] <= 1e-9), ('c', _ti['c'] <= 10 * h), ('Hd', _ti['Hd'] <= 2 * h)) if not ok_] import subprocess as _sp _env = dict(os.environ, CONT_HALFCLOSE='closed', CONT_GATEONLY='1') _r = _sp.run([sys.executable, '-B', os.path.abspath(__file__)] + sys.argv[1:], env=_env, capture_output=True, text=True) _gl = [l_ for l_ in _r.stdout.splitlines() if 'GATE' in l_] if _r.returncode != 0: _fail.append('closed-1/2 exactness (exit %d): %s' % (_r.returncode, _gl[-1].strip() if _gl else _r.stderr[-300:])) print(f" GATE (trapezoid: G/h <= 1e-9, c <= 10h, H diag <= 2h; closed-1/2 exactness <= 1e-9): {'PASS' if not _fail else 'FAIL ' + str(_fail)}; " f"{_ti}; closed: {_gl[-1].strip() if _gl else 'no output'}", flush=True) if _fail and not os.environ.get('CONT_NOGATE'): sys.exit(f'cont_hms_rec5: GATE FAILED {_fail}') if os.environ.get('CONT_GATEONLY'): sys.exit(0) if '--check' in sys.argv: sys.exit() # ---- optional diagnostics ---- TDEAD = None if os.environ.get('DEAD_SVD'): # diagnostic: spectrum of the dead rows' G-block (and H-block size), then exit dead_ = [r for r, (jD_, s_, m_) in enumerate(row_list) if jD_ >= 1 and s_ >= N - 2 * ia + 2] gc_ = np.array(sorted(set(gid.values())), dtype=int) hc_ = np.array(sorted(set(hid.values())), dtype=int) Bd = T[np.ix_(dead_, gc_)] U_, sv, _ = np.linalg.svd(Bd, full_matrices=False) rk = int((sv > 1e-9 * sv[0]).sum()) NL = U_[:, rk:] # left null space of the dead G-block (dead-row combinations G cannot touch) Cd = T[np.ix_(dead_, np.arange(nc))] if NL.shape[1]: Mc = NL.T @ Cd; svc = np.linalg.svd(Mc, compute_uv=False) touched = int((np.abs(Cd).max(axis=0) > 1e-12).sum()) print(f" DEAD_NULL: G-rank {rk} of {len(dead_)} dead rows -> {NL.shape[1]} dead-row combinations G cannot touch; on the class constants they have rank " f"{(svc > 1e-9 * svc[0]).sum() if svc[0] > 0 else 0} (largest sv {svc[0]:.2e}); constants touched by dead rows {touched} of {nc}", flush=True) else: print(f" DEAD_NULL: G-block has full row rank {rk} = {len(dead_)}: no G-free dead-row combination", flush=True) ks = [k for k in (1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024) if k < len(sv)] print(f" DEAD_SVD s_max: dead rows {len(dead_)} x G cols {len(gc_)}; sv0 {sv[0]:.3e}; sv/sv0 at k={ks}: " + ", ".join(f"{sv[k]/sv[0]:.1e}" for k in ks) + f"; #(>1e-3 sv0) {(sv > 1e-3*sv[0]).sum()}, #(>1e-6) {(sv > 1e-6*sv[0]).sum()}, #(>1e-9) {(sv > 1e-9*sv[0]).sum()}; H-block max {np.abs(T[np.ix_(dead_, hc_)]).max():.2e}", flush=True) sys.exit(0) if os.environ.get('CONT_DEADEPS'): # relax the dead rows to |T_dead x| <= eps (they make the equality LP numerically degenerate) dead = [r for r, (jD_, s_, m_) in enumerate(row_list) if jD_ >= 1 and s_ >= N - 2 * ia] live = [r for r in range(nr_) if r not in set(dead)] TDEAD = T[dead].copy(); T = T[live]; nr_ = len(live) print(f" CONT_DEADEPS = {os.environ['CONT_DEADEPS']}: {len(dead)} dead rows relaxed to |T x| <= eps", flush=True) if os.environ.get('CONT_DROPDEAD'): keep = [r for r, (jD_, s_, m_) in enumerate(row_list) if jD_ == 0 or s_ < N - 2 * ia] print(f" CONT_DROPDEAD: keeping {len(keep)} of {nr_} rows", flush=True) T = T[keep]; nr_ = len(keep) # ---- positivity (sparse) ---- from scipy.sparse import coo_matrix, csr_matrix as _csr GCAP = float(os.environ.get('CONT_GCAP', '50')); MSUB = int(os.environ.get('CONT_MSUB', '1')) lb = np.full(nv, -np.inf); ub = np.full(nv, np.inf); lb[:nc] = -1.0; ub[:nc] = PHIMAX nvac = 0 for (j, tau, m), col in hid.items(): if os.environ.get('CONT_FIX_VAC') and tau >= N - 2 * ia: lb[col] = 0.0; ub[col] = 0.0; nvac += 1 # DIAGNOSTIC: rec4's default back else: lb[col] = -GCAP; ub[col] = GCAP # rec6 (2) for col in gid.values(): lb[col] = -GCAP; ub[col] = GCAP if os.environ.get('CONT_NOGREC'): for col in gid.values(): lb[col] = 0.0; ub[col] = 0.0 print(f" CONT_NOGREC: {len(set(gid.values()))} G columns fixed to 0", flush=True) if os.environ.get('CONT_LBC'): sys.exit('cont_hms_rec4: CONT_LBC is not supported (focused read M1)') LBC = os.environ.get('CONT_LBC'); low_cls = [] if LBC is not None: LBC = float(LBC) for cl_ in classes: if cl_[0] >= 1 and (cl_[1] >= 2 or (cl_[1] == 1 and cl_[0] >= 2 and not os.environ.get('CONT_NOGREC'))): lb[cid[cl_]] = LBC; low_cls.append(cl_) hid_arr = -np.ones((MS + 1, na, na), dtype=int) for (j, tau, m), col in hid.items(): hid_arr[j, tau, m] = col gid_arr = -np.ones((JG + 1, N + 1, na), dtype=int) for (j, tg, m), col in gid.items(): gid_arr[j, tg, m] = col # cumulative kernels as arrays over the maximum: LJc[jy][m, tau_y, a], LGc[jy][m, tau_y, a], LASc[jy][m, a] mi = np.array(mgrid); NM = len(mgrid) LJc = {jy: np.array([RK[m]['LJ'][jy] for m in mgrid]) for jy in range(1, MS + 1)} LGc = {jy: np.array([RK[m]['LG'][jy] for m in mgrid]) for jy in range(1, JG + 1)} LASc = {jy: np.array([RK[m]['LAS'][jy] for m in mgrid]) for jy in range(2, JG + 1)} def exact_from(cum, mb): """rows over M = mb..ia of the 'maximum exactly M, or <= mb at M = mb' kernel: cum has a leading axis over mgrid (18:0x: callers now pass cum already sliced at the a-index, so this copies small arrays only)""" k0 = mb - ie out = cum[k0:].copy() out[1:] -= cum[k0:-1] return out # out[0] = cum(<= mb); out[i] = cum(<= mb+i) - cum(<= mb+i-1) LJa = {jy: np.ascontiguousarray(np.moveaxis(LJc[jy], 2, 0)) for jy in LJc} # [a][m, tau_y] LGa = {jy: np.ascontiguousarray(np.moveaxis(LGc[jy], 2, 0)) for jy in LGc} LASa = {jy: np.ascontiguousarray(LASc[jy].T) for jy in LASc} # [a][m] from array import array as _arr Rr, Rc, Rv = _arr('q'), _arr('q'), _arr('d'); nrow = 0; desc = [] def emit(entries): global nrow for c_, v_ in entries.items(): if v_ != 0.0: Rr.append(nrow); Rc.append(c_); Rv.append(v_) nrow += 1 def emit_arrays(base_dict, cols, vals): d = dict(base_dict) for c_, v_ in zip(cols, vals): d[c_] = d.get(c_, 0.0) + v_ emit(d) # free rows for cl in classes: j, k, b = cl if not in_sup(support(j, k, b), 1.0) or j == 0 or k == 0: continue supR = support(0, k, b) for (jj, tau, m), col in hid.items(): if jj == j and in_sup(supR, 1 - tau * h) and lb[col] != ub[col]: emit({cid[cl]: 1.0, col: 1.0}); desc.append(('free', cl)) if k == 1: seen = set() for (jj, tg, m), col in gid.items(): if jj == j and in_sup(supR, 1 - tg * h) and col not in seen: seen.add(col); emit({cid[cl]: 1.0, col: 1.0}); desc.append(('freeG', cl)) # every free grid shape; classes with a shape carrying no active added term keep c >= -1 _fs_cls, _fs_h, _fs_g = [], [], [] def _parts(total, lower, prefix): if total == 0: yield prefix; return for p_ in range(lower, min(ihf - 1, total) + 1): if total == p_ or total - p_ >= p_: yield from _parts(total - p_, p_, prefix + (p_,)) _pure = set() for _sh in _parts(N, ie, ()): _sm = [q for q in _sh if q <= ia]; _ro = [q for q in _sh if q > ia] _j, _k, _tau = len(_sm), len(_ro), sum(_sm) _b = 0 if not _ro else next(i_ for i_ in range(1, len(iedges)) if max(_ro) <= iedges[i_]) _cl = (_j, _k, _b) if _cl not in cid: continue _hc = hid.get((_j, _tau, max(_sm))) if (_sm and _tau <= ia) else None _gc = gid.get((_j, _tau, max(_sm))) if (_sm and _k == 1) else None _act = [cc_ for cc_ in (_hc, _gc) if cc_ is not None and not (lb[cc_] == 0 and ub[cc_] == 0)] if not _act: _pure.add(_cl) _fs_cls.append(cid[_cl]); _fs_h.append(_hc if _hc is not None else -1); _fs_g.append(_gc if _gc is not None else -1) _fs_cls = np.array(_fs_cls); _fs_h = np.array(_fs_h); _fs_g = np.array(_fs_g) _restored = [] for _cl in sorted(_pure): if lb[cid[_cl]] < -1: lb[cid[_cl]] = -1.0; _restored.append(_cl) _cn_cls, _cn_h, _cn_g, _cn_bare = [], [], [], [] # rec4 (3): closed-interval free nodes, as band() weights them for _cl in classes: _j, _k, _b = _cl if _k == 0: _taus = [N] if (_j >= 1 and _j * ie <= N <= _j * ia) else [] else: _sR = support(0, _k, _b); _taus = [t_ for t_ in range(_j * ie, min(_j * ia, N) + 1) if in_sup(_sR, 1 - t_ * h)] if _j >= 1 else ([0] if in_sup(_sR, 1.0) else []) for _t in _taus: for _m in (feas_max(_j, _t) if _j >= 1 else [None]): _hc = hid.get((_j, _t, _m)) if (_j >= 1 and _t <= ia) else None _gc = gid.get((_j, _t, _m)) if (_j >= 1 and _k == 1) else None _hc = _hc if (_hc is not None and not (lb[_hc] == 0 and ub[_hc] == 0)) else None _gc = _gc if (_gc is not None and not (lb[_gc] == 0 and ub[_gc] == 0)) else None _cn_cls.append(cid[_cl]); _cn_h.append(-1 if _hc is None else _hc); _cn_g.append(-1 if _gc is None else _gc) _cn_bare.append(_hc is None and _gc is None) _cn_cls = np.array(_cn_cls); _cn_h = np.array(_cn_h); _cn_g = np.array(_cn_g); _cn_bare = np.array(_cn_bare) print(f" REC4: {len(_cn_cls)} closed-interval free nodes (class, tau, m), {int(_cn_bare.sum())} bare (constant only); constants lower bound min {lb[:nc].min():g}", flush=True) print(f" REC3: {len(_fs_cls)} free grid shapes; classes with a pure-constant free shape: {len(_pure)}; lower bound restored to -1 on {len(_restored)}: {_restored}", flush=True) npure = 0 if LBC is not None: for cl_ in low_cls: j_, k_, b_ = cl_ if k_ != 1: continue missing = False for rho in range(max(iedges[b_ - 1], ia + 1), min(iedges[b_], ihf - 1) + 1): tg = N - rho if not (j_ * ie <= tg <= j_ * ia): continue for m_ in feas_max(j_, tg): if (j_, tg, m_) not in gid: missing = True; break if missing: break if missing: emit({cid[cl_]: 1.0}); desc.append(('pure_c', cl_)); npure += 1 print(f" CONT_LBC = {LBC}: lowered {len(low_cls)} class constants; restored c >= -1 rows: {npure}", flush=True) nfree = nrow alist = list(range(ihf, N, ASUB)) for bc in w_classes: jb, kb, bb = bc if bc == (0, 0, 0): continue supR = support(0, kb, bb) for ai in alist: a = ai * h; tb = N - ai if not in_sup(support(jb, kb, bb), tb * h): continue base = {} for key, L in Lam.items(): jy, ky, bp = key; tgt = cid.get((jb + jy, kb + ky, max(bb, bp))) if tgt is not None: base[tgt] = base.get(tgt, 0.0) + a * L[ai] if jb == 0: cols = []; vals = [] for jy in range(1, MS + 1): # fragments carry the whole small part: H(jy, tau_y, M), M = their maximum ex = exact_from(LJa[jy][ai], ie) # (NM, na): maximum exactly M for k0, M in enumerate(mgrid): cc = hid_arr[jy, :, M]; ok = cc >= 0 if ok.any(): cols += list(cc[ok]); vals += list(h * a * ex[k0][ok]) if kb == 1: for jy in range(2, JG + 1): ex = exact_from(LASa[jy][ai], ie) for k0, M in enumerate(mgrid): cg = gid_arr[jy, ai, M] if ai <= N else -1 if cg >= 0: cols.append(cg); vals.append(a * ex[k0]) if bc == (0, 1, NB) and ai == ihf and not os.environ.get('CONT_KEEP_E'): continue # rec6 (1); CONT_KEEP_E: rec4's default back emit_arrays(base, cols, vals); desc.append(('ext', bc)); continue if kb: cands = [t_ for t_ in range(jb * ie, min(jb * ia, tb) + 1, SSUB) if in_sup(supR, (tb - t_) * h)] else: cands = [tb] if jb * ie <= tb <= jb * ia else [] done_const = False for tbeta in cands: mbs = feas_max(jb, tbeta) if tbeta <= ia else [None] if tbeta > ia and kb >= 2: if not done_const: emit(dict(base)); desc.append(('ext', bc)); done_const = True continue if tbeta > ia: mbs = feas_max(jb, tbeta) for mb in mbs[::MSUB]: cols = []; vals = [] if tbeta <= ia: c0 = hid_arr[jb, tbeta, mb] if jb <= MS else -1 if c0 >= 0: cols.append(c0); vals.append(a * Lam0[ai]) for jy in range(1, MS - jb + 1): ex = exact_from(LJa[jy][ai], mb) # (ia - mb + 1, na) for k0 in range(ex.shape[0]): M = mb + k0; taus_y = np.arange(jy * ie, ia - tbeta + 1) if len(taus_y) == 0: continue cc = hid_arr[jb + jy, tbeta + taus_y, M]; ok = cc >= 0 if ok.any(): cols += list(cc[ok]); vals += list(h * a * ex[k0][taus_y[ok]]) if kb == 0: for jy in range(1, JG - jb + 1): ex = exact_from(LGa[jy][ai], mb) taus_y = np.arange(jy * ie, ng); tt_ = tbeta + taus_y; okt = tt_ <= N taus_y, tt_ = taus_y[okt], tt_[okt] for k0 in range(ex.shape[0]): M = mb + k0; cc = gid_arr[jb + jy, tt_, M]; ok = cc >= 0 if ok.any(): cols += list(cc[ok]); vals += list(h * a * ex[k0][taus_y[ok]]) elif kb == 1: for jy in range(2, JG - jb + 1): ex = exact_from(LASa[jy][ai], mb) tt_ = tbeta + ai if tt_ > N: continue for k0 in range(len(ex)): cg = gid_arr[jb + jy, tt_, mb + k0] if cg >= 0: cols.append(cg); vals.append(a * ex[k0]) emit_arrays(base, cols, vals); desc.append(('ext', bc)) Gm = coo_matrix((Rv, (Rr, Rc)), shape=(nrow, nv)).tocsr() print(f"[{time.time()-t0:.1f}s] positivity rows: free {nfree}, extended {nrow - nfree}; nnz {Gm.nnz}; vacuous H fixed {nvac}; |L| <= {GCAP}", flush=True) # ---- control: tie L(j, tau, m) across m (must reproduce cont_hmg) ---- Aeq = _csr(T) if os.environ.get('CONT_TIEM'): tr, tc, tv = [], [], []; nt = 0 for store in (hid, gid): groups = {} for (j, tau, m), col in store.items(): groups.setdefault((j, tau), []).append(col) for cols in groups.values(): cols = sorted(set(cols)) for c1, c2 in zip(cols[:-1], cols[1:]): tr += [nt, nt]; tc += [c1, c2]; tv += [1.0, -1.0]; nt += 1 Aeq = _csr(np.vstack([T, coo_matrix((tv, (tr, tc)), shape=(nt, nv)).toarray()])) if nt else Aeq print(f" CONT_TIEM: {nt} tie rows", flush=True) NOUP = bool(os.environ.get('CONT_NOUPPER')) from scipy.sparse import vstack as _svs Aub = -Gm if NOUP else _svs([-Gm, Gm]).tocsr(); bub = np.ones(nrow) if NOUP else np.concatenate([np.ones(nrow), np.full(nrow, PHIMAX)]) _RAW = dict(T=T.copy(), f1=f1.copy(), Gm=Gm.copy(), TDEAD=(TDEAD.copy() if TDEAD is not None else None)) TINY = float(os.environ.get('CONT_TINY', '1e-9')) # remove coefficients below CONT_TINY from T, the dead rows, the positivity rows and the objective; this perturbs the rows and the objective, and the residuals reported below refer to the modified matrices T[np.abs(T) < TINY] = 0.0; f1[np.abs(f1) < TINY] = 0.0 Gm.data[np.abs(Gm.data) < TINY] = 0.0; Gm.eliminate_zeros() Aeq = _csr(T) if not os.environ.get('CONT_TIEM') else Aeq if os.environ.get('CONT_TIEM'): Aeq = Aeq.tocsr(); Aeq.data[np.abs(Aeq.data) < TINY] = 0.0; Aeq.eliminate_zeros() Aub = -Gm if NOUP else _svs([-Gm, Gm]).tocsr() if TDEAD is not None: TDEAD[np.abs(TDEAD) < TINY] = 0.0; epsd = float(os.environ['CONT_DEADEPS']) DS = float(os.environ.get('CONT_DEADSCALE', '1')) Aub = _svs([Aub, _csr(DS * TDEAD), _csr(-DS * TDEAD)]).tocsr(); bub = np.concatenate([bub, np.full(2 * TDEAD.shape[0], DS * epsd)]) print(f" sanity: finite T {np.isfinite(T).all()}, f1 {np.isfinite(f1).all()}, G {np.isfinite(Gm.data).all()}; max|T| {np.abs(T).max():.3e}, max|G| {np.abs(Gm.data).max():.3e}, max|f1| {np.abs(f1).max():.3e}; min|nonzero T| {np.abs(T[T != 0]).min():.3e}", flush=True) if os.environ.get('CONT_FEASCHECK'): r0 = linprog(np.zeros(nv), A_ub=Aub, b_ub=bub, A_eq=Aeq, b_eq=np.zeros(Aeq.shape[0]), bounds=np.column_stack([lb, ub]), method='highs', options={'presolve': True}) print(f" feasibility LP (zero objective): status {r0.status} ({r0.message})", flush=True) if os.environ.get('CONT_NOX'): # bundled HiGHS, IPM, run_crossover off; primal value + residuals + a float dual bound from scipy.optimize._highspy import _core as _hc from scipy.sparse import vstack as _vs2 from types import SimpleNamespace as _NS hh_ = _hc._Highs() for k_, v_ in (('output_flag', bool(os.environ.get('CONT_DISP'))), ('solver', 'ipm'), ('run_crossover', 'off'), ('presolve', 'on' if int(os.environ.get('CONT_PRESOLVE', '0')) else 'off'), ('ipm_optimality_tolerance', float(os.environ.get('CONT_OTOL', '1e-8'))), ('primal_feasibility_tolerance', float(os.environ.get('CONT_PTOL', '1e-7'))), ('dual_feasibility_tolerance', float(os.environ.get('CONT_PTOL', '1e-7'))), ('time_limit', float(os.environ.get('CONT_TLIM', '1e9')))): hh_.setOptionValue(k_, v_) INF_ = hh_.getInfinity() A_ = _vs2([Aeq, Aub]).tocsc(); A_.sort_indices(); neq_ = Aeq.shape[0]; m_ = A_.shape[0] rl_ = np.concatenate([np.zeros(neq_), np.full(Aub.shape[0], -INF_)]); ru_ = np.concatenate([np.zeros(neq_), np.asarray(bub, dtype=np.float64)]) hh_.addRows(m_, rl_, ru_, 0, np.zeros(m_, dtype=np.int32), np.zeros(0, dtype=np.int32), np.zeros(0)) lbh_ = np.where(np.isinf(lb), -INF_, lb); ubh_ = np.where(np.isinf(ub), INF_, ub) hh_.addCols(nv, np.asarray(f1, dtype=np.float64), lbh_, ubh_, A_.nnz, A_.indptr[:-1].astype(np.int32), A_.indices.astype(np.int32), A_.data.astype(np.float64)) print(f"[{time.time()-t0:.1f}s] NOX: model {hh_.getNumRow()} rows x {hh_.getNumCol()} cols, nnz {hh_.getNumNz()}; running IPM (no crossover)", flush=True) hh_.run() st_ = hh_.modelStatusToString(hh_.getModelStatus()); info_ = hh_.getInfo(); sol_ = hh_.getSolution() xx_ = np.asarray(sol_.col_value, dtype=np.float64); yy_ = np.asarray(sol_.row_dual, dtype=np.float64) pv_ = float(f1 @ xx_) if len(xx_) == nv else float('nan') if len(xx_) == nv: eqr_ = float(np.abs(Aeq @ xx_).max()) if neq_ else 0.0; ubv_ = float(np.maximum(Aub @ xx_ - bub, 0).max()) bdv_ = float(max(np.maximum(lb - xx_, 0).max(), np.maximum(xx_ - ub, 0).max())) else: eqr_ = ubv_ = bdv_ = float('nan') # float weak-duality bound: y free on equality rows, y <= 0 on the <= rows (clipped), d = c - A^T y, every column boxed db_ = float('nan') if len(yy_) == m_: y_ = yy_.copy(); y_[neq_:] = np.minimum(y_[neq_:], 0.0) d_ = np.asarray(f1, dtype=np.float64) - A_.T @ y_ if np.isfinite(lb).all() and np.isfinite(ub).all(): db_ = float(y_[neq_:] @ np.asarray(bub, dtype=np.float64) + np.sum(np.where(d_ >= 0, d_ * lb, d_ * ub))) print(f" NOX: status {st_}; ipm iters {info_.ipm_iteration_count}, crossover iters {info_.crossover_iteration_count}; HiGHS objective " f"{info_.objective_function_value:.9f}; primal value f1.x = {pv_:.9f} (max |Aeq x| {eqr_:.1e}, max ub violation {ubv_:.1e}, " f"max bound violation {bdv_:.1e}); float dual bound {db_:.9f}; HiGHS max primal/dual infeas {info_.max_primal_infeasibility:.1e} / " f"{info_.max_dual_infeasibility:.1e}", flush=True) if os.environ.get('CONT_SAVEX'): np.save(os.environ['CONT_SAVEX'], xx_) if len(xx_) == nv and len(yy_) == m_: # residuals, acceptance, net-multiplier dual bounds (pruned and raw) DS_ = float(os.environ.get('CONT_DEADSCALE', '1')); epsd_ = float(os.environ.get('CONT_DEADEPS', '0')) nd_ = TDEAD.shape[0] if TDEAD is not None else 0; npos_ = Gm.shape[0] live_p = float(np.abs(Aeq @ xx_).max()) if neq_ else 0.0 dead_p = float(np.abs(TDEAD @ xx_).max()) if nd_ else 0.0 pos_p = float((Gm @ xx_).min()); boxv = float(max(np.maximum(lb - xx_, 0).max(), np.maximum(xx_ - ub, 0).max())) live_r = float(max(np.abs(_RAW['T'] @ xx_).max(), (np.abs(Aeq[_RAW['T'].shape[0]:] @ xx_).max() if neq_ > _RAW['T'].shape[0] else 0.0))) if neq_ else 0.0 dead_r = float(np.abs(_RAW['TDEAD'] @ xx_).max()) if nd_ else 0.0 pos_r = float((_RAW['Gm'] @ xx_).min()); obj_r = float(_RAW['f1'] @ xx_) acc_ = (st_ == 'Optimal') and (live_p <= 1e-7) and (dead_p <= epsd_ + 1e-10) and (pos_p >= -1 - 1e-7) and (boxv <= 1e-7) y_ = yy_.copy(); y_[neq_:neq_ + npos_] = np.minimum(y_[neq_:neq_ + npos_], 0.0) if nd_: yp_, ym_ = y_[neq_ + npos_:neq_ + npos_ + nd_].copy(), y_[neq_ + npos_ + nd_:].copy(); lam_ = yp_ - ym_ y_[neq_ + npos_:neq_ + npos_ + nd_] = np.minimum(lam_, 0.0); y_[neq_ + npos_ + nd_:] = np.minimum(-lam_, 0.0) bfull_ = np.concatenate([np.zeros(neq_), np.asarray(bub, dtype=np.float64)]) def _bound(Amat, cvec): d__ = np.asarray(cvec, dtype=np.float64) - Amat.T @ y_ terms = np.where(d__ >= 0, d__ * lb, d__ * ub); val = float(bfull_ @ y_ + terms.sum()) allow = 1e-13 * (float(np.abs(bfull_ * y_).sum()) + float(np.abs(terms).sum())) return val - allow Lp_ = _bound(A_, f1) nT_ = _RAW['T'].shape[0] parts = [_csr(_RAW['T'])] + ([Aeq[nT_:]] if neq_ > nT_ else []) + [-_RAW['Gm']] + ([_csr(DS_ * _RAW['TDEAD']), _csr(-DS_ * _RAW['TDEAD'])] if nd_ else []) Lr_ = _bound(_vs2(parts).tocsr(), _RAW['f1']) _fv = xx_[_fs_cls] + np.where(_fs_h >= 0, xx_[np.maximum(_fs_h, 0)], 0.0) + np.where(_fs_g >= 0, xx_[np.maximum(_fs_g, 0)], 0.0) _fvc = xx_[_cn_cls] + np.where(_cn_h >= 0, xx_[np.maximum(_cn_h, 0)], 0.0) + np.where(_cn_g >= 0, xx_[np.maximum(_cn_g, 0)], 0.0) print(f" FREEMIN_CLOSED: min f over {len(_fvc)} closed-interval free nodes = {float(_fvc.min()):.12f}; over bare nodes = {float(_fvc[_cn_bare].min()) if _cn_bare.any() else float('nan'):.12f}", flush=True) print(f" FREEMIN: min f over all {len(_fs_cls)} free grid shapes = {float(_fv.min()):.12f}", flush=True) print(f" REC: status {st_}; objective pruned {pv_:.9f} raw {obj_r:.9f}; float dual lower bound pruned {Lp_:.9f} raw {Lr_:.9f}; " f"pruned residuals live {live_p:.2e} dead {dead_p:.2e} (eps {epsd_:g}, scale {DS_:g}) min positivity {pos_p:.12f} box viol {boxv:.1e}; " f"raw residuals live {live_r:.2e} dead {dead_r:.2e} min positivity {pos_r:.12f}; ACCEPT {acc_}; rows: live {neq_}, positivity {npos_}, dead {nd_}", flush=True) if os.environ.get('CONT_SAVEX'): np.savez_compressed(os.environ['CONT_SAVEX'] + '.rec.npz', x=xx_, y=yy_, lb=lb, ub=ub) res = _NS(status=0 if st_ == 'Optimal' else 1, message=f'NOX {st_}', x=xx_ if st_ == 'Optimal' else None, fun=pv_ if st_ == 'Optimal' else None, nit=info_.ipm_iteration_count) else: res = linprog(f1, A_ub=Aub, b_ub=bub, A_eq=Aeq, b_eq=np.zeros(Aeq.shape[0]), bounds=np.column_stack([lb, ub]), method=os.environ.get('CONT_METHOD', 'highs-ipm'), options=dict(presolve=bool(int(os.environ.get('CONT_PRESOLVE', '0'))), **({'primal_feasibility_tolerance': float(os.environ['CONT_PTOL']), 'dual_feasibility_tolerance': float(os.environ['CONT_PTOL'])} if os.environ.get('CONT_PTOL') else {}), **({'ipm_optimality_tolerance': float(os.environ['CONT_OTOL'])} if os.environ.get('CONT_OTOL') else {}), **({'time_limit': float(os.environ['CONT_TLIM'])} if os.environ.get('CONT_TLIM') else {}), **({'disp': True} if os.environ.get('CONT_DISP') else {}))) print(f"[{time.time()-t0:.1f}s] LP status {res.status} ({res.message})", flush=True) print(f" result: fun {getattr(res, 'fun', None)}; nit {getattr(res, 'nit', None)}", flush=True) # night2 if res.status != 0 and getattr(res, 'x', None) is not None: # report the returned nonoptimal point and its residuals; feasibility is not asserted xx_ = np.asarray(res.x); print(f" NONOPT POINT: f1.x = {float(f1 @ xx_):.6f}; max eq residual {float(np.abs(Aeq @ xx_).max()):.2e}; max ub violation {float(np.maximum(Aub @ xx_ - bub, 0).max()):.2e}", flush=True) np.save(os.environ.get('CONT_SAVEX', 'nonopt_x.npy'), xx_) if res.status == 0: x = res.x; z = f1 @ x print(f" min f(1) = {z:.6f} -> {'PRIME-FREE after rescaling, margin %.4f' % (1 - 1/abs(z)) if z < -1 else 'NOT prime-free, C- >= %.6f' % (1 + z)}") hv = x[Hc]; gv = x[Gc] if len(Gc) else np.zeros(1) print(f" H in [{hv.min():.3f}, {hv.max():.3f}] (at cap {int((np.abs(hv) > GCAP - 1e-6).sum())}); G in [{gv.min():.3f}, {gv.max():.3f}] (at cap {int((np.abs(gv) > GCAP - 1e-6).sum())}); |T x| {np.abs(T @ x).max():.1e}") ~~~~~~~~~~~~ #### file: check_sec8_values.py - kind: script - as run: SHA-256 a9a53ee00780b06a57a8cefb13e9f971138605206f3c36b0217809c65d7bb132 - as shipped: SHA-256 02a79d25c646ce5dbefa149f5b9827afdc029de6824d955cb47cbcb41c6e3baf - identity: identical, after dropping docstrings and comments, to the file as run - trailing newline: yes ~~~~~~~~~~~~ python #!/usr/bin/env python3 """check_sec8_values.py: checks every family value that section 8 quotes against the thresholds that section 8's method paragraph states, reading only the raw logs written by cont_hms_rec6.py. Usage: python3 -B check_sec8_values.py DIR [DIR ...] (each named raw log is looked up in the DIRs in order) For each quoted point the accepted solve is the first of _p0.raw, _p1.raw whose REC line says ACCEPT True (as rec6_table.py). It must have: the gate's PASS line; status Optimal; on the matrices after the coefficient deletion, equality residuals <= 1e-7, the rows of Lemma 8.1 within eps + 1e-10 with eps = 1e-9 (eps = 1e9 and no residual requirement on those rows in the stated comparison without them, the Fdrop points), positivity >= -1 - 1e-7, box violation <= 1e-7; on the matrices before the deletion, residuals of the retained twisted rows <= 5e-6 and positivity >= -1 - 5e-6; FREEMIN_CLOSED >= -1 - 1e-9; and a finite binary64 weak-duality lower bound (raw). Negative control: the two accepted sweep solves that section 8 does not quote (G at U = C = 20000, the forced family with bins of one grid step at U = 10^4) must be refused. Exit status 0 only when every quoted point passes and both controls are refused. Floats; this checks printed solver output against stated thresholds and is not a certificate.""" import os, re, sys, math PAT = re.compile(r'REC: status (\w[\w ]*); objective pruned ([-0-9.]+) raw ([-0-9.]+); float dual lower bound pruned ([-0-9.naif]+) raw ([-0-9.naif]+); ' r'pruned residuals live ([0-9.e+-]+) dead ([0-9.e+-]+) \(eps ([0-9.e+-]+), scale [0-9.]+\) min positivity ([-0-9.]+) box viol ([0-9.e+-]+); ' r'raw residuals live ([0-9.e+-]+) dead ([0-9.e+-]+) min positivity ([-0-9.]+); ACCEPT (\w+)') FMC = re.compile(r'FREEMIN_CLOSED: min f over (\d+) closed-interval free nodes = ([-0-9.]+)') QUOTED = { # the 31 runs of record (rec6_table.py's points) and the quoted sweep points 'M72_24_u4': 'R6M_n72_e36_a24_u4_lnone_g50_eps1e-9', 'M72_25_u4': 'R6M_n72_e36_a25_u4_lnone_g50_eps1e-9', 'M72_25_u50': 'R6M_n72_e36_a25_u50_lnone_g50_eps1e-9', 'M72_25_u1000': 'R6M_n72_e36_a25_u1000_lnone_g1000_eps1e-9', 'M72_24_u1000': 'R6M_n72_e36_a24_u1000_lnone_g1000_eps1e-9', 'M72_26_u2000': 'R6M_n72_e36_a26_u2000_lnone_g2000_eps1e-9', 'M72_27_u2000': 'R6M_n72_e36_a27_u2000_lnone_g2000_eps1e-9', 'M90_31': 'R6M_n90_e45_a31_u4_lnone_g50_eps1e-9', 'M90_32': 'R6M_n90_e45_a32_u4_lnone_g50_eps1e-9', 'M96_34': 'R6M_n96_e48_a34_u4_lnone_g50_eps1e-9', 'M96_35': 'R6M_n96_e48_a35_u4_lnone_g50_eps1e-9', 'M108_36': 'R6M_n108_e36_a36_u4_lnone_g50_eps1e-9', 'M108_37': 'R6M_n108_e36_a37_u4_lnone_g50_eps1e-9', 'M90e30_30': 'R6M_n90_e30_a30_u4_lnone_g50_eps1e-9', 'M108e27_36': 'R6M_n108_e27_a36_u4_lnone_g50_eps1e-9', 'F23_u4': 'R6F_n72_e36_a23_u4_lnone_g10000_eps1e-9', 'F24_u4': 'R6F_n72_e36_a24_u4_lnone_g10000_eps1e-9', 'F25_u4': 'R6F_n72_e36_a25_u4_lnone_g10000_eps1e-9', 'F26_u4': 'R6F_n72_e36_a26_u4_lnone_g10000_eps1e-9', 'F23_u1000': 'R6F_n72_e36_a23_u1000_lnone_g10000_eps1e-9', 'F24_u1000': 'R6F_n72_e36_a24_u1000_lnone_g10000_eps1e-9', 'F25_u1000': 'R6F_n72_e36_a25_u1000_lnone_g10000_eps1e-9', 'F26_u1000': 'R6F_n72_e36_a26_u1000_lnone_g10000_eps1e-9', 'Fdrop25': 'R6Fdrop_n72_e36_a25_u4_lnone_g10000_eps1e9', 'Fdrop26': 'R6Fdrop_n72_e36_a26_u4_lnone_g10000_eps1e9', 'Fbin25_u4': 'R6Fbin1_n72_e36_a25_u4_lnone_g10000_eps1e-9', 'Fbin26_u4': 'R6Fbin1_n72_e36_a26_u4_lnone_g10000_eps1e-9', 'Fbin25_u1000': 'R6Fbin1_n72_e36_a25_u1000_lnone_g10000_eps1e-9', 'Fbin26_u1000': 'R6Fbin1_n72_e36_a26_u1000_lnone_g10000_eps1e-9', 'G25_u4': 'R6G_n72_e36_a25_u4_lnone_g50_eps1e-9', 'G25_u1000': 'R6G_n72_e36_a25_u1000_lnone_g1000_eps1e-9', 'G25_UC50': 'G25_UC50', 'G25_UC200': 'G25_UC200', 'G25_UC1000': 'G25_UC1000', 'G25_UC5000': 'G25_UC5000', } CONTROLS = {'G25_UC20000': 'G25_UC20000', 'Fbin25_U10000': 'Fbin25_U10000'} # accepted solves that section 8 does not quote def find(dirs, name): for d in dirs: p = os.path.join(d, name) if os.path.exists(p): return p return None def check(dirs, key, stem): """returns (verdict line, passed)""" for p in (0, 1): f = find(dirs, f'{stem}_p{p}.raw') if f is None: continue t = open(f).read(); m = PAT.search(t) if not m or m.group(14) != 'True': continue fm = FMC.search(t); exempt = key.startswith('Fdrop') st, val, val_r, bd_r = m.group(1), float(m.group(2)), float(m.group(3)), float(m.group(5)) lp, dp, eps, pp, bx = (float(m.group(i)) for i in (6, 7, 8, 9, 10)) lr, dr, pr = float(m.group(11)), float(m.group(12)), float(m.group(13)) fmc = float(fm.group(2)) if fm else float('nan') fails = [] if not ('GATE (trapezoid' in t and ': PASS' in t.split('GATE (trapezoid', 1)[1][:200]): fails.append('no gate PASS') if st != 'Optimal': fails.append('status ' + st) if not lp <= 1e-7: fails.append('pruned live %.2e' % lp) if not (eps == (1e9 if exempt else 1e-9) and dp <= eps + 1e-10): fails.append('pruned dead %.2e (eps %g)' % (dp, eps)) if not pp >= -1 - 1e-7: fails.append('pruned positivity %.12f' % pp) if not bx <= 1e-7: fails.append('box %.1e' % bx) if not lr <= 5e-6: fails.append('raw live %.2e' % lr) if not (exempt or dr <= 5e-6): fails.append('raw dead %.2e' % dr) if not pr >= -1 - 5e-6: fails.append('raw positivity %.12f' % pr) if not fmc >= -1 - 1e-9: fails.append('FREEMIN_CLOSED %s' % fmc) if not math.isfinite(bd_r): fails.append('no finite raw dual bound') line = (f"{key:14s} {os.path.basename(f):52s} value {val:+.9f} (raw {val_r:+.9f}) bound raw {bd_r:+.9f} | pruned live {lp:.1e} " f"dead {dp:.1e} pos {pp:.10f} | raw live {lr:.2e} dead {dr:.2e}{' (exempt)' if exempt else ''} pos {pr:.9f} | FMC {fmc:.12f} | " + ('PASS' if not fails else 'FAIL: ' + '; '.join(fails))) return line, not fails return f'{key:14s} {stem}: no accepted solve found | FAIL', False if __name__ == '__main__': dirs = sys.argv[1:] if not dirs: sys.exit(__doc__) npass = 0 print('# quoted values (each must PASS)') for k, s in QUOTED.items(): line, ok = check(dirs, k, s); npass += ok; print(line) print('# negative controls (accepted solves not quoted; each must FAIL)') nref = 0 for k, s in CONTROLS.items(): line, ok = check(dirs, k, s); nref += (not ok) and 'no accepted solve' not in line; print(line) good = npass == len(QUOTED) and nref == len(CONTROLS) print(f'VERDICT: {npass}/{len(QUOTED)} quoted values pass; {nref}/{len(CONTROLS)} negative controls refused -> {"OK" if good else "NOT OK"}') sys.exit(0 if good else 1) ~~~~~~~~~~~~ #### file: cont_hms_rec6g.py - kind: script - as run: SHA-256 9f91a75469f75a1e912d3884b5ffe5460cf2c9da6d5cbfecc86616b16cdc7e08 - as shipped: SHA-256 1f76b928b50241c5d9d1dea26d1f1142c06b9d5d4bb161bccad8e745c32374e9 - identity: identical, after dropping docstrings and comments, to the file as run - trailing newline: yes ~~~~~~~~~~~~ python # cont_hms_rec6g.py: a diagnostic copy of cont_hms_rec6.py (exact support of the G columns); see its docstring. """cont_hms_rec6g.py: a diagnostic copy of cont_hms_rec6.py with one programme change: the support of a G column is decided by exact integer feasibility instead of the sign of its floating-point (FFT) value. It prints the restored keys (its GEXACT line); CONT_GLISTONLY=1 stops after that line (enumeration only). Otherwise identical to cont_hms_rec6.py (see its docstring). Used for the measurement of the omitted G columns (keys.out, gexact_runs.out, gexact_runs2.out); not a source of the paper's values. Floats; not a certificate.""" import sys, time, os, pickle from fractions import Fraction as Fr import numpy as np from scipy.signal import fftconvolve from scipy.optimize import linprog from scipy.sparse import csr_matrix t0 = time.time() N = int(sys.argv[1]); eta = Fr(int(sys.argv[2]), int(sys.argv[3])); alpha = Fr(int(sys.argv[4]), int(sys.argv[5])) EDG = [Fr(e) for e in sys.argv[6].split(',')] if sys.argv[6] else [] PHIMAX = float(sys.argv[7]); ASUB = int(sys.argv[8]); SSUB = int(sys.argv[9]) assert eta <= alpha < Fr(1, 2) h = 1.0 / N def idx(x): v = x * N; assert v.denominator == 1, f"{x} not on grid N={N}"; return int(v) ie, ia, ihf = idx(eta), idx(alpha), idx(Fr(1, 2)) edges = [alpha] + EDG + [Fr(1, 2)]; iedges = [idx(e) for e in edges]; NB = len(edges) - 1 assert all(edges[i_] < edges[i_ + 1] for i_ in range(len(edges) - 1)), 'bin edges must be strictly increasing between alpha and 1/2' inv = np.zeros(N + 1); inv[1:] = 1.0 / (np.arange(1, N + 1) * h) def conv(F, G): return fftconvolve(F, G)[:N + 1] * h delta = np.zeros(N + 1); delta[0] = 1.0 / h HC = os.environ.get('CONT_HALFCLOSE', 'trap'); HCF = {'trap': 0.5, 'closed': 1.0}[HC] # rec5: weight of a boundary at exactly 1/2 SELFTEST = bool(os.environ.get('CONT_GATE_SELFTEST')) # rec5: positive control of the gate (emulates the old a = 1/2 defect) def band(lo, hi): F = np.zeros(N + 1); F[lo:hi + 1] = inv[lo:hi + 1]; F[lo] *= 0.5; F[hi] *= 0.5; return F def trunc_ge(F, i0): G = F.copy(); G[:max(i0, 0)] = 0.0 if 0 <= i0 <= N: G[i0] *= HCF return G def shift(F, i): G = np.zeros(N + 1); G[i:] = F[:N + 1 - i]; return G s_den = band(ie, ia) JMAX = int(1 / eta) + 1; KMAX = int(1 / alpha) + 1 S = [delta] for j in range(1, JMAX + 1): S.append(conv(S[-1], s_den) / j) S2 = S[2] Rcap = {} for e in iedges[1:]: r = band(ia, e) if HC == 'closed' and e == ihf: r[ihf] = 0.0 # rec5 closed: a component exactly 1/2 is large P = [delta] for k in range(1, KMAX + 1): P.append(conv(P[-1], r) / k) Rcap[e] = P def Rbin(k, b): if k == 0: return delta hi = iedges[b] return Rcap[hi][k] if b == 1 else Rcap[hi][k] - Rcap[iedges[b - 1]][k] Rall = Rcap[ihf] bins_k = lambda k: [0] if k == 0 else list(range(1, NB + 1)) etaf, alf, ef = float(eta), float(alpha), [float(e) for e in edges] def support(j, k, b): if k == 0: return (j * etaf, j * alf) return (j * etaf + (k - 1) * alf + ef[b - 1], j * alf + k * ef[b]) def in_sup(sup, t): return sup[1] > sup[0] and sup[0] - 1e-12 <= t <= sup[1] + 1e-12 print(f"[{time.time()-t0:.1f}s] N={N} eta={eta} alpha={alpha} edges={[str(e) for e in edges]}", flush=True) # ---- Lemma K kernels (c-part) ---- PS = {} for e in iedges[1:]: for bs in range(JMAX + 1): for br in range(KMAX + 1): PS[(e, bs, br)] = conv(S[bs], Rcap[e][br]) if br else S[bs] def Lam_cap(jy, ky, e): K = jy + ky; L = -PS[(e, jy, ky)].copy(); acc = np.zeros(N + 1) for bs in range(jy + 1): for br in range(ky + 1): if (bs, br) in ((0, 0), (jy, ky)): continue acc += (-1) ** (bs + br) * conv(trunc_ge(PS[(e, bs, br)], ihf), PS[(e, jy - bs, ky - br)]) return L + (-1) ** (K + 1) * acc Lam = {} for jy in range(JMAX + 1): for ky in range(KMAX + 1): if jy + ky < 2: continue if ky == 0: Lam[(jy, 0, 0)] = Lam_cap(jy, 0, ihf); continue prev = None for b in range(1, NB + 1): cur = Lam_cap(jy, ky, iedges[b]); Lam[(jy, ky, b)] = cur if prev is None else cur - prev; prev = cur Lam0 = sum(Lam_cap(0, ky, ihf) for ky in range(2, KMAX + 1)) del PS # ---- XS(t; theta) for theta = 1/2 - x, x in [0, ia] ---- XS = {} for x in range(0, ia + 1): acc = np.zeros(N + 1) for k in range(1, KMAX + 1): for br in range(1, k + 1): acc += (-1) ** (k + br) * conv(trunc_ge(Rall[br], ihf - x), Rall[k - br]) XS[x] = acc # ======================= cont_hms ======================= MS = ia // ie while MS * ie > ia: MS -= 1 Hgrid = {j: list(range(j * ie, ia + 1)) for j in range(1, MS + 1)} Rall = Rcap[ihf] Rsgn = sum(((-1) ** k) * Rall[k] for k in range(0, KMAX + 1)); Runs = sum(Rall[k] for k in range(0, KMAX + 1)) na = ia + 1; ng = N - ia + 1 JG = min(JMAX, (N - ia) // ie) R1 = Rall[1] XSmat = np.array([XS[x] for x in range(0, ia + 1)]) XS1 = np.zeros((ng, N + 1)) for sg_ in range(0, min(ng, ihf)): XS1[sg_] = trunc_ge(R1, ihf - sg_) if ihf < ng: XS1[ihf] = (1.0 - HCF) * R1 # small sub-sum exactly 1/2: S and S + r cancel in the weight HCF tr_mask = (np.arange(N + 1) >= ihf).astype(float); tr_mask[ihf] = HCF Lam0Tr = trunc_ge(Lam0, ihf) K0sgn = conv(Rsgn, Lam0Tr); K0uns = conv(Runs, Lam0Tr) mgrid = list(range(ie, ia + 1)) JS = max(JMAX, JG) def restricted(m): """kernels with every EXTRA small part <= m (grid index)""" sm = s_den.copy(); sm[m + 1:] = 0.0 Sm = [delta] for j in range(1, JS + 1): Sm.append(conv(Sm[-1], sm) / j) w = h * sm # M_j over sigma, tau in [0, ng) Mj = [None] * (JG + 1); Mj[0] = np.zeros((ng, ng)); Mj[0][0, 0] = 1.0 / h for j in range(1, JG + 1): acc = np.zeros((ng, ng)) for t in range(ie, m + 1): if w[t] == 0.0: continue acc[:, t:] += w[t] * Mj[j - 1][:, :ng - t] acc[t:, t:] -= w[t] * Mj[j - 1][:ng - t, :ng - t] Mj[j] = acc / j LJ = {}; Ks = {}; Ku = {} for j in range(1, MS + 1): A = (Mj[j][:na, :na].T @ XSmat) * ((-1) ** (j - 1)) L = np.zeros((na, N + 1)) for tau in Hgrid[j]: L[tau, tau:] = A[tau, :N + 1 - tau] if HC == 'trap': L[:, ihf] *= 2.0 # rec4 (1): right-limit value at a = 1/2, as Lam_cap if SELFTEST: L[:, ihf] *= 0.5 LJ[j] = L LT = L * tr_mask[None, :] Ks[j] = np.array([conv(Rsgn, LT[t]) for t in range(na)]); Ku[j] = np.array([conv(Runs, LT[t]) for t in range(na)]) LG = {}; LGT = {} for j in range(1, JG + 1): A = (Mj[j].T @ XS1) * ((-1) ** (j - 1)) L = np.zeros((ng, N + 1)) for tau in range(j * ie, ng): L[tau, tau:] = A[tau, :N + 1 - tau] if HC == 'trap': L[:, ihf] *= 2.0 # rec4 (1), G kernels if SELFTEST: L[:, ihf] *= 0.5 LG[j] = L; LGT[j] = L * tr_mask[None, :] LAS = {} for jy in range(2, JG + 1): acc = np.zeros(N + 1) for bs in range(1, jy): acc += (-1) ** bs * conv(trunc_ge(Sm[bs], ihf), Sm[jy - bs]) LAS[jy] = -Sm[jy] + ((-1) ** (jy + 1)) * acc # all-small Lemma K kernel, parts <= m (Lam_cap(jy, 0, .) with S restricted) LAST = {jy: trunc_ge(L, ihf) for jy, L in LAS.items()} return dict(S=Sm, LJ=LJ, Ks=Ks, Ku=Ku, LG=LG, LGT=LGT, LAS=LAS, LAST=LAST) RK = {} for m in mgrid: RK[m] = restricted(m) EMPTY = None print(f"[{time.time()-t0:.1f}s] restricted kernels for {len(mgrid)} maxima (MS {MS}, JG {JG})", flush=True) if '--kcheck' in sys.argv: top = RK[ia] print(" kcheck S_j(<= ia) vs S_j:", max(np.abs(top['S'][j] - S[j]).max() for j in range(1, min(JS, JMAX) + 1))) print(" kcheck LAS(<= ia) vs Lam[(jy,0,0)]:", max(np.abs(top['LAS'][jy] - Lam[(jy, 0, 0)]).max() for jy in top['LAS'] if (jy, 0, 0) in Lam)) def Sd(m, j, x): # density of j extra parts with total x and maximum EXACTLY m (vectorised in x) a = RK[m]['S'][j][x] return a - RK[m - 1]['S'][j][x] if m - 1 >= ie else a # ---- variables ---- classes = [(j, k, b) for j in range(JMAX + 1) for k in range(KMAX + 1) if j + k >= 1 for b in bins_k(k)] D = {cl: (conv(S[cl[0]], Rbin(cl[1], cl[2])) if cl[1] else S[cl[0]]) for cl in classes}; D[(0, 0, 0)] = delta cid = {cl: n for n, cl in enumerate(classes)}; nc = len(classes) def feas_max(j, tau): # maxima m with j parts of total tau and maximum exactly m: exact combinatorial test (17:5x; testing if j == 1: return [tau] if ie <= tau <= ia else [] # Sd > 0 admitted FFT round-off, giving impossible (tau, m) rows and variables) return [m for m in mgrid if m <= tau - (j - 1) * ie and j * m >= tau] hid = {}; gid = {}; nv = nc for j in range(1, MS + 1): for tau in Hgrid[j]: for m in feas_max(j, tau): hid[(j, tau, m)] = nv; nv += 1 GBAND = bool(os.environ.get('CONT_GBAND')) # restricted family: G resolved by m only on the dead band tau - m in [1-2a, a] glump = {} _g_restored = [] for j in range(2, JG + 1): for tg in range(ihf + 1, N - ia + 1): if not (j * ie <= tg <= j * ia): continue # rec6g: support by exact integer feasibility (j parts in [eta, alpha'], total tg) if S[j][tg] <= 0.0: _g_restored.extend((j, tg, m_) for m_ in feas_max(j, tg)) # rec6g: keys rec6's FFT-sign test dropped for m in feas_max(j, tg): if GBAND and not (N - 2 * ia <= tg - m <= ia): if (j, tg) not in glump: glump[(j, tg)] = nv; nv += 1 gid[(j, tg, m)] = glump[(j, tg)] else: gid[(j, tg, m)] = nv; nv += 1 print(f" GEXACT: {len(_g_restored)} G keys restored (rec6 dropped them on S[j][tg] <= 0): {_g_restored}", flush=True) if os.environ.get('CONT_GLISTONLY'): sys.exit(0) print(f" variables: {nc} constants, {len(hid)} H(j, tau, m), {len(gid)} G(j, tau, m) keys in {len(set(gid.values()))} columns" f"{' (GBAND: %d lumped)' % len(glump) if GBAND else ''}", flush=True) def new(): return np.zeros(nv) LamTr = {k_: trunc_ge(L, ihf) for k_, L in Lam.items()} w_classes = [(0, 0, 0)] + [cl for cl in classes if cl[0] + cl[1] >= 1 and in_sup(support(*cl), support(*cl)[0]) and support(*cl)[0] <= 0.5 + 1e-12] _cc = {} def conv_w_lam(wc, key): k_ = (wc, key) if k_ not in _cc: _cc[k_] = LamTr[key] if wc == (0, 0, 0) else conv(D[wc], LamTr[key]) return _cc[k_] # ---- f(1) ---- f1 = new() for key, L in Lam.items(): if (cl := cid.get(key)) is not None: f1[cl] += L[N] def dmax(m, key, j, idx): # RK[m][key][j][idx] - RK[m-1][key][j][idx] a = RK[m][key][j][idx] return a - RK[m - 1][key][j][idx] if m - 1 >= ie else a for (j, tau, m), col in hid.items(): f1[col] += h * dmax(m, 'LJ', j, (tau, N)) for (j, tg, m), col in gid.items(): f1[col] += h * dmax(m, 'LG', j, (tg, N)) # ---- rows (jD, sigma, mD) ---- row_list = [(0, 0, 0)] for jD in range(1, MS + 1): for s_ in Hgrid[jD]: for mD in feas_max(jD, s_): row_list.append((jD, s_, mD)) rid = {r: n for n, r in enumerate(row_list)}; nr_ = len(row_list) T = np.zeros((nr_, nv)); I = np.zeros((nr_, nv)); DIAG = np.zeros((nr_, nv), dtype=bool) # c-part (independent of mD): computed per (jD, sigma) as in cont_hm, copied to every mD cpart = {} for jD in range(0, MS + 1): sig = [0] if jD == 0 else Hgrid[jD]; tt = np.array([N - s_ for s_ in sig]) Tc = np.zeros((len(sig), nc)); Ic = np.zeros((len(sig), nc)) for cl in classes: jp, kp, bp = cl; tgt = cid.get((jD + jp, kp, bp)) if tgt is None: continue vals = D[cl][tt]; sg = (-1) ** (jD + jp + kp); Tc[:, tgt] += sg * vals; Ic[:, tgt] += vals for wc in w_classes: jw, kw, bw = wc; sgn = (-1) ** (1 + jD + jw + kw) for key in Lam: jy, ky, byy = key; tgt = cid.get((jD + jw + jy, kw + ky, max(bw, byy))) if tgt is None: continue v = conv_w_lam(wc, key)[tt]; Tc[:, tgt] += sgn * v; Ic[:, tgt] += v for n_, s_ in enumerate(sig): cpart[(jD, s_)] = (Tc[n_], Ic[n_]) for r, (jD, s_, mD) in enumerate(row_list): T[r, :nc] = cpart[(jD, s_)][0]; I[r, :nc] = cpart[(jD, s_)][1] print(f"[{time.time()-t0:.1f}s] c-part of {nr_} rows", flush=True) # extra-parts kernels per maximum m: KH[m][D][sigma, tau] (signed, jD-independent part) and KG[m][D][sigma, tg]; unsigned KHu, KGu def build_K(m): R = RK[m]; Sm = R['S'] KH = {}; KHu = {}; KG = {}; KGu = {} sig_all = np.arange(na) for Dl in range(1, MS + 1): Ht = np.zeros((na, na)); Hu = np.zeros((na, na)) for s_ in range(na): taus = np.arange(s_, na); x = taus - s_ v = h * Sm[Dl][x] * Rsgn[N - taus]; Ht[s_, taus] += ((-1) ** Dl) * v; Hu[s_, taus] += h * Sm[Dl][x] * Runs[N - taus] for jw in range(0, Dl + 1): jy = Dl - jw; sg = (-1) ** (1 + jw) if jw == 0: vt = h * R['Ks'][Dl][x, N - s_]; vu = h * R['Ku'][Dl][x, N - s_] elif jy == 0: vt = h * Sm[jw][x] * K0sgn[N - taus]; vu = h * Sm[jw][x] * K0uns[N - taus] else: vt = np.zeros(len(taus)); vu = np.zeros(len(taus)) for i_, t_ in enumerate(taus): vv = np.arange(s_, t_ + 1) vt[i_] = h * h * np.dot(Sm[jw][vv - s_], R['Ks'][jy][t_ - vv, N - vv]) vu[i_] = h * h * np.dot(Sm[jw][vv - s_], R['Ku'][jy][t_ - vv, N - vv]) Ht[s_, taus] += sg * vt; Hu[s_, taus] += vu KH[Dl] = Ht; KHu[Dl] = Hu gt = np.arange(ihf + 1, ng) for Dl in range(1, JG + 1): Gt = np.zeros((na, ng)); Gu = np.zeros((na, ng)) for s_ in range(na): x = gt - s_ v = h * Sm[Dl][x] * R1[N - gt]; Gt[s_, gt] += ((-1) ** (Dl + 1)) * v; Gu[s_, gt] += v for jw in range(0, Dl + 1): jy = Dl - jw if jy >= 1: if jw == 0: vt = h * R['LGT'][jy][x, N - s_] else: vt = np.zeros(len(gt)) for i_, t_ in enumerate(gt): vv = np.arange(s_, t_ + 1) vt[i_] = h * h * np.dot(Sm[jw][vv - s_], R['LGT'][jy][t_ - vv, N - vv]) Gt[s_, gt] += ((-1) ** (1 + jw)) * vt; Gu[s_, gt] += vt if jy >= 2: base = R['LAST'][jy] if jw == 0 else conv(Sm[jw], R['LAST'][jy]) vt = h * R1[N - gt] * base[x] Gt[s_, gt] += ((-1) ** jw) * vt; Gu[s_, gt] += vt KG[Dl] = Gt; KGu[Dl] = Gu return KH, KHu, KG, KGu KK = {m: build_K(m) for m in mgrid} print(f"[{time.time()-t0:.1f}s] extra-part kernels built", flush=True) def Kexact(m, which, Dl): # kernel with the extra parts' maximum EXACTLY m: K(<= m) - K(<= m-1) a = KK[m][which][Dl] return a - KK[m - 1][which][Dl] if m - 1 >= ie else a # assemble the H- and G-parts of every row for r, (jD, s_, mD) in enumerate(row_list): sgD = (-1) ** jD if jD >= 1: # diagonal: H_jD(sigma, mD), all-rough completions (free and extended) col = hid[(jD, s_, mD)] T[r, col] += sgD * (Rsgn[N - s_] - K0sgn[N - s_]); I[r, col] += Runs[N - s_] + K0uns[N - s_]; DIAG[r, col] = True for (J, tau, M), col in hid.items(): if J <= jD or tau < s_: continue Dl = J - jD if M < mD: continue if jD == 0: kt = Kexact(M, 0, Dl)[0, tau]; ku = Kexact(M, 1, Dl)[0, tau] elif M == mD: kt = KK[mD][0][Dl][s_, tau]; ku = KK[mD][1][Dl][s_, tau] else: kt = Kexact(M, 0, Dl)[s_, tau]; ku = Kexact(M, 1, Dl)[s_, tau] T[r, col] += sgD * kt; I[r, col] += ku for (J, tg, M), col in gid.items(): if J <= jD: continue Dl = J - jD if Dl > JG or M < mD: continue if jD == 0: kt = Kexact(M, 2, Dl)[0, tg]; ku = Kexact(M, 3, Dl)[0, tg] elif M == mD: kt = KK[mD][2][Dl][s_, tg]; ku = KK[mD][3][Dl][s_, tg] else: kt = Kexact(M, 2, Dl)[s_, tg]; ku = Kexact(M, 3, Dl)[s_, tg] T[r, col] += sgD * kt; I[r, col] += ku print(f"[{time.time()-t0:.1f}s] rows assembled", flush=True) Hc = np.array(sorted(set(hid.values()))); Gc = np.array(sorted(set(gid.values())), dtype=int) offH = np.where(DIAG[:, Hc], 0.0, I[:, Hc]) print(f" Type I check: c-block {np.abs(I[:, :nc]).max():.3e}; H diagonal {np.abs(I[DIAG]).max():.3e}; H integrated/h {np.abs(offH).max()/h:.3e};" f" G/h {np.abs(I[:, Gc]).max()/h if len(Gc) else 0:.3e}; T0 row: c {np.abs(I[0, :nc]).max():.3e}, H/h {np.abs(I[0, Hc]).max()/h:.3e}," f" G/h {np.abs(I[0, Gc]).max()/h if len(Gc) else 0:.3e}", flush=True) _ti = dict(c=float(np.abs(I[:, :nc]).max()), Hd=float(np.abs(I[DIAG]).max()), Hi_h=float(np.abs(offH).max() / h), G_h=float(np.abs(I[:, Gc]).max() / h) if len(Gc) else 0.0) if HC == 'closed': _fail = [k_ for k_ in ('c', 'Hd', 'Hi_h', 'G_h') if not _ti[k_] <= 1e-9] print(f" GATE (closed-1/2 exactness): {'PASS' if not _fail else 'FAIL ' + str(_fail)} {_ti}", flush=True) if os.environ.get('CONT_GATEONLY'): sys.exit(3 if _fail else 0) else: _fail = [k_ for k_, ok_ in (('G_h', _ti['G_h'] <= 1e-9), ('c', _ti['c'] <= 10 * h), ('Hd', _ti['Hd'] <= 2 * h)) if not ok_] import subprocess as _sp _env = dict(os.environ, CONT_HALFCLOSE='closed', CONT_GATEONLY='1') _r = _sp.run([sys.executable, '-B', os.path.abspath(__file__)] + sys.argv[1:], env=_env, capture_output=True, text=True) _gl = [l_ for l_ in _r.stdout.splitlines() if 'GATE' in l_] if _r.returncode != 0: _fail.append('closed-1/2 exactness (exit %d): %s' % (_r.returncode, _gl[-1].strip() if _gl else _r.stderr[-300:])) print(f" GATE (trapezoid: G/h <= 1e-9, c <= 10h, H diag <= 2h; closed-1/2 exactness <= 1e-9): {'PASS' if not _fail else 'FAIL ' + str(_fail)}; " f"{_ti}; closed: {_gl[-1].strip() if _gl else 'no output'}", flush=True) if _fail and not os.environ.get('CONT_NOGATE'): sys.exit(f'cont_hms_rec5: GATE FAILED {_fail}') if os.environ.get('CONT_GATEONLY'): sys.exit(0) if '--check' in sys.argv: sys.exit() # ---- optional diagnostics ---- TDEAD = None if os.environ.get('DEAD_SVD'): # diagnostic: spectrum of the dead rows' G-block (and H-block size), then exit dead_ = [r for r, (jD_, s_, m_) in enumerate(row_list) if jD_ >= 1 and s_ >= N - 2 * ia + 2] gc_ = np.array(sorted(set(gid.values())), dtype=int) hc_ = np.array(sorted(set(hid.values())), dtype=int) Bd = T[np.ix_(dead_, gc_)] U_, sv, _ = np.linalg.svd(Bd, full_matrices=False) rk = int((sv > 1e-9 * sv[0]).sum()) NL = U_[:, rk:] # left null space of the dead G-block (dead-row combinations G cannot touch) Cd = T[np.ix_(dead_, np.arange(nc))] if NL.shape[1]: Mc = NL.T @ Cd; svc = np.linalg.svd(Mc, compute_uv=False) touched = int((np.abs(Cd).max(axis=0) > 1e-12).sum()) print(f" DEAD_NULL: G-rank {rk} of {len(dead_)} dead rows -> {NL.shape[1]} dead-row combinations G cannot touch; on the class constants they have rank " f"{(svc > 1e-9 * svc[0]).sum() if svc[0] > 0 else 0} (largest sv {svc[0]:.2e}); constants touched by dead rows {touched} of {nc}", flush=True) else: print(f" DEAD_NULL: G-block has full row rank {rk} = {len(dead_)}: no G-free dead-row combination", flush=True) ks = [k for k in (1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024) if k < len(sv)] print(f" DEAD_SVD s_max: dead rows {len(dead_)} x G cols {len(gc_)}; sv0 {sv[0]:.3e}; sv/sv0 at k={ks}: " + ", ".join(f"{sv[k]/sv[0]:.1e}" for k in ks) + f"; #(>1e-3 sv0) {(sv > 1e-3*sv[0]).sum()}, #(>1e-6) {(sv > 1e-6*sv[0]).sum()}, #(>1e-9) {(sv > 1e-9*sv[0]).sum()}; H-block max {np.abs(T[np.ix_(dead_, hc_)]).max():.2e}", flush=True) sys.exit(0) if os.environ.get('CONT_DEADEPS'): # relax the dead rows to |T_dead x| <= eps (they make the equality LP numerically degenerate) dead = [r for r, (jD_, s_, m_) in enumerate(row_list) if jD_ >= 1 and s_ >= N - 2 * ia] live = [r for r in range(nr_) if r not in set(dead)] TDEAD = T[dead].copy(); T = T[live]; nr_ = len(live) print(f" CONT_DEADEPS = {os.environ['CONT_DEADEPS']}: {len(dead)} dead rows relaxed to |T x| <= eps", flush=True) if os.environ.get('CONT_DROPDEAD'): keep = [r for r, (jD_, s_, m_) in enumerate(row_list) if jD_ == 0 or s_ < N - 2 * ia] print(f" CONT_DROPDEAD: keeping {len(keep)} of {nr_} rows", flush=True) T = T[keep]; nr_ = len(keep) # ---- positivity (sparse) ---- from scipy.sparse import coo_matrix, csr_matrix as _csr GCAP = float(os.environ.get('CONT_GCAP', '50')); MSUB = int(os.environ.get('CONT_MSUB', '1')) lb = np.full(nv, -np.inf); ub = np.full(nv, np.inf); lb[:nc] = -1.0; ub[:nc] = PHIMAX nvac = 0 for (j, tau, m), col in hid.items(): lb[col] = -GCAP; ub[col] = GCAP # rec6 (2): the H columns of the interval of Lemma 8.1 are free too for col in gid.values(): lb[col] = -GCAP; ub[col] = GCAP if os.environ.get('CONT_NOGREC'): for col in gid.values(): lb[col] = 0.0; ub[col] = 0.0 print(f" CONT_NOGREC: {len(set(gid.values()))} G columns fixed to 0", flush=True) if os.environ.get('CONT_LBC'): sys.exit('cont_hms_rec4: CONT_LBC is not supported (focused read M1)') LBC = os.environ.get('CONT_LBC'); low_cls = [] if LBC is not None: LBC = float(LBC) for cl_ in classes: if cl_[0] >= 1 and (cl_[1] >= 2 or (cl_[1] == 1 and cl_[0] >= 2 and not os.environ.get('CONT_NOGREC'))): lb[cid[cl_]] = LBC; low_cls.append(cl_) hid_arr = -np.ones((MS + 1, na, na), dtype=int) for (j, tau, m), col in hid.items(): hid_arr[j, tau, m] = col gid_arr = -np.ones((JG + 1, N + 1, na), dtype=int) for (j, tg, m), col in gid.items(): gid_arr[j, tg, m] = col # cumulative kernels as arrays over the maximum: LJc[jy][m, tau_y, a], LGc[jy][m, tau_y, a], LASc[jy][m, a] mi = np.array(mgrid); NM = len(mgrid) LJc = {jy: np.array([RK[m]['LJ'][jy] for m in mgrid]) for jy in range(1, MS + 1)} LGc = {jy: np.array([RK[m]['LG'][jy] for m in mgrid]) for jy in range(1, JG + 1)} LASc = {jy: np.array([RK[m]['LAS'][jy] for m in mgrid]) for jy in range(2, JG + 1)} def exact_from(cum, mb): """rows over M = mb..ia of the 'maximum exactly M, or <= mb at M = mb' kernel: cum has a leading axis over mgrid (18:0x: callers now pass cum already sliced at the a-index, so this copies small arrays only)""" k0 = mb - ie out = cum[k0:].copy() out[1:] -= cum[k0:-1] return out # out[0] = cum(<= mb); out[i] = cum(<= mb+i) - cum(<= mb+i-1) LJa = {jy: np.ascontiguousarray(np.moveaxis(LJc[jy], 2, 0)) for jy in LJc} # [a][m, tau_y] LGa = {jy: np.ascontiguousarray(np.moveaxis(LGc[jy], 2, 0)) for jy in LGc} LASa = {jy: np.ascontiguousarray(LASc[jy].T) for jy in LASc} # [a][m] from array import array as _arr Rr, Rc, Rv = _arr('q'), _arr('q'), _arr('d'); nrow = 0; desc = [] def emit(entries): global nrow for c_, v_ in entries.items(): if v_ != 0.0: Rr.append(nrow); Rc.append(c_); Rv.append(v_) nrow += 1 def emit_arrays(base_dict, cols, vals): d = dict(base_dict) for c_, v_ in zip(cols, vals): d[c_] = d.get(c_, 0.0) + v_ emit(d) # free rows for cl in classes: j, k, b = cl if not in_sup(support(j, k, b), 1.0) or j == 0 or k == 0: continue supR = support(0, k, b) for (jj, tau, m), col in hid.items(): if jj == j and in_sup(supR, 1 - tau * h) and lb[col] != ub[col]: emit({cid[cl]: 1.0, col: 1.0}); desc.append(('free', cl)) if k == 1: seen = set() for (jj, tg, m), col in gid.items(): if jj == j and in_sup(supR, 1 - tg * h) and col not in seen: seen.add(col); emit({cid[cl]: 1.0, col: 1.0}); desc.append(('freeG', cl)) # every free grid shape; classes with a shape carrying no active added term keep c >= -1 _fs_cls, _fs_h, _fs_g = [], [], [] def _parts(total, lower, prefix): if total == 0: yield prefix; return for p_ in range(lower, min(ihf - 1, total) + 1): if total == p_ or total - p_ >= p_: yield from _parts(total - p_, p_, prefix + (p_,)) _pure = set() for _sh in _parts(N, ie, ()): _sm = [q for q in _sh if q <= ia]; _ro = [q for q in _sh if q > ia] _j, _k, _tau = len(_sm), len(_ro), sum(_sm) _b = 0 if not _ro else next(i_ for i_ in range(1, len(iedges)) if max(_ro) <= iedges[i_]) _cl = (_j, _k, _b) if _cl not in cid: continue _hc = hid.get((_j, _tau, max(_sm))) if (_sm and _tau <= ia) else None _gc = gid.get((_j, _tau, max(_sm))) if (_sm and _k == 1) else None _act = [cc_ for cc_ in (_hc, _gc) if cc_ is not None and not (lb[cc_] == 0 and ub[cc_] == 0)] if not _act: _pure.add(_cl) _fs_cls.append(cid[_cl]); _fs_h.append(_hc if _hc is not None else -1); _fs_g.append(_gc if _gc is not None else -1) _fs_cls = np.array(_fs_cls); _fs_h = np.array(_fs_h); _fs_g = np.array(_fs_g) _restored = [] for _cl in sorted(_pure): if lb[cid[_cl]] < -1: lb[cid[_cl]] = -1.0; _restored.append(_cl) _cn_cls, _cn_h, _cn_g, _cn_bare = [], [], [], [] # rec4 (3): closed-interval free nodes, as band() weights them for _cl in classes: _j, _k, _b = _cl if _k == 0: _taus = [N] if (_j >= 1 and _j * ie <= N <= _j * ia) else [] else: _sR = support(0, _k, _b); _taus = [t_ for t_ in range(_j * ie, min(_j * ia, N) + 1) if in_sup(_sR, 1 - t_ * h)] if _j >= 1 else ([0] if in_sup(_sR, 1.0) else []) for _t in _taus: for _m in (feas_max(_j, _t) if _j >= 1 else [None]): _hc = hid.get((_j, _t, _m)) if (_j >= 1 and _t <= ia) else None _gc = gid.get((_j, _t, _m)) if (_j >= 1 and _k == 1) else None _hc = _hc if (_hc is not None and not (lb[_hc] == 0 and ub[_hc] == 0)) else None _gc = _gc if (_gc is not None and not (lb[_gc] == 0 and ub[_gc] == 0)) else None _cn_cls.append(cid[_cl]); _cn_h.append(-1 if _hc is None else _hc); _cn_g.append(-1 if _gc is None else _gc) _cn_bare.append(_hc is None and _gc is None) _cn_cls = np.array(_cn_cls); _cn_h = np.array(_cn_h); _cn_g = np.array(_cn_g); _cn_bare = np.array(_cn_bare) print(f" REC4: {len(_cn_cls)} closed-interval free nodes (class, tau, m), {int(_cn_bare.sum())} bare (constant only); constants lower bound min {lb[:nc].min():g}", flush=True) print(f" REC3: {len(_fs_cls)} free grid shapes; classes with a pure-constant free shape: {len(_pure)}; lower bound restored to -1 on {len(_restored)}: {_restored}", flush=True) npure = 0 if LBC is not None: for cl_ in low_cls: j_, k_, b_ = cl_ if k_ != 1: continue missing = False for rho in range(max(iedges[b_ - 1], ia + 1), min(iedges[b_], ihf - 1) + 1): tg = N - rho if not (j_ * ie <= tg <= j_ * ia): continue for m_ in feas_max(j_, tg): if (j_, tg, m_) not in gid: missing = True; break if missing: break if missing: emit({cid[cl_]: 1.0}); desc.append(('pure_c', cl_)); npure += 1 print(f" CONT_LBC = {LBC}: lowered {len(low_cls)} class constants; restored c >= -1 rows: {npure}", flush=True) nfree = nrow alist = list(range(ihf, N, ASUB)) for bc in w_classes: jb, kb, bb = bc if bc == (0, 0, 0): continue supR = support(0, kb, bb) for ai in alist: a = ai * h; tb = N - ai if not in_sup(support(jb, kb, bb), tb * h): continue base = {} for key, L in Lam.items(): jy, ky, bp = key; tgt = cid.get((jb + jy, kb + ky, max(bb, bp))) if tgt is not None: base[tgt] = base.get(tgt, 0.0) + a * L[ai] if jb == 0: cols = []; vals = [] for jy in range(1, MS + 1): # fragments carry the whole small part: H(jy, tau_y, M), M = their maximum ex = exact_from(LJa[jy][ai], ie) # (NM, na): maximum exactly M for k0, M in enumerate(mgrid): cc = hid_arr[jy, :, M]; ok = cc >= 0 if ok.any(): cols += list(cc[ok]); vals += list(h * a * ex[k0][ok]) if kb == 1: for jy in range(2, JG + 1): ex = exact_from(LASa[jy][ai], ie) for k0, M in enumerate(mgrid): cg = gid_arr[jy, ai, M] if ai <= N else -1 if cg >= 0: cols.append(cg); vals.append(a * ex[k0]) if bc == (0, 1, NB) and ai == ihf: continue # rec6 (1): no positivity row at E = (1/2, 1/2) emit_arrays(base, cols, vals); desc.append(('ext', bc)); continue if kb: cands = [t_ for t_ in range(jb * ie, min(jb * ia, tb) + 1, SSUB) if in_sup(supR, (tb - t_) * h)] else: cands = [tb] if jb * ie <= tb <= jb * ia else [] done_const = False for tbeta in cands: mbs = feas_max(jb, tbeta) if tbeta <= ia else [None] if tbeta > ia and kb >= 2: if not done_const: emit(dict(base)); desc.append(('ext', bc)); done_const = True continue if tbeta > ia: mbs = feas_max(jb, tbeta) for mb in mbs[::MSUB]: cols = []; vals = [] if tbeta <= ia: c0 = hid_arr[jb, tbeta, mb] if jb <= MS else -1 if c0 >= 0: cols.append(c0); vals.append(a * Lam0[ai]) for jy in range(1, MS - jb + 1): ex = exact_from(LJa[jy][ai], mb) # (ia - mb + 1, na) for k0 in range(ex.shape[0]): M = mb + k0; taus_y = np.arange(jy * ie, ia - tbeta + 1) if len(taus_y) == 0: continue cc = hid_arr[jb + jy, tbeta + taus_y, M]; ok = cc >= 0 if ok.any(): cols += list(cc[ok]); vals += list(h * a * ex[k0][taus_y[ok]]) if kb == 0: for jy in range(1, JG - jb + 1): ex = exact_from(LGa[jy][ai], mb) taus_y = np.arange(jy * ie, ng); tt_ = tbeta + taus_y; okt = tt_ <= N taus_y, tt_ = taus_y[okt], tt_[okt] for k0 in range(ex.shape[0]): M = mb + k0; cc = gid_arr[jb + jy, tt_, M]; ok = cc >= 0 if ok.any(): cols += list(cc[ok]); vals += list(h * a * ex[k0][taus_y[ok]]) elif kb == 1: for jy in range(2, JG - jb + 1): ex = exact_from(LASa[jy][ai], mb) tt_ = tbeta + ai if tt_ > N: continue for k0 in range(len(ex)): cg = gid_arr[jb + jy, tt_, mb + k0] if cg >= 0: cols.append(cg); vals.append(a * ex[k0]) emit_arrays(base, cols, vals); desc.append(('ext', bc)) Gm = coo_matrix((Rv, (Rr, Rc)), shape=(nrow, nv)).tocsr() print(f"[{time.time()-t0:.1f}s] positivity rows: free {nfree}, extended {nrow - nfree}; nnz {Gm.nnz}; vacuous H fixed {nvac}; |L| <= {GCAP}", flush=True) # ---- control: tie L(j, tau, m) across m (must reproduce cont_hmg) ---- Aeq = _csr(T) if os.environ.get('CONT_TIEM'): tr, tc, tv = [], [], []; nt = 0 for store in (hid, gid): groups = {} for (j, tau, m), col in store.items(): groups.setdefault((j, tau), []).append(col) for cols in groups.values(): cols = sorted(set(cols)) for c1, c2 in zip(cols[:-1], cols[1:]): tr += [nt, nt]; tc += [c1, c2]; tv += [1.0, -1.0]; nt += 1 Aeq = _csr(np.vstack([T, coo_matrix((tv, (tr, tc)), shape=(nt, nv)).toarray()])) if nt else Aeq print(f" CONT_TIEM: {nt} tie rows", flush=True) NOUP = bool(os.environ.get('CONT_NOUPPER')) from scipy.sparse import vstack as _svs Aub = -Gm if NOUP else _svs([-Gm, Gm]).tocsr(); bub = np.ones(nrow) if NOUP else np.concatenate([np.ones(nrow), np.full(nrow, PHIMAX)]) _RAW = dict(T=T.copy(), f1=f1.copy(), Gm=Gm.copy(), TDEAD=(TDEAD.copy() if TDEAD is not None else None)) TINY = float(os.environ.get('CONT_TINY', '1e-9')) # remove coefficients below CONT_TINY from T, the dead rows, the positivity rows and the objective; this perturbs the rows and the objective, and the residuals reported below refer to the modified matrices T[np.abs(T) < TINY] = 0.0; f1[np.abs(f1) < TINY] = 0.0 Gm.data[np.abs(Gm.data) < TINY] = 0.0; Gm.eliminate_zeros() Aeq = _csr(T) if not os.environ.get('CONT_TIEM') else Aeq if os.environ.get('CONT_TIEM'): Aeq = Aeq.tocsr(); Aeq.data[np.abs(Aeq.data) < TINY] = 0.0; Aeq.eliminate_zeros() Aub = -Gm if NOUP else _svs([-Gm, Gm]).tocsr() if TDEAD is not None: TDEAD[np.abs(TDEAD) < TINY] = 0.0; epsd = float(os.environ['CONT_DEADEPS']) DS = float(os.environ.get('CONT_DEADSCALE', '1')) Aub = _svs([Aub, _csr(DS * TDEAD), _csr(-DS * TDEAD)]).tocsr(); bub = np.concatenate([bub, np.full(2 * TDEAD.shape[0], DS * epsd)]) print(f" sanity: finite T {np.isfinite(T).all()}, f1 {np.isfinite(f1).all()}, G {np.isfinite(Gm.data).all()}; max|T| {np.abs(T).max():.3e}, max|G| {np.abs(Gm.data).max():.3e}, max|f1| {np.abs(f1).max():.3e}; min|nonzero T| {np.abs(T[T != 0]).min():.3e}", flush=True) if os.environ.get('CONT_FEASCHECK'): r0 = linprog(np.zeros(nv), A_ub=Aub, b_ub=bub, A_eq=Aeq, b_eq=np.zeros(Aeq.shape[0]), bounds=np.column_stack([lb, ub]), method='highs', options={'presolve': True}) print(f" feasibility LP (zero objective): status {r0.status} ({r0.message})", flush=True) if os.environ.get('CONT_NOX'): # bundled HiGHS, IPM, run_crossover off; primal value + residuals + a float dual bound from scipy.optimize._highspy import _core as _hc from scipy.sparse import vstack as _vs2 from types import SimpleNamespace as _NS hh_ = _hc._Highs() for k_, v_ in (('output_flag', bool(os.environ.get('CONT_DISP'))), ('solver', 'ipm'), ('run_crossover', 'off'), ('presolve', 'on' if int(os.environ.get('CONT_PRESOLVE', '0')) else 'off'), ('ipm_optimality_tolerance', float(os.environ.get('CONT_OTOL', '1e-8'))), ('primal_feasibility_tolerance', float(os.environ.get('CONT_PTOL', '1e-7'))), ('dual_feasibility_tolerance', float(os.environ.get('CONT_PTOL', '1e-7'))), ('time_limit', float(os.environ.get('CONT_TLIM', '1e9')))): hh_.setOptionValue(k_, v_) INF_ = hh_.getInfinity() A_ = _vs2([Aeq, Aub]).tocsc(); A_.sort_indices(); neq_ = Aeq.shape[0]; m_ = A_.shape[0] rl_ = np.concatenate([np.zeros(neq_), np.full(Aub.shape[0], -INF_)]); ru_ = np.concatenate([np.zeros(neq_), np.asarray(bub, dtype=np.float64)]) hh_.addRows(m_, rl_, ru_, 0, np.zeros(m_, dtype=np.int32), np.zeros(0, dtype=np.int32), np.zeros(0)) lbh_ = np.where(np.isinf(lb), -INF_, lb); ubh_ = np.where(np.isinf(ub), INF_, ub) hh_.addCols(nv, np.asarray(f1, dtype=np.float64), lbh_, ubh_, A_.nnz, A_.indptr[:-1].astype(np.int32), A_.indices.astype(np.int32), A_.data.astype(np.float64)) print(f"[{time.time()-t0:.1f}s] NOX: model {hh_.getNumRow()} rows x {hh_.getNumCol()} cols, nnz {hh_.getNumNz()}; running IPM (no crossover)", flush=True) hh_.run() st_ = hh_.modelStatusToString(hh_.getModelStatus()); info_ = hh_.getInfo(); sol_ = hh_.getSolution() xx_ = np.asarray(sol_.col_value, dtype=np.float64); yy_ = np.asarray(sol_.row_dual, dtype=np.float64) pv_ = float(f1 @ xx_) if len(xx_) == nv else float('nan') if len(xx_) == nv: eqr_ = float(np.abs(Aeq @ xx_).max()) if neq_ else 0.0; ubv_ = float(np.maximum(Aub @ xx_ - bub, 0).max()) bdv_ = float(max(np.maximum(lb - xx_, 0).max(), np.maximum(xx_ - ub, 0).max())) else: eqr_ = ubv_ = bdv_ = float('nan') # float weak-duality bound: y free on equality rows, y <= 0 on the <= rows (clipped), d = c - A^T y, every column boxed db_ = float('nan') if len(yy_) == m_: y_ = yy_.copy(); y_[neq_:] = np.minimum(y_[neq_:], 0.0) d_ = np.asarray(f1, dtype=np.float64) - A_.T @ y_ if np.isfinite(lb).all() and np.isfinite(ub).all(): db_ = float(y_[neq_:] @ np.asarray(bub, dtype=np.float64) + np.sum(np.where(d_ >= 0, d_ * lb, d_ * ub))) print(f" NOX: status {st_}; ipm iters {info_.ipm_iteration_count}, crossover iters {info_.crossover_iteration_count}; HiGHS objective " f"{info_.objective_function_value:.9f}; primal value f1.x = {pv_:.9f} (max |Aeq x| {eqr_:.1e}, max ub violation {ubv_:.1e}, " f"max bound violation {bdv_:.1e}); float dual bound {db_:.9f}; HiGHS max primal/dual infeas {info_.max_primal_infeasibility:.1e} / " f"{info_.max_dual_infeasibility:.1e}", flush=True) if os.environ.get('CONT_SAVEX'): np.save(os.environ['CONT_SAVEX'], xx_) if len(xx_) == nv and len(yy_) == m_: # residuals, acceptance, net-multiplier dual bounds (pruned and raw) DS_ = float(os.environ.get('CONT_DEADSCALE', '1')); epsd_ = float(os.environ.get('CONT_DEADEPS', '0')) nd_ = TDEAD.shape[0] if TDEAD is not None else 0; npos_ = Gm.shape[0] live_p = float(np.abs(Aeq @ xx_).max()) if neq_ else 0.0 dead_p = float(np.abs(TDEAD @ xx_).max()) if nd_ else 0.0 pos_p = float((Gm @ xx_).min()); boxv = float(max(np.maximum(lb - xx_, 0).max(), np.maximum(xx_ - ub, 0).max())) live_r = float(max(np.abs(_RAW['T'] @ xx_).max(), (np.abs(Aeq[_RAW['T'].shape[0]:] @ xx_).max() if neq_ > _RAW['T'].shape[0] else 0.0))) if neq_ else 0.0 dead_r = float(np.abs(_RAW['TDEAD'] @ xx_).max()) if nd_ else 0.0 pos_r = float((_RAW['Gm'] @ xx_).min()); obj_r = float(_RAW['f1'] @ xx_) acc_ = (st_ == 'Optimal') and (live_p <= 1e-7) and (dead_p <= epsd_ + 1e-10) and (pos_p >= -1 - 1e-7) and (boxv <= 1e-7) y_ = yy_.copy(); y_[neq_:neq_ + npos_] = np.minimum(y_[neq_:neq_ + npos_], 0.0) if nd_: yp_, ym_ = y_[neq_ + npos_:neq_ + npos_ + nd_].copy(), y_[neq_ + npos_ + nd_:].copy(); lam_ = yp_ - ym_ y_[neq_ + npos_:neq_ + npos_ + nd_] = np.minimum(lam_, 0.0); y_[neq_ + npos_ + nd_:] = np.minimum(-lam_, 0.0) bfull_ = np.concatenate([np.zeros(neq_), np.asarray(bub, dtype=np.float64)]) def _bound(Amat, cvec): d__ = np.asarray(cvec, dtype=np.float64) - Amat.T @ y_ terms = np.where(d__ >= 0, d__ * lb, d__ * ub); val = float(bfull_ @ y_ + terms.sum()) allow = 1e-13 * (float(np.abs(bfull_ * y_).sum()) + float(np.abs(terms).sum())) return val - allow Lp_ = _bound(A_, f1) nT_ = _RAW['T'].shape[0] parts = [_csr(_RAW['T'])] + ([Aeq[nT_:]] if neq_ > nT_ else []) + [-_RAW['Gm']] + ([_csr(DS_ * _RAW['TDEAD']), _csr(-DS_ * _RAW['TDEAD'])] if nd_ else []) Lr_ = _bound(_vs2(parts).tocsr(), _RAW['f1']) _fv = xx_[_fs_cls] + np.where(_fs_h >= 0, xx_[np.maximum(_fs_h, 0)], 0.0) + np.where(_fs_g >= 0, xx_[np.maximum(_fs_g, 0)], 0.0) _fvc = xx_[_cn_cls] + np.where(_cn_h >= 0, xx_[np.maximum(_cn_h, 0)], 0.0) + np.where(_cn_g >= 0, xx_[np.maximum(_cn_g, 0)], 0.0) print(f" FREEMIN_CLOSED: min f over {len(_fvc)} closed-interval free nodes = {float(_fvc.min()):.12f}; over bare nodes = {float(_fvc[_cn_bare].min()) if _cn_bare.any() else float('nan'):.12f}", flush=True) print(f" FREEMIN: min f over all {len(_fs_cls)} free grid shapes = {float(_fv.min()):.12f}", flush=True) print(f" REC: status {st_}; objective pruned {pv_:.9f} raw {obj_r:.9f}; float dual lower bound pruned {Lp_:.9f} raw {Lr_:.9f}; " f"pruned residuals live {live_p:.2e} dead {dead_p:.2e} (eps {epsd_:g}, scale {DS_:g}) min positivity {pos_p:.12f} box viol {boxv:.1e}; " f"raw residuals live {live_r:.2e} dead {dead_r:.2e} min positivity {pos_r:.12f}; ACCEPT {acc_}; rows: live {neq_}, positivity {npos_}, dead {nd_}", flush=True) if os.environ.get('CONT_SAVEX'): np.savez_compressed(os.environ['CONT_SAVEX'] + '.rec.npz', x=xx_, y=yy_, lb=lb, ub=ub) res = _NS(status=0 if st_ == 'Optimal' else 1, message=f'NOX {st_}', x=xx_ if st_ == 'Optimal' else None, fun=pv_ if st_ == 'Optimal' else None, nit=info_.ipm_iteration_count) else: res = linprog(f1, A_ub=Aub, b_ub=bub, A_eq=Aeq, b_eq=np.zeros(Aeq.shape[0]), bounds=np.column_stack([lb, ub]), method=os.environ.get('CONT_METHOD', 'highs-ipm'), options=dict(presolve=bool(int(os.environ.get('CONT_PRESOLVE', '0'))), **({'primal_feasibility_tolerance': float(os.environ['CONT_PTOL']), 'dual_feasibility_tolerance': float(os.environ['CONT_PTOL'])} if os.environ.get('CONT_PTOL') else {}), **({'ipm_optimality_tolerance': float(os.environ['CONT_OTOL'])} if os.environ.get('CONT_OTOL') else {}), **({'time_limit': float(os.environ['CONT_TLIM'])} if os.environ.get('CONT_TLIM') else {}), **({'disp': True} if os.environ.get('CONT_DISP') else {}))) print(f"[{time.time()-t0:.1f}s] LP status {res.status} ({res.message})", flush=True) print(f" result: fun {getattr(res, 'fun', None)}; nit {getattr(res, 'nit', None)}", flush=True) # night2 if res.status != 0 and getattr(res, 'x', None) is not None: # report the returned nonoptimal point and its residuals; feasibility is not asserted xx_ = np.asarray(res.x); print(f" NONOPT POINT: f1.x = {float(f1 @ xx_):.6f}; max eq residual {float(np.abs(Aeq @ xx_).max()):.2e}; max ub violation {float(np.maximum(Aub @ xx_ - bub, 0).max()):.2e}", flush=True) np.save(os.environ.get('CONT_SAVEX', 'nonopt_x.npy'), xx_) if res.status == 0: x = res.x; z = f1 @ x print(f" min f(1) = {z:.6f} -> {'PRIME-FREE after rescaling, margin %.4f' % (1 - 1/abs(z)) if z < -1 else 'NOT prime-free, C- >= %.6f' % (1 + z)}") hv = x[Hc]; gv = x[Gc] if len(Gc) else np.zeros(1) print(f" H in [{hv.min():.3f}, {hv.max():.3f}] (at cap {int((np.abs(hv) > GCAP - 1e-6).sum())}); G in [{gv.min():.3f}, {gv.max():.3f}] (at cap {int((np.abs(gv) > GCAP - 1e-6).sum())}); |T x| {np.abs(T @ x).max():.1e}") ~~~~~~~~~~~~ #### file: cont_hms_rec6e.py - kind: script - as run: SHA-256 aed2ead2cdb5ac9c9eb6c75dacb8e32c4284015dc882d24750b99bafff5dc61e - as shipped: SHA-256 11961ba73b7def6f4c18b0d7c6c5dd2e85b682d8b58b99a9f95a3e9dcd3bf6df - identity: identical, after dropping docstrings and comments, to the file as run - trailing newline: yes ~~~~~~~~~~~~ python # cont_hms_rec6e.py: a diagnostic copy of cont_hms_rec6.py (diagnostics at E); see its docstring. """cont_hms_rec6e.py: a diagnostic copy of cont_hms_rec6.py with diagnostics at the reading of E = (1/2, 1/2) as two rough components, class (0, 2, top bin): an EINFO line after the rows (that class's column and its T0 and f1 coefficients; CONT_EINFO_ONLY=1 stops there), an EOPT line after a solve (its value and T0' x), and CONT_ECAP=v, which caps that one constant at v. Without CONT_ECAP the programme is cont_hms_rec6.py's. Used for einfo.out and ediag.out; not a source of the paper's values. Floats; not a certificate.""" import sys, time, os, pickle from fractions import Fraction as Fr import numpy as np from scipy.signal import fftconvolve from scipy.optimize import linprog from scipy.sparse import csr_matrix t0 = time.time() N = int(sys.argv[1]); eta = Fr(int(sys.argv[2]), int(sys.argv[3])); alpha = Fr(int(sys.argv[4]), int(sys.argv[5])) EDG = [Fr(e) for e in sys.argv[6].split(',')] if sys.argv[6] else [] PHIMAX = float(sys.argv[7]); ASUB = int(sys.argv[8]); SSUB = int(sys.argv[9]) assert eta <= alpha < Fr(1, 2) h = 1.0 / N def idx(x): v = x * N; assert v.denominator == 1, f"{x} not on grid N={N}"; return int(v) ie, ia, ihf = idx(eta), idx(alpha), idx(Fr(1, 2)) edges = [alpha] + EDG + [Fr(1, 2)]; iedges = [idx(e) for e in edges]; NB = len(edges) - 1 assert all(edges[i_] < edges[i_ + 1] for i_ in range(len(edges) - 1)), 'bin edges must be strictly increasing between alpha and 1/2' inv = np.zeros(N + 1); inv[1:] = 1.0 / (np.arange(1, N + 1) * h) def conv(F, G): return fftconvolve(F, G)[:N + 1] * h delta = np.zeros(N + 1); delta[0] = 1.0 / h HC = os.environ.get('CONT_HALFCLOSE', 'trap'); HCF = {'trap': 0.5, 'closed': 1.0}[HC] # rec5: weight of a boundary at exactly 1/2 SELFTEST = bool(os.environ.get('CONT_GATE_SELFTEST')) # rec5: positive control of the gate (emulates the old a = 1/2 defect) def band(lo, hi): F = np.zeros(N + 1); F[lo:hi + 1] = inv[lo:hi + 1]; F[lo] *= 0.5; F[hi] *= 0.5; return F def trunc_ge(F, i0): G = F.copy(); G[:max(i0, 0)] = 0.0 if 0 <= i0 <= N: G[i0] *= HCF return G def shift(F, i): G = np.zeros(N + 1); G[i:] = F[:N + 1 - i]; return G s_den = band(ie, ia) JMAX = int(1 / eta) + 1; KMAX = int(1 / alpha) + 1 S = [delta] for j in range(1, JMAX + 1): S.append(conv(S[-1], s_den) / j) S2 = S[2] Rcap = {} for e in iedges[1:]: r = band(ia, e) if HC == 'closed' and e == ihf: r[ihf] = 0.0 # rec5 closed: a component exactly 1/2 is large P = [delta] for k in range(1, KMAX + 1): P.append(conv(P[-1], r) / k) Rcap[e] = P def Rbin(k, b): if k == 0: return delta hi = iedges[b] return Rcap[hi][k] if b == 1 else Rcap[hi][k] - Rcap[iedges[b - 1]][k] Rall = Rcap[ihf] bins_k = lambda k: [0] if k == 0 else list(range(1, NB + 1)) etaf, alf, ef = float(eta), float(alpha), [float(e) for e in edges] def support(j, k, b): if k == 0: return (j * etaf, j * alf) return (j * etaf + (k - 1) * alf + ef[b - 1], j * alf + k * ef[b]) def in_sup(sup, t): return sup[1] > sup[0] and sup[0] - 1e-12 <= t <= sup[1] + 1e-12 print(f"[{time.time()-t0:.1f}s] N={N} eta={eta} alpha={alpha} edges={[str(e) for e in edges]}", flush=True) # ---- Lemma K kernels (c-part) ---- PS = {} for e in iedges[1:]: for bs in range(JMAX + 1): for br in range(KMAX + 1): PS[(e, bs, br)] = conv(S[bs], Rcap[e][br]) if br else S[bs] def Lam_cap(jy, ky, e): K = jy + ky; L = -PS[(e, jy, ky)].copy(); acc = np.zeros(N + 1) for bs in range(jy + 1): for br in range(ky + 1): if (bs, br) in ((0, 0), (jy, ky)): continue acc += (-1) ** (bs + br) * conv(trunc_ge(PS[(e, bs, br)], ihf), PS[(e, jy - bs, ky - br)]) return L + (-1) ** (K + 1) * acc Lam = {} for jy in range(JMAX + 1): for ky in range(KMAX + 1): if jy + ky < 2: continue if ky == 0: Lam[(jy, 0, 0)] = Lam_cap(jy, 0, ihf); continue prev = None for b in range(1, NB + 1): cur = Lam_cap(jy, ky, iedges[b]); Lam[(jy, ky, b)] = cur if prev is None else cur - prev; prev = cur Lam0 = sum(Lam_cap(0, ky, ihf) for ky in range(2, KMAX + 1)) del PS # ---- XS(t; theta) for theta = 1/2 - x, x in [0, ia] ---- XS = {} for x in range(0, ia + 1): acc = np.zeros(N + 1) for k in range(1, KMAX + 1): for br in range(1, k + 1): acc += (-1) ** (k + br) * conv(trunc_ge(Rall[br], ihf - x), Rall[k - br]) XS[x] = acc # ======================= cont_hms ======================= MS = ia // ie while MS * ie > ia: MS -= 1 Hgrid = {j: list(range(j * ie, ia + 1)) for j in range(1, MS + 1)} Rall = Rcap[ihf] Rsgn = sum(((-1) ** k) * Rall[k] for k in range(0, KMAX + 1)); Runs = sum(Rall[k] for k in range(0, KMAX + 1)) na = ia + 1; ng = N - ia + 1 JG = min(JMAX, (N - ia) // ie) R1 = Rall[1] XSmat = np.array([XS[x] for x in range(0, ia + 1)]) XS1 = np.zeros((ng, N + 1)) for sg_ in range(0, min(ng, ihf)): XS1[sg_] = trunc_ge(R1, ihf - sg_) if ihf < ng: XS1[ihf] = (1.0 - HCF) * R1 # small sub-sum exactly 1/2: S and S + r cancel in the weight HCF tr_mask = (np.arange(N + 1) >= ihf).astype(float); tr_mask[ihf] = HCF Lam0Tr = trunc_ge(Lam0, ihf) K0sgn = conv(Rsgn, Lam0Tr); K0uns = conv(Runs, Lam0Tr) mgrid = list(range(ie, ia + 1)) JS = max(JMAX, JG) def restricted(m): """kernels with every EXTRA small part <= m (grid index)""" sm = s_den.copy(); sm[m + 1:] = 0.0 Sm = [delta] for j in range(1, JS + 1): Sm.append(conv(Sm[-1], sm) / j) w = h * sm # M_j over sigma, tau in [0, ng) Mj = [None] * (JG + 1); Mj[0] = np.zeros((ng, ng)); Mj[0][0, 0] = 1.0 / h for j in range(1, JG + 1): acc = np.zeros((ng, ng)) for t in range(ie, m + 1): if w[t] == 0.0: continue acc[:, t:] += w[t] * Mj[j - 1][:, :ng - t] acc[t:, t:] -= w[t] * Mj[j - 1][:ng - t, :ng - t] Mj[j] = acc / j LJ = {}; Ks = {}; Ku = {} for j in range(1, MS + 1): A = (Mj[j][:na, :na].T @ XSmat) * ((-1) ** (j - 1)) L = np.zeros((na, N + 1)) for tau in Hgrid[j]: L[tau, tau:] = A[tau, :N + 1 - tau] if HC == 'trap': L[:, ihf] *= 2.0 # rec4 (1): right-limit value at a = 1/2, as Lam_cap if SELFTEST: L[:, ihf] *= 0.5 LJ[j] = L LT = L * tr_mask[None, :] Ks[j] = np.array([conv(Rsgn, LT[t]) for t in range(na)]); Ku[j] = np.array([conv(Runs, LT[t]) for t in range(na)]) LG = {}; LGT = {} for j in range(1, JG + 1): A = (Mj[j].T @ XS1) * ((-1) ** (j - 1)) L = np.zeros((ng, N + 1)) for tau in range(j * ie, ng): L[tau, tau:] = A[tau, :N + 1 - tau] if HC == 'trap': L[:, ihf] *= 2.0 # rec4 (1), G kernels if SELFTEST: L[:, ihf] *= 0.5 LG[j] = L; LGT[j] = L * tr_mask[None, :] LAS = {} for jy in range(2, JG + 1): acc = np.zeros(N + 1) for bs in range(1, jy): acc += (-1) ** bs * conv(trunc_ge(Sm[bs], ihf), Sm[jy - bs]) LAS[jy] = -Sm[jy] + ((-1) ** (jy + 1)) * acc # all-small Lemma K kernel, parts <= m (Lam_cap(jy, 0, .) with S restricted) LAST = {jy: trunc_ge(L, ihf) for jy, L in LAS.items()} return dict(S=Sm, LJ=LJ, Ks=Ks, Ku=Ku, LG=LG, LGT=LGT, LAS=LAS, LAST=LAST) RK = {} for m in mgrid: RK[m] = restricted(m) EMPTY = None print(f"[{time.time()-t0:.1f}s] restricted kernels for {len(mgrid)} maxima (MS {MS}, JG {JG})", flush=True) if '--kcheck' in sys.argv: top = RK[ia] print(" kcheck S_j(<= ia) vs S_j:", max(np.abs(top['S'][j] - S[j]).max() for j in range(1, min(JS, JMAX) + 1))) print(" kcheck LAS(<= ia) vs Lam[(jy,0,0)]:", max(np.abs(top['LAS'][jy] - Lam[(jy, 0, 0)]).max() for jy in top['LAS'] if (jy, 0, 0) in Lam)) def Sd(m, j, x): # density of j extra parts with total x and maximum EXACTLY m (vectorised in x) a = RK[m]['S'][j][x] return a - RK[m - 1]['S'][j][x] if m - 1 >= ie else a # ---- variables ---- classes = [(j, k, b) for j in range(JMAX + 1) for k in range(KMAX + 1) if j + k >= 1 for b in bins_k(k)] D = {cl: (conv(S[cl[0]], Rbin(cl[1], cl[2])) if cl[1] else S[cl[0]]) for cl in classes}; D[(0, 0, 0)] = delta cid = {cl: n for n, cl in enumerate(classes)}; nc = len(classes) def feas_max(j, tau): # maxima m with j parts of total tau and maximum exactly m: exact combinatorial test (17:5x; testing if j == 1: return [tau] if ie <= tau <= ia else [] # Sd > 0 admitted FFT round-off, giving impossible (tau, m) rows and variables) return [m for m in mgrid if m <= tau - (j - 1) * ie and j * m >= tau] hid = {}; gid = {}; nv = nc for j in range(1, MS + 1): for tau in Hgrid[j]: for m in feas_max(j, tau): hid[(j, tau, m)] = nv; nv += 1 GBAND = bool(os.environ.get('CONT_GBAND')) # restricted family: G resolved by m only on the dead band tau - m in [1-2a, a] glump = {} for j in range(2, JG + 1): for tg in range(ihf + 1, N - ia + 1): if S[j][tg] <= 0.0: continue for m in feas_max(j, tg): if GBAND and not (N - 2 * ia <= tg - m <= ia): if (j, tg) not in glump: glump[(j, tg)] = nv; nv += 1 gid[(j, tg, m)] = glump[(j, tg)] else: gid[(j, tg, m)] = nv; nv += 1 print(f" variables: {nc} constants, {len(hid)} H(j, tau, m), {len(gid)} G(j, tau, m) keys in {len(set(gid.values()))} columns" f"{' (GBAND: %d lumped)' % len(glump) if GBAND else ''}", flush=True) def new(): return np.zeros(nv) LamTr = {k_: trunc_ge(L, ihf) for k_, L in Lam.items()} w_classes = [(0, 0, 0)] + [cl for cl in classes if cl[0] + cl[1] >= 1 and in_sup(support(*cl), support(*cl)[0]) and support(*cl)[0] <= 0.5 + 1e-12] _cc = {} def conv_w_lam(wc, key): k_ = (wc, key) if k_ not in _cc: _cc[k_] = LamTr[key] if wc == (0, 0, 0) else conv(D[wc], LamTr[key]) return _cc[k_] # ---- f(1) ---- f1 = new() for key, L in Lam.items(): if (cl := cid.get(key)) is not None: f1[cl] += L[N] def dmax(m, key, j, idx): # RK[m][key][j][idx] - RK[m-1][key][j][idx] a = RK[m][key][j][idx] return a - RK[m - 1][key][j][idx] if m - 1 >= ie else a for (j, tau, m), col in hid.items(): f1[col] += h * dmax(m, 'LJ', j, (tau, N)) for (j, tg, m), col in gid.items(): f1[col] += h * dmax(m, 'LG', j, (tg, N)) # ---- rows (jD, sigma, mD) ---- row_list = [(0, 0, 0)] for jD in range(1, MS + 1): for s_ in Hgrid[jD]: for mD in feas_max(jD, s_): row_list.append((jD, s_, mD)) rid = {r: n for n, r in enumerate(row_list)}; nr_ = len(row_list) T = np.zeros((nr_, nv)); I = np.zeros((nr_, nv)); DIAG = np.zeros((nr_, nv), dtype=bool) # c-part (independent of mD): computed per (jD, sigma) as in cont_hm, copied to every mD cpart = {} for jD in range(0, MS + 1): sig = [0] if jD == 0 else Hgrid[jD]; tt = np.array([N - s_ for s_ in sig]) Tc = np.zeros((len(sig), nc)); Ic = np.zeros((len(sig), nc)) for cl in classes: jp, kp, bp = cl; tgt = cid.get((jD + jp, kp, bp)) if tgt is None: continue vals = D[cl][tt]; sg = (-1) ** (jD + jp + kp); Tc[:, tgt] += sg * vals; Ic[:, tgt] += vals for wc in w_classes: jw, kw, bw = wc; sgn = (-1) ** (1 + jD + jw + kw) for key in Lam: jy, ky, byy = key; tgt = cid.get((jD + jw + jy, kw + ky, max(bw, byy))) if tgt is None: continue v = conv_w_lam(wc, key)[tt]; Tc[:, tgt] += sgn * v; Ic[:, tgt] += v for n_, s_ in enumerate(sig): cpart[(jD, s_)] = (Tc[n_], Ic[n_]) for r, (jD, s_, mD) in enumerate(row_list): T[r, :nc] = cpart[(jD, s_)][0]; I[r, :nc] = cpart[(jD, s_)][1] print(f"[{time.time()-t0:.1f}s] c-part of {nr_} rows", flush=True) # extra-parts kernels per maximum m: KH[m][D][sigma, tau] (signed, jD-independent part) and KG[m][D][sigma, tg]; unsigned KHu, KGu def build_K(m): R = RK[m]; Sm = R['S'] KH = {}; KHu = {}; KG = {}; KGu = {} sig_all = np.arange(na) for Dl in range(1, MS + 1): Ht = np.zeros((na, na)); Hu = np.zeros((na, na)) for s_ in range(na): taus = np.arange(s_, na); x = taus - s_ v = h * Sm[Dl][x] * Rsgn[N - taus]; Ht[s_, taus] += ((-1) ** Dl) * v; Hu[s_, taus] += h * Sm[Dl][x] * Runs[N - taus] for jw in range(0, Dl + 1): jy = Dl - jw; sg = (-1) ** (1 + jw) if jw == 0: vt = h * R['Ks'][Dl][x, N - s_]; vu = h * R['Ku'][Dl][x, N - s_] elif jy == 0: vt = h * Sm[jw][x] * K0sgn[N - taus]; vu = h * Sm[jw][x] * K0uns[N - taus] else: vt = np.zeros(len(taus)); vu = np.zeros(len(taus)) for i_, t_ in enumerate(taus): vv = np.arange(s_, t_ + 1) vt[i_] = h * h * np.dot(Sm[jw][vv - s_], R['Ks'][jy][t_ - vv, N - vv]) vu[i_] = h * h * np.dot(Sm[jw][vv - s_], R['Ku'][jy][t_ - vv, N - vv]) Ht[s_, taus] += sg * vt; Hu[s_, taus] += vu KH[Dl] = Ht; KHu[Dl] = Hu gt = np.arange(ihf + 1, ng) for Dl in range(1, JG + 1): Gt = np.zeros((na, ng)); Gu = np.zeros((na, ng)) for s_ in range(na): x = gt - s_ v = h * Sm[Dl][x] * R1[N - gt]; Gt[s_, gt] += ((-1) ** (Dl + 1)) * v; Gu[s_, gt] += v for jw in range(0, Dl + 1): jy = Dl - jw if jy >= 1: if jw == 0: vt = h * R['LGT'][jy][x, N - s_] else: vt = np.zeros(len(gt)) for i_, t_ in enumerate(gt): vv = np.arange(s_, t_ + 1) vt[i_] = h * h * np.dot(Sm[jw][vv - s_], R['LGT'][jy][t_ - vv, N - vv]) Gt[s_, gt] += ((-1) ** (1 + jw)) * vt; Gu[s_, gt] += vt if jy >= 2: base = R['LAST'][jy] if jw == 0 else conv(Sm[jw], R['LAST'][jy]) vt = h * R1[N - gt] * base[x] Gt[s_, gt] += ((-1) ** jw) * vt; Gu[s_, gt] += vt KG[Dl] = Gt; KGu[Dl] = Gu return KH, KHu, KG, KGu KK = {m: build_K(m) for m in mgrid} print(f"[{time.time()-t0:.1f}s] extra-part kernels built", flush=True) def Kexact(m, which, Dl): # kernel with the extra parts' maximum EXACTLY m: K(<= m) - K(<= m-1) a = KK[m][which][Dl] return a - KK[m - 1][which][Dl] if m - 1 >= ie else a # assemble the H- and G-parts of every row for r, (jD, s_, mD) in enumerate(row_list): sgD = (-1) ** jD if jD >= 1: # diagonal: H_jD(sigma, mD), all-rough completions (free and extended) col = hid[(jD, s_, mD)] T[r, col] += sgD * (Rsgn[N - s_] - K0sgn[N - s_]); I[r, col] += Runs[N - s_] + K0uns[N - s_]; DIAG[r, col] = True for (J, tau, M), col in hid.items(): if J <= jD or tau < s_: continue Dl = J - jD if M < mD: continue if jD == 0: kt = Kexact(M, 0, Dl)[0, tau]; ku = Kexact(M, 1, Dl)[0, tau] elif M == mD: kt = KK[mD][0][Dl][s_, tau]; ku = KK[mD][1][Dl][s_, tau] else: kt = Kexact(M, 0, Dl)[s_, tau]; ku = Kexact(M, 1, Dl)[s_, tau] T[r, col] += sgD * kt; I[r, col] += ku for (J, tg, M), col in gid.items(): if J <= jD: continue Dl = J - jD if Dl > JG or M < mD: continue if jD == 0: kt = Kexact(M, 2, Dl)[0, tg]; ku = Kexact(M, 3, Dl)[0, tg] elif M == mD: kt = KK[mD][2][Dl][s_, tg]; ku = KK[mD][3][Dl][s_, tg] else: kt = Kexact(M, 2, Dl)[s_, tg]; ku = Kexact(M, 3, Dl)[s_, tg] T[r, col] += sgD * kt; I[r, col] += ku print(f"[{time.time()-t0:.1f}s] rows assembled", flush=True) _cE = cid[(0, 2, NB)] # rec6e: the constant of E = (1/2, 1/2) read as two rough components print(f" EINFO: class (0, 2, {NB}) column {_cE}; T0 coefficient {T[0, _cE]:.15g}; f1 coefficient {f1[_cE]:.3g}; other twisted rows with a nonzero entry {int((np.abs(T[1:, _cE]) > 0).sum())} (max {np.abs(T[1:, _cE]).max():.2e})", flush=True) if os.environ.get("CONT_EINFO_ONLY"): sys.exit(0) Hc = np.array(sorted(set(hid.values()))); Gc = np.array(sorted(set(gid.values())), dtype=int) offH = np.where(DIAG[:, Hc], 0.0, I[:, Hc]) print(f" Type I check: c-block {np.abs(I[:, :nc]).max():.3e}; H diagonal {np.abs(I[DIAG]).max():.3e}; H integrated/h {np.abs(offH).max()/h:.3e};" f" G/h {np.abs(I[:, Gc]).max()/h if len(Gc) else 0:.3e}; T0 row: c {np.abs(I[0, :nc]).max():.3e}, H/h {np.abs(I[0, Hc]).max()/h:.3e}," f" G/h {np.abs(I[0, Gc]).max()/h if len(Gc) else 0:.3e}", flush=True) _ti = dict(c=float(np.abs(I[:, :nc]).max()), Hd=float(np.abs(I[DIAG]).max()), Hi_h=float(np.abs(offH).max() / h), G_h=float(np.abs(I[:, Gc]).max() / h) if len(Gc) else 0.0) if HC == 'closed': _fail = [k_ for k_ in ('c', 'Hd', 'Hi_h', 'G_h') if not _ti[k_] <= 1e-9] print(f" GATE (closed-1/2 exactness): {'PASS' if not _fail else 'FAIL ' + str(_fail)} {_ti}", flush=True) if os.environ.get('CONT_GATEONLY'): sys.exit(3 if _fail else 0) else: _fail = [k_ for k_, ok_ in (('G_h', _ti['G_h'] <= 1e-9), ('c', _ti['c'] <= 10 * h), ('Hd', _ti['Hd'] <= 2 * h)) if not ok_] import subprocess as _sp _env = dict(os.environ, CONT_HALFCLOSE='closed', CONT_GATEONLY='1') _r = _sp.run([sys.executable, '-B', os.path.abspath(__file__)] + sys.argv[1:], env=_env, capture_output=True, text=True) _gl = [l_ for l_ in _r.stdout.splitlines() if 'GATE' in l_] if _r.returncode != 0: _fail.append('closed-1/2 exactness (exit %d): %s' % (_r.returncode, _gl[-1].strip() if _gl else _r.stderr[-300:])) print(f" GATE (trapezoid: G/h <= 1e-9, c <= 10h, H diag <= 2h; closed-1/2 exactness <= 1e-9): {'PASS' if not _fail else 'FAIL ' + str(_fail)}; " f"{_ti}; closed: {_gl[-1].strip() if _gl else 'no output'}", flush=True) if _fail and not os.environ.get('CONT_NOGATE'): sys.exit(f'cont_hms_rec5: GATE FAILED {_fail}') if os.environ.get('CONT_GATEONLY'): sys.exit(0) if '--check' in sys.argv: sys.exit() # ---- optional diagnostics ---- TDEAD = None if os.environ.get('DEAD_SVD'): # diagnostic: spectrum of the dead rows' G-block (and H-block size), then exit dead_ = [r for r, (jD_, s_, m_) in enumerate(row_list) if jD_ >= 1 and s_ >= N - 2 * ia + 2] gc_ = np.array(sorted(set(gid.values())), dtype=int) hc_ = np.array(sorted(set(hid.values())), dtype=int) Bd = T[np.ix_(dead_, gc_)] U_, sv, _ = np.linalg.svd(Bd, full_matrices=False) rk = int((sv > 1e-9 * sv[0]).sum()) NL = U_[:, rk:] # left null space of the dead G-block (dead-row combinations G cannot touch) Cd = T[np.ix_(dead_, np.arange(nc))] if NL.shape[1]: Mc = NL.T @ Cd; svc = np.linalg.svd(Mc, compute_uv=False) touched = int((np.abs(Cd).max(axis=0) > 1e-12).sum()) print(f" DEAD_NULL: G-rank {rk} of {len(dead_)} dead rows -> {NL.shape[1]} dead-row combinations G cannot touch; on the class constants they have rank " f"{(svc > 1e-9 * svc[0]).sum() if svc[0] > 0 else 0} (largest sv {svc[0]:.2e}); constants touched by dead rows {touched} of {nc}", flush=True) else: print(f" DEAD_NULL: G-block has full row rank {rk} = {len(dead_)}: no G-free dead-row combination", flush=True) ks = [k for k in (1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024) if k < len(sv)] print(f" DEAD_SVD s_max: dead rows {len(dead_)} x G cols {len(gc_)}; sv0 {sv[0]:.3e}; sv/sv0 at k={ks}: " + ", ".join(f"{sv[k]/sv[0]:.1e}" for k in ks) + f"; #(>1e-3 sv0) {(sv > 1e-3*sv[0]).sum()}, #(>1e-6) {(sv > 1e-6*sv[0]).sum()}, #(>1e-9) {(sv > 1e-9*sv[0]).sum()}; H-block max {np.abs(T[np.ix_(dead_, hc_)]).max():.2e}", flush=True) sys.exit(0) if os.environ.get('CONT_DEADEPS'): # relax the dead rows to |T_dead x| <= eps (they make the equality LP numerically degenerate) dead = [r for r, (jD_, s_, m_) in enumerate(row_list) if jD_ >= 1 and s_ >= N - 2 * ia] live = [r for r in range(nr_) if r not in set(dead)] TDEAD = T[dead].copy(); T = T[live]; nr_ = len(live) print(f" CONT_DEADEPS = {os.environ['CONT_DEADEPS']}: {len(dead)} dead rows relaxed to |T x| <= eps", flush=True) if os.environ.get('CONT_DROPDEAD'): keep = [r for r, (jD_, s_, m_) in enumerate(row_list) if jD_ == 0 or s_ < N - 2 * ia] print(f" CONT_DROPDEAD: keeping {len(keep)} of {nr_} rows", flush=True) T = T[keep]; nr_ = len(keep) # ---- positivity (sparse) ---- from scipy.sparse import coo_matrix, csr_matrix as _csr GCAP = float(os.environ.get('CONT_GCAP', '50')); MSUB = int(os.environ.get('CONT_MSUB', '1')) lb = np.full(nv, -np.inf); ub = np.full(nv, np.inf); lb[:nc] = -1.0; ub[:nc] = PHIMAX nvac = 0 for (j, tau, m), col in hid.items(): lb[col] = -GCAP; ub[col] = GCAP # rec6 (2): the H columns of the interval of Lemma 8.1 are free too for col in gid.values(): lb[col] = -GCAP; ub[col] = GCAP if os.environ.get('CONT_ECAP'): # rec6e diagnostic: cap the constant of E's (rough, rough) reading ub[cid[(0, 2, NB)]] = float(os.environ['CONT_ECAP']); print(f" CONT_ECAP: c(0, 2, {NB}) <= {ub[cid[(0, 2, NB)]]}", flush=True) if os.environ.get('CONT_NOGREC'): for col in gid.values(): lb[col] = 0.0; ub[col] = 0.0 print(f" CONT_NOGREC: {len(set(gid.values()))} G columns fixed to 0", flush=True) if os.environ.get('CONT_LBC'): sys.exit('cont_hms_rec4: CONT_LBC is not supported (focused read M1)') LBC = os.environ.get('CONT_LBC'); low_cls = [] if LBC is not None: LBC = float(LBC) for cl_ in classes: if cl_[0] >= 1 and (cl_[1] >= 2 or (cl_[1] == 1 and cl_[0] >= 2 and not os.environ.get('CONT_NOGREC'))): lb[cid[cl_]] = LBC; low_cls.append(cl_) hid_arr = -np.ones((MS + 1, na, na), dtype=int) for (j, tau, m), col in hid.items(): hid_arr[j, tau, m] = col gid_arr = -np.ones((JG + 1, N + 1, na), dtype=int) for (j, tg, m), col in gid.items(): gid_arr[j, tg, m] = col # cumulative kernels as arrays over the maximum: LJc[jy][m, tau_y, a], LGc[jy][m, tau_y, a], LASc[jy][m, a] mi = np.array(mgrid); NM = len(mgrid) LJc = {jy: np.array([RK[m]['LJ'][jy] for m in mgrid]) for jy in range(1, MS + 1)} LGc = {jy: np.array([RK[m]['LG'][jy] for m in mgrid]) for jy in range(1, JG + 1)} LASc = {jy: np.array([RK[m]['LAS'][jy] for m in mgrid]) for jy in range(2, JG + 1)} def exact_from(cum, mb): """rows over M = mb..ia of the 'maximum exactly M, or <= mb at M = mb' kernel: cum has a leading axis over mgrid (18:0x: callers now pass cum already sliced at the a-index, so this copies small arrays only)""" k0 = mb - ie out = cum[k0:].copy() out[1:] -= cum[k0:-1] return out # out[0] = cum(<= mb); out[i] = cum(<= mb+i) - cum(<= mb+i-1) LJa = {jy: np.ascontiguousarray(np.moveaxis(LJc[jy], 2, 0)) for jy in LJc} # [a][m, tau_y] LGa = {jy: np.ascontiguousarray(np.moveaxis(LGc[jy], 2, 0)) for jy in LGc} LASa = {jy: np.ascontiguousarray(LASc[jy].T) for jy in LASc} # [a][m] from array import array as _arr Rr, Rc, Rv = _arr('q'), _arr('q'), _arr('d'); nrow = 0; desc = [] def emit(entries): global nrow for c_, v_ in entries.items(): if v_ != 0.0: Rr.append(nrow); Rc.append(c_); Rv.append(v_) nrow += 1 def emit_arrays(base_dict, cols, vals): d = dict(base_dict) for c_, v_ in zip(cols, vals): d[c_] = d.get(c_, 0.0) + v_ emit(d) # free rows for cl in classes: j, k, b = cl if not in_sup(support(j, k, b), 1.0) or j == 0 or k == 0: continue supR = support(0, k, b) for (jj, tau, m), col in hid.items(): if jj == j and in_sup(supR, 1 - tau * h) and lb[col] != ub[col]: emit({cid[cl]: 1.0, col: 1.0}); desc.append(('free', cl)) if k == 1: seen = set() for (jj, tg, m), col in gid.items(): if jj == j and in_sup(supR, 1 - tg * h) and col not in seen: seen.add(col); emit({cid[cl]: 1.0, col: 1.0}); desc.append(('freeG', cl)) # every free grid shape; classes with a shape carrying no active added term keep c >= -1 _fs_cls, _fs_h, _fs_g = [], [], [] def _parts(total, lower, prefix): if total == 0: yield prefix; return for p_ in range(lower, min(ihf - 1, total) + 1): if total == p_ or total - p_ >= p_: yield from _parts(total - p_, p_, prefix + (p_,)) _pure = set() for _sh in _parts(N, ie, ()): _sm = [q for q in _sh if q <= ia]; _ro = [q for q in _sh if q > ia] _j, _k, _tau = len(_sm), len(_ro), sum(_sm) _b = 0 if not _ro else next(i_ for i_ in range(1, len(iedges)) if max(_ro) <= iedges[i_]) _cl = (_j, _k, _b) if _cl not in cid: continue _hc = hid.get((_j, _tau, max(_sm))) if (_sm and _tau <= ia) else None _gc = gid.get((_j, _tau, max(_sm))) if (_sm and _k == 1) else None _act = [cc_ for cc_ in (_hc, _gc) if cc_ is not None and not (lb[cc_] == 0 and ub[cc_] == 0)] if not _act: _pure.add(_cl) _fs_cls.append(cid[_cl]); _fs_h.append(_hc if _hc is not None else -1); _fs_g.append(_gc if _gc is not None else -1) _fs_cls = np.array(_fs_cls); _fs_h = np.array(_fs_h); _fs_g = np.array(_fs_g) _restored = [] for _cl in sorted(_pure): if lb[cid[_cl]] < -1: lb[cid[_cl]] = -1.0; _restored.append(_cl) _cn_cls, _cn_h, _cn_g, _cn_bare = [], [], [], [] # rec4 (3): closed-interval free nodes, as band() weights them for _cl in classes: _j, _k, _b = _cl if _k == 0: _taus = [N] if (_j >= 1 and _j * ie <= N <= _j * ia) else [] else: _sR = support(0, _k, _b); _taus = [t_ for t_ in range(_j * ie, min(_j * ia, N) + 1) if in_sup(_sR, 1 - t_ * h)] if _j >= 1 else ([0] if in_sup(_sR, 1.0) else []) for _t in _taus: for _m in (feas_max(_j, _t) if _j >= 1 else [None]): _hc = hid.get((_j, _t, _m)) if (_j >= 1 and _t <= ia) else None _gc = gid.get((_j, _t, _m)) if (_j >= 1 and _k == 1) else None _hc = _hc if (_hc is not None and not (lb[_hc] == 0 and ub[_hc] == 0)) else None _gc = _gc if (_gc is not None and not (lb[_gc] == 0 and ub[_gc] == 0)) else None _cn_cls.append(cid[_cl]); _cn_h.append(-1 if _hc is None else _hc); _cn_g.append(-1 if _gc is None else _gc) _cn_bare.append(_hc is None and _gc is None) _cn_cls = np.array(_cn_cls); _cn_h = np.array(_cn_h); _cn_g = np.array(_cn_g); _cn_bare = np.array(_cn_bare) print(f" REC4: {len(_cn_cls)} closed-interval free nodes (class, tau, m), {int(_cn_bare.sum())} bare (constant only); constants lower bound min {lb[:nc].min():g}", flush=True) print(f" REC3: {len(_fs_cls)} free grid shapes; classes with a pure-constant free shape: {len(_pure)}; lower bound restored to -1 on {len(_restored)}: {_restored}", flush=True) npure = 0 if LBC is not None: for cl_ in low_cls: j_, k_, b_ = cl_ if k_ != 1: continue missing = False for rho in range(max(iedges[b_ - 1], ia + 1), min(iedges[b_], ihf - 1) + 1): tg = N - rho if not (j_ * ie <= tg <= j_ * ia): continue for m_ in feas_max(j_, tg): if (j_, tg, m_) not in gid: missing = True; break if missing: break if missing: emit({cid[cl_]: 1.0}); desc.append(('pure_c', cl_)); npure += 1 print(f" CONT_LBC = {LBC}: lowered {len(low_cls)} class constants; restored c >= -1 rows: {npure}", flush=True) nfree = nrow alist = list(range(ihf, N, ASUB)) for bc in w_classes: jb, kb, bb = bc if bc == (0, 0, 0): continue supR = support(0, kb, bb) for ai in alist: a = ai * h; tb = N - ai if not in_sup(support(jb, kb, bb), tb * h): continue base = {} for key, L in Lam.items(): jy, ky, bp = key; tgt = cid.get((jb + jy, kb + ky, max(bb, bp))) if tgt is not None: base[tgt] = base.get(tgt, 0.0) + a * L[ai] if jb == 0: cols = []; vals = [] for jy in range(1, MS + 1): # fragments carry the whole small part: H(jy, tau_y, M), M = their maximum ex = exact_from(LJa[jy][ai], ie) # (NM, na): maximum exactly M for k0, M in enumerate(mgrid): cc = hid_arr[jy, :, M]; ok = cc >= 0 if ok.any(): cols += list(cc[ok]); vals += list(h * a * ex[k0][ok]) if kb == 1: for jy in range(2, JG + 1): ex = exact_from(LASa[jy][ai], ie) for k0, M in enumerate(mgrid): cg = gid_arr[jy, ai, M] if ai <= N else -1 if cg >= 0: cols.append(cg); vals.append(a * ex[k0]) if bc == (0, 1, NB) and ai == ihf: continue # rec6 (1): no positivity row at E = (1/2, 1/2) emit_arrays(base, cols, vals); desc.append(('ext', bc)); continue if kb: cands = [t_ for t_ in range(jb * ie, min(jb * ia, tb) + 1, SSUB) if in_sup(supR, (tb - t_) * h)] else: cands = [tb] if jb * ie <= tb <= jb * ia else [] done_const = False for tbeta in cands: mbs = feas_max(jb, tbeta) if tbeta <= ia else [None] if tbeta > ia and kb >= 2: if not done_const: emit(dict(base)); desc.append(('ext', bc)); done_const = True continue if tbeta > ia: mbs = feas_max(jb, tbeta) for mb in mbs[::MSUB]: cols = []; vals = [] if tbeta <= ia: c0 = hid_arr[jb, tbeta, mb] if jb <= MS else -1 if c0 >= 0: cols.append(c0); vals.append(a * Lam0[ai]) for jy in range(1, MS - jb + 1): ex = exact_from(LJa[jy][ai], mb) # (ia - mb + 1, na) for k0 in range(ex.shape[0]): M = mb + k0; taus_y = np.arange(jy * ie, ia - tbeta + 1) if len(taus_y) == 0: continue cc = hid_arr[jb + jy, tbeta + taus_y, M]; ok = cc >= 0 if ok.any(): cols += list(cc[ok]); vals += list(h * a * ex[k0][taus_y[ok]]) if kb == 0: for jy in range(1, JG - jb + 1): ex = exact_from(LGa[jy][ai], mb) taus_y = np.arange(jy * ie, ng); tt_ = tbeta + taus_y; okt = tt_ <= N taus_y, tt_ = taus_y[okt], tt_[okt] for k0 in range(ex.shape[0]): M = mb + k0; cc = gid_arr[jb + jy, tt_, M]; ok = cc >= 0 if ok.any(): cols += list(cc[ok]); vals += list(h * a * ex[k0][taus_y[ok]]) elif kb == 1: for jy in range(2, JG - jb + 1): ex = exact_from(LASa[jy][ai], mb) tt_ = tbeta + ai if tt_ > N: continue for k0 in range(len(ex)): cg = gid_arr[jb + jy, tt_, mb + k0] if cg >= 0: cols.append(cg); vals.append(a * ex[k0]) emit_arrays(base, cols, vals); desc.append(('ext', bc)) Gm = coo_matrix((Rv, (Rr, Rc)), shape=(nrow, nv)).tocsr() print(f"[{time.time()-t0:.1f}s] positivity rows: free {nfree}, extended {nrow - nfree}; nnz {Gm.nnz}; vacuous H fixed {nvac}; |L| <= {GCAP}", flush=True) # ---- control: tie L(j, tau, m) across m (must reproduce cont_hmg) ---- Aeq = _csr(T) if os.environ.get('CONT_TIEM'): tr, tc, tv = [], [], []; nt = 0 for store in (hid, gid): groups = {} for (j, tau, m), col in store.items(): groups.setdefault((j, tau), []).append(col) for cols in groups.values(): cols = sorted(set(cols)) for c1, c2 in zip(cols[:-1], cols[1:]): tr += [nt, nt]; tc += [c1, c2]; tv += [1.0, -1.0]; nt += 1 Aeq = _csr(np.vstack([T, coo_matrix((tv, (tr, tc)), shape=(nt, nv)).toarray()])) if nt else Aeq print(f" CONT_TIEM: {nt} tie rows", flush=True) NOUP = bool(os.environ.get('CONT_NOUPPER')) from scipy.sparse import vstack as _svs Aub = -Gm if NOUP else _svs([-Gm, Gm]).tocsr(); bub = np.ones(nrow) if NOUP else np.concatenate([np.ones(nrow), np.full(nrow, PHIMAX)]) _RAW = dict(T=T.copy(), f1=f1.copy(), Gm=Gm.copy(), TDEAD=(TDEAD.copy() if TDEAD is not None else None)) TINY = float(os.environ.get('CONT_TINY', '1e-9')) # remove coefficients below CONT_TINY from T, the dead rows, the positivity rows and the objective; this perturbs the rows and the objective, and the residuals reported below refer to the modified matrices T[np.abs(T) < TINY] = 0.0; f1[np.abs(f1) < TINY] = 0.0 Gm.data[np.abs(Gm.data) < TINY] = 0.0; Gm.eliminate_zeros() Aeq = _csr(T) if not os.environ.get('CONT_TIEM') else Aeq if os.environ.get('CONT_TIEM'): Aeq = Aeq.tocsr(); Aeq.data[np.abs(Aeq.data) < TINY] = 0.0; Aeq.eliminate_zeros() Aub = -Gm if NOUP else _svs([-Gm, Gm]).tocsr() if TDEAD is not None: TDEAD[np.abs(TDEAD) < TINY] = 0.0; epsd = float(os.environ['CONT_DEADEPS']) DS = float(os.environ.get('CONT_DEADSCALE', '1')) Aub = _svs([Aub, _csr(DS * TDEAD), _csr(-DS * TDEAD)]).tocsr(); bub = np.concatenate([bub, np.full(2 * TDEAD.shape[0], DS * epsd)]) print(f" sanity: finite T {np.isfinite(T).all()}, f1 {np.isfinite(f1).all()}, G {np.isfinite(Gm.data).all()}; max|T| {np.abs(T).max():.3e}, max|G| {np.abs(Gm.data).max():.3e}, max|f1| {np.abs(f1).max():.3e}; min|nonzero T| {np.abs(T[T != 0]).min():.3e}", flush=True) if os.environ.get('CONT_FEASCHECK'): r0 = linprog(np.zeros(nv), A_ub=Aub, b_ub=bub, A_eq=Aeq, b_eq=np.zeros(Aeq.shape[0]), bounds=np.column_stack([lb, ub]), method='highs', options={'presolve': True}) print(f" feasibility LP (zero objective): status {r0.status} ({r0.message})", flush=True) if os.environ.get('CONT_NOX'): # bundled HiGHS, IPM, run_crossover off; primal value + residuals + a float dual bound from scipy.optimize._highspy import _core as _hc from scipy.sparse import vstack as _vs2 from types import SimpleNamespace as _NS hh_ = _hc._Highs() for k_, v_ in (('output_flag', bool(os.environ.get('CONT_DISP'))), ('solver', 'ipm'), ('run_crossover', 'off'), ('presolve', 'on' if int(os.environ.get('CONT_PRESOLVE', '0')) else 'off'), ('ipm_optimality_tolerance', float(os.environ.get('CONT_OTOL', '1e-8'))), ('primal_feasibility_tolerance', float(os.environ.get('CONT_PTOL', '1e-7'))), ('dual_feasibility_tolerance', float(os.environ.get('CONT_PTOL', '1e-7'))), ('time_limit', float(os.environ.get('CONT_TLIM', '1e9')))): hh_.setOptionValue(k_, v_) INF_ = hh_.getInfinity() A_ = _vs2([Aeq, Aub]).tocsc(); A_.sort_indices(); neq_ = Aeq.shape[0]; m_ = A_.shape[0] rl_ = np.concatenate([np.zeros(neq_), np.full(Aub.shape[0], -INF_)]); ru_ = np.concatenate([np.zeros(neq_), np.asarray(bub, dtype=np.float64)]) hh_.addRows(m_, rl_, ru_, 0, np.zeros(m_, dtype=np.int32), np.zeros(0, dtype=np.int32), np.zeros(0)) lbh_ = np.where(np.isinf(lb), -INF_, lb); ubh_ = np.where(np.isinf(ub), INF_, ub) hh_.addCols(nv, np.asarray(f1, dtype=np.float64), lbh_, ubh_, A_.nnz, A_.indptr[:-1].astype(np.int32), A_.indices.astype(np.int32), A_.data.astype(np.float64)) print(f"[{time.time()-t0:.1f}s] NOX: model {hh_.getNumRow()} rows x {hh_.getNumCol()} cols, nnz {hh_.getNumNz()}; running IPM (no crossover)", flush=True) hh_.run() st_ = hh_.modelStatusToString(hh_.getModelStatus()); info_ = hh_.getInfo(); sol_ = hh_.getSolution() xx_ = np.asarray(sol_.col_value, dtype=np.float64); yy_ = np.asarray(sol_.row_dual, dtype=np.float64) pv_ = float(f1 @ xx_) if len(xx_) == nv else float('nan') if len(xx_) == nv: eqr_ = float(np.abs(Aeq @ xx_).max()) if neq_ else 0.0; ubv_ = float(np.maximum(Aub @ xx_ - bub, 0).max()) bdv_ = float(max(np.maximum(lb - xx_, 0).max(), np.maximum(xx_ - ub, 0).max())) else: eqr_ = ubv_ = bdv_ = float('nan') # float weak-duality bound: y free on equality rows, y <= 0 on the <= rows (clipped), d = c - A^T y, every column boxed db_ = float('nan') if len(yy_) == m_: y_ = yy_.copy(); y_[neq_:] = np.minimum(y_[neq_:], 0.0) d_ = np.asarray(f1, dtype=np.float64) - A_.T @ y_ if np.isfinite(lb).all() and np.isfinite(ub).all(): db_ = float(y_[neq_:] @ np.asarray(bub, dtype=np.float64) + np.sum(np.where(d_ >= 0, d_ * lb, d_ * ub))) print(f" NOX: status {st_}; ipm iters {info_.ipm_iteration_count}, crossover iters {info_.crossover_iteration_count}; HiGHS objective " f"{info_.objective_function_value:.9f}; primal value f1.x = {pv_:.9f} (max |Aeq x| {eqr_:.1e}, max ub violation {ubv_:.1e}, " f"max bound violation {bdv_:.1e}); float dual bound {db_:.9f}; HiGHS max primal/dual infeas {info_.max_primal_infeasibility:.1e} / " f"{info_.max_dual_infeasibility:.1e}", flush=True) if os.environ.get('CONT_SAVEX'): np.save(os.environ['CONT_SAVEX'], xx_) if len(xx_) == nv and len(yy_) == m_: # residuals, acceptance, net-multiplier dual bounds (pruned and raw) DS_ = float(os.environ.get('CONT_DEADSCALE', '1')); epsd_ = float(os.environ.get('CONT_DEADEPS', '0')) nd_ = TDEAD.shape[0] if TDEAD is not None else 0; npos_ = Gm.shape[0] live_p = float(np.abs(Aeq @ xx_).max()) if neq_ else 0.0 dead_p = float(np.abs(TDEAD @ xx_).max()) if nd_ else 0.0 pos_p = float((Gm @ xx_).min()); boxv = float(max(np.maximum(lb - xx_, 0).max(), np.maximum(xx_ - ub, 0).max())) live_r = float(max(np.abs(_RAW['T'] @ xx_).max(), (np.abs(Aeq[_RAW['T'].shape[0]:] @ xx_).max() if neq_ > _RAW['T'].shape[0] else 0.0))) if neq_ else 0.0 dead_r = float(np.abs(_RAW['TDEAD'] @ xx_).max()) if nd_ else 0.0 pos_r = float((_RAW['Gm'] @ xx_).min()); obj_r = float(_RAW['f1'] @ xx_) acc_ = (st_ == 'Optimal') and (live_p <= 1e-7) and (dead_p <= epsd_ + 1e-10) and (pos_p >= -1 - 1e-7) and (boxv <= 1e-7) y_ = yy_.copy(); y_[neq_:neq_ + npos_] = np.minimum(y_[neq_:neq_ + npos_], 0.0) if nd_: yp_, ym_ = y_[neq_ + npos_:neq_ + npos_ + nd_].copy(), y_[neq_ + npos_ + nd_:].copy(); lam_ = yp_ - ym_ y_[neq_ + npos_:neq_ + npos_ + nd_] = np.minimum(lam_, 0.0); y_[neq_ + npos_ + nd_:] = np.minimum(-lam_, 0.0) bfull_ = np.concatenate([np.zeros(neq_), np.asarray(bub, dtype=np.float64)]) def _bound(Amat, cvec): d__ = np.asarray(cvec, dtype=np.float64) - Amat.T @ y_ terms = np.where(d__ >= 0, d__ * lb, d__ * ub); val = float(bfull_ @ y_ + terms.sum()) allow = 1e-13 * (float(np.abs(bfull_ * y_).sum()) + float(np.abs(terms).sum())) return val - allow Lp_ = _bound(A_, f1) nT_ = _RAW['T'].shape[0] parts = [_csr(_RAW['T'])] + ([Aeq[nT_:]] if neq_ > nT_ else []) + [-_RAW['Gm']] + ([_csr(DS_ * _RAW['TDEAD']), _csr(-DS_ * _RAW['TDEAD'])] if nd_ else []) Lr_ = _bound(_vs2(parts).tocsr(), _RAW['f1']) _fv = xx_[_fs_cls] + np.where(_fs_h >= 0, xx_[np.maximum(_fs_h, 0)], 0.0) + np.where(_fs_g >= 0, xx_[np.maximum(_fs_g, 0)], 0.0) _fvc = xx_[_cn_cls] + np.where(_cn_h >= 0, xx_[np.maximum(_cn_h, 0)], 0.0) + np.where(_cn_g >= 0, xx_[np.maximum(_cn_g, 0)], 0.0) print(f" FREEMIN_CLOSED: min f over {len(_fvc)} closed-interval free nodes = {float(_fvc.min()):.12f}; over bare nodes = {float(_fvc[_cn_bare].min()) if _cn_bare.any() else float('nan'):.12f}", flush=True) print(f" FREEMIN: min f over all {len(_fs_cls)} free grid shapes = {float(_fv.min()):.12f}", flush=True) print(f" REC: status {st_}; objective pruned {pv_:.9f} raw {obj_r:.9f}; float dual lower bound pruned {Lp_:.9f} raw {Lr_:.9f}; " f"pruned residuals live {live_p:.2e} dead {dead_p:.2e} (eps {epsd_:g}, scale {DS_:g}) min positivity {pos_p:.12f} box viol {boxv:.1e}; " f"raw residuals live {live_r:.2e} dead {dead_r:.2e} min positivity {pos_r:.12f}; ACCEPT {acc_}; rows: live {neq_}, positivity {npos_}, dead {nd_}", flush=True) _cE = cid[(0, 2, NB)]; print(f" EOPT: c(0, 2, {NB}) = {xx_[_cE]:.9f}; T0 x = {float(T[0] @ xx_):.3e}; T0' x (without that term) = {float(T[0] @ xx_ - T[0, _cE] * xx_[_cE]):.9f}", flush=True) if os.environ.get('CONT_SAVEX'): np.savez_compressed(os.environ['CONT_SAVEX'] + '.rec.npz', x=xx_, y=yy_, lb=lb, ub=ub) res = _NS(status=0 if st_ == 'Optimal' else 1, message=f'NOX {st_}', x=xx_ if st_ == 'Optimal' else None, fun=pv_ if st_ == 'Optimal' else None, nit=info_.ipm_iteration_count) else: res = linprog(f1, A_ub=Aub, b_ub=bub, A_eq=Aeq, b_eq=np.zeros(Aeq.shape[0]), bounds=np.column_stack([lb, ub]), method=os.environ.get('CONT_METHOD', 'highs-ipm'), options=dict(presolve=bool(int(os.environ.get('CONT_PRESOLVE', '0'))), **({'primal_feasibility_tolerance': float(os.environ['CONT_PTOL']), 'dual_feasibility_tolerance': float(os.environ['CONT_PTOL'])} if os.environ.get('CONT_PTOL') else {}), **({'ipm_optimality_tolerance': float(os.environ['CONT_OTOL'])} if os.environ.get('CONT_OTOL') else {}), **({'time_limit': float(os.environ['CONT_TLIM'])} if os.environ.get('CONT_TLIM') else {}), **({'disp': True} if os.environ.get('CONT_DISP') else {}))) print(f"[{time.time()-t0:.1f}s] LP status {res.status} ({res.message})", flush=True) print(f" result: fun {getattr(res, 'fun', None)}; nit {getattr(res, 'nit', None)}", flush=True) # night2 if res.status != 0 and getattr(res, 'x', None) is not None: # report the returned nonoptimal point and its residuals; feasibility is not asserted xx_ = np.asarray(res.x); print(f" NONOPT POINT: f1.x = {float(f1 @ xx_):.6f}; max eq residual {float(np.abs(Aeq @ xx_).max()):.2e}; max ub violation {float(np.maximum(Aub @ xx_ - bub, 0).max()):.2e}", flush=True) np.save(os.environ.get('CONT_SAVEX', 'nonopt_x.npy'), xx_) if res.status == 0: x = res.x; z = f1 @ x print(f" min f(1) = {z:.6f} -> {'PRIME-FREE after rescaling, margin %.4f' % (1 - 1/abs(z)) if z < -1 else 'NOT prime-free, C- >= %.6f' % (1 + z)}") hv = x[Hc]; gv = x[Gc] if len(Gc) else np.zeros(1) print(f" H in [{hv.min():.3f}, {hv.max():.3f}] (at cap {int((np.abs(hv) > GCAP - 1e-6).sum())}); G in [{gv.min():.3f}, {gv.max():.3f}] (at cap {int((np.abs(gv) > GCAP - 1e-6).sum())}); |T x| {np.abs(T @ x).max():.1e}") ~~~~~~~~~~~~ #### file: cont_hms_rec6e2.py - kind: script - as run: SHA-256 36e621a1c6d01aa4e4125157716201d8bc82786f384cd73df611adeb07841272 - as shipped: SHA-256 b92ac6f7ac611c38dcf42bad03526cee1f95ce1b348f65955eba73dd488070d8 - identity: identical, after dropping docstrings and comments, to the file as run - trailing newline: yes ~~~~~~~~~~~~ python # cont_hms_rec6e2.py: cont_hms_rec6e.py plus a lower bound for the constant at E; see its docstring. """cont_hms_rec6e2.py: cont_hms_rec6e.py plus CONT_ELO=v, a lower bound for the same constant; with CONT_ECAP=0 and CONT_ELO=0 it fixes that constant at 0 (the efix0 solves in ediag.out). Floats; not a certificate.""" import sys, time, os, pickle from fractions import Fraction as Fr import numpy as np from scipy.signal import fftconvolve from scipy.optimize import linprog from scipy.sparse import csr_matrix t0 = time.time() N = int(sys.argv[1]); eta = Fr(int(sys.argv[2]), int(sys.argv[3])); alpha = Fr(int(sys.argv[4]), int(sys.argv[5])) EDG = [Fr(e) for e in sys.argv[6].split(',')] if sys.argv[6] else [] PHIMAX = float(sys.argv[7]); ASUB = int(sys.argv[8]); SSUB = int(sys.argv[9]) assert eta <= alpha < Fr(1, 2) h = 1.0 / N def idx(x): v = x * N; assert v.denominator == 1, f"{x} not on grid N={N}"; return int(v) ie, ia, ihf = idx(eta), idx(alpha), idx(Fr(1, 2)) edges = [alpha] + EDG + [Fr(1, 2)]; iedges = [idx(e) for e in edges]; NB = len(edges) - 1 assert all(edges[i_] < edges[i_ + 1] for i_ in range(len(edges) - 1)), 'bin edges must be strictly increasing between alpha and 1/2' inv = np.zeros(N + 1); inv[1:] = 1.0 / (np.arange(1, N + 1) * h) def conv(F, G): return fftconvolve(F, G)[:N + 1] * h delta = np.zeros(N + 1); delta[0] = 1.0 / h HC = os.environ.get('CONT_HALFCLOSE', 'trap'); HCF = {'trap': 0.5, 'closed': 1.0}[HC] # rec5: weight of a boundary at exactly 1/2 SELFTEST = bool(os.environ.get('CONT_GATE_SELFTEST')) # rec5: positive control of the gate (emulates the old a = 1/2 defect) def band(lo, hi): F = np.zeros(N + 1); F[lo:hi + 1] = inv[lo:hi + 1]; F[lo] *= 0.5; F[hi] *= 0.5; return F def trunc_ge(F, i0): G = F.copy(); G[:max(i0, 0)] = 0.0 if 0 <= i0 <= N: G[i0] *= HCF return G def shift(F, i): G = np.zeros(N + 1); G[i:] = F[:N + 1 - i]; return G s_den = band(ie, ia) JMAX = int(1 / eta) + 1; KMAX = int(1 / alpha) + 1 S = [delta] for j in range(1, JMAX + 1): S.append(conv(S[-1], s_den) / j) S2 = S[2] Rcap = {} for e in iedges[1:]: r = band(ia, e) if HC == 'closed' and e == ihf: r[ihf] = 0.0 # rec5 closed: a component exactly 1/2 is large P = [delta] for k in range(1, KMAX + 1): P.append(conv(P[-1], r) / k) Rcap[e] = P def Rbin(k, b): if k == 0: return delta hi = iedges[b] return Rcap[hi][k] if b == 1 else Rcap[hi][k] - Rcap[iedges[b - 1]][k] Rall = Rcap[ihf] bins_k = lambda k: [0] if k == 0 else list(range(1, NB + 1)) etaf, alf, ef = float(eta), float(alpha), [float(e) for e in edges] def support(j, k, b): if k == 0: return (j * etaf, j * alf) return (j * etaf + (k - 1) * alf + ef[b - 1], j * alf + k * ef[b]) def in_sup(sup, t): return sup[1] > sup[0] and sup[0] - 1e-12 <= t <= sup[1] + 1e-12 print(f"[{time.time()-t0:.1f}s] N={N} eta={eta} alpha={alpha} edges={[str(e) for e in edges]}", flush=True) # ---- Lemma K kernels (c-part) ---- PS = {} for e in iedges[1:]: for bs in range(JMAX + 1): for br in range(KMAX + 1): PS[(e, bs, br)] = conv(S[bs], Rcap[e][br]) if br else S[bs] def Lam_cap(jy, ky, e): K = jy + ky; L = -PS[(e, jy, ky)].copy(); acc = np.zeros(N + 1) for bs in range(jy + 1): for br in range(ky + 1): if (bs, br) in ((0, 0), (jy, ky)): continue acc += (-1) ** (bs + br) * conv(trunc_ge(PS[(e, bs, br)], ihf), PS[(e, jy - bs, ky - br)]) return L + (-1) ** (K + 1) * acc Lam = {} for jy in range(JMAX + 1): for ky in range(KMAX + 1): if jy + ky < 2: continue if ky == 0: Lam[(jy, 0, 0)] = Lam_cap(jy, 0, ihf); continue prev = None for b in range(1, NB + 1): cur = Lam_cap(jy, ky, iedges[b]); Lam[(jy, ky, b)] = cur if prev is None else cur - prev; prev = cur Lam0 = sum(Lam_cap(0, ky, ihf) for ky in range(2, KMAX + 1)) del PS # ---- XS(t; theta) for theta = 1/2 - x, x in [0, ia] ---- XS = {} for x in range(0, ia + 1): acc = np.zeros(N + 1) for k in range(1, KMAX + 1): for br in range(1, k + 1): acc += (-1) ** (k + br) * conv(trunc_ge(Rall[br], ihf - x), Rall[k - br]) XS[x] = acc # ======================= cont_hms ======================= MS = ia // ie while MS * ie > ia: MS -= 1 Hgrid = {j: list(range(j * ie, ia + 1)) for j in range(1, MS + 1)} Rall = Rcap[ihf] Rsgn = sum(((-1) ** k) * Rall[k] for k in range(0, KMAX + 1)); Runs = sum(Rall[k] for k in range(0, KMAX + 1)) na = ia + 1; ng = N - ia + 1 JG = min(JMAX, (N - ia) // ie) R1 = Rall[1] XSmat = np.array([XS[x] for x in range(0, ia + 1)]) XS1 = np.zeros((ng, N + 1)) for sg_ in range(0, min(ng, ihf)): XS1[sg_] = trunc_ge(R1, ihf - sg_) if ihf < ng: XS1[ihf] = (1.0 - HCF) * R1 # small sub-sum exactly 1/2: S and S + r cancel in the weight HCF tr_mask = (np.arange(N + 1) >= ihf).astype(float); tr_mask[ihf] = HCF Lam0Tr = trunc_ge(Lam0, ihf) K0sgn = conv(Rsgn, Lam0Tr); K0uns = conv(Runs, Lam0Tr) mgrid = list(range(ie, ia + 1)) JS = max(JMAX, JG) def restricted(m): """kernels with every EXTRA small part <= m (grid index)""" sm = s_den.copy(); sm[m + 1:] = 0.0 Sm = [delta] for j in range(1, JS + 1): Sm.append(conv(Sm[-1], sm) / j) w = h * sm # M_j over sigma, tau in [0, ng) Mj = [None] * (JG + 1); Mj[0] = np.zeros((ng, ng)); Mj[0][0, 0] = 1.0 / h for j in range(1, JG + 1): acc = np.zeros((ng, ng)) for t in range(ie, m + 1): if w[t] == 0.0: continue acc[:, t:] += w[t] * Mj[j - 1][:, :ng - t] acc[t:, t:] -= w[t] * Mj[j - 1][:ng - t, :ng - t] Mj[j] = acc / j LJ = {}; Ks = {}; Ku = {} for j in range(1, MS + 1): A = (Mj[j][:na, :na].T @ XSmat) * ((-1) ** (j - 1)) L = np.zeros((na, N + 1)) for tau in Hgrid[j]: L[tau, tau:] = A[tau, :N + 1 - tau] if HC == 'trap': L[:, ihf] *= 2.0 # rec4 (1): right-limit value at a = 1/2, as Lam_cap if SELFTEST: L[:, ihf] *= 0.5 LJ[j] = L LT = L * tr_mask[None, :] Ks[j] = np.array([conv(Rsgn, LT[t]) for t in range(na)]); Ku[j] = np.array([conv(Runs, LT[t]) for t in range(na)]) LG = {}; LGT = {} for j in range(1, JG + 1): A = (Mj[j].T @ XS1) * ((-1) ** (j - 1)) L = np.zeros((ng, N + 1)) for tau in range(j * ie, ng): L[tau, tau:] = A[tau, :N + 1 - tau] if HC == 'trap': L[:, ihf] *= 2.0 # rec4 (1), G kernels if SELFTEST: L[:, ihf] *= 0.5 LG[j] = L; LGT[j] = L * tr_mask[None, :] LAS = {} for jy in range(2, JG + 1): acc = np.zeros(N + 1) for bs in range(1, jy): acc += (-1) ** bs * conv(trunc_ge(Sm[bs], ihf), Sm[jy - bs]) LAS[jy] = -Sm[jy] + ((-1) ** (jy + 1)) * acc # all-small Lemma K kernel, parts <= m (Lam_cap(jy, 0, .) with S restricted) LAST = {jy: trunc_ge(L, ihf) for jy, L in LAS.items()} return dict(S=Sm, LJ=LJ, Ks=Ks, Ku=Ku, LG=LG, LGT=LGT, LAS=LAS, LAST=LAST) RK = {} for m in mgrid: RK[m] = restricted(m) EMPTY = None print(f"[{time.time()-t0:.1f}s] restricted kernels for {len(mgrid)} maxima (MS {MS}, JG {JG})", flush=True) if '--kcheck' in sys.argv: top = RK[ia] print(" kcheck S_j(<= ia) vs S_j:", max(np.abs(top['S'][j] - S[j]).max() for j in range(1, min(JS, JMAX) + 1))) print(" kcheck LAS(<= ia) vs Lam[(jy,0,0)]:", max(np.abs(top['LAS'][jy] - Lam[(jy, 0, 0)]).max() for jy in top['LAS'] if (jy, 0, 0) in Lam)) def Sd(m, j, x): # density of j extra parts with total x and maximum EXACTLY m (vectorised in x) a = RK[m]['S'][j][x] return a - RK[m - 1]['S'][j][x] if m - 1 >= ie else a # ---- variables ---- classes = [(j, k, b) for j in range(JMAX + 1) for k in range(KMAX + 1) if j + k >= 1 for b in bins_k(k)] D = {cl: (conv(S[cl[0]], Rbin(cl[1], cl[2])) if cl[1] else S[cl[0]]) for cl in classes}; D[(0, 0, 0)] = delta cid = {cl: n for n, cl in enumerate(classes)}; nc = len(classes) def feas_max(j, tau): # maxima m with j parts of total tau and maximum exactly m: exact combinatorial test (17:5x; testing if j == 1: return [tau] if ie <= tau <= ia else [] # Sd > 0 admitted FFT round-off, giving impossible (tau, m) rows and variables) return [m for m in mgrid if m <= tau - (j - 1) * ie and j * m >= tau] hid = {}; gid = {}; nv = nc for j in range(1, MS + 1): for tau in Hgrid[j]: for m in feas_max(j, tau): hid[(j, tau, m)] = nv; nv += 1 GBAND = bool(os.environ.get('CONT_GBAND')) # restricted family: G resolved by m only on the dead band tau - m in [1-2a, a] glump = {} for j in range(2, JG + 1): for tg in range(ihf + 1, N - ia + 1): if S[j][tg] <= 0.0: continue for m in feas_max(j, tg): if GBAND and not (N - 2 * ia <= tg - m <= ia): if (j, tg) not in glump: glump[(j, tg)] = nv; nv += 1 gid[(j, tg, m)] = glump[(j, tg)] else: gid[(j, tg, m)] = nv; nv += 1 print(f" variables: {nc} constants, {len(hid)} H(j, tau, m), {len(gid)} G(j, tau, m) keys in {len(set(gid.values()))} columns" f"{' (GBAND: %d lumped)' % len(glump) if GBAND else ''}", flush=True) def new(): return np.zeros(nv) LamTr = {k_: trunc_ge(L, ihf) for k_, L in Lam.items()} w_classes = [(0, 0, 0)] + [cl for cl in classes if cl[0] + cl[1] >= 1 and in_sup(support(*cl), support(*cl)[0]) and support(*cl)[0] <= 0.5 + 1e-12] _cc = {} def conv_w_lam(wc, key): k_ = (wc, key) if k_ not in _cc: _cc[k_] = LamTr[key] if wc == (0, 0, 0) else conv(D[wc], LamTr[key]) return _cc[k_] # ---- f(1) ---- f1 = new() for key, L in Lam.items(): if (cl := cid.get(key)) is not None: f1[cl] += L[N] def dmax(m, key, j, idx): # RK[m][key][j][idx] - RK[m-1][key][j][idx] a = RK[m][key][j][idx] return a - RK[m - 1][key][j][idx] if m - 1 >= ie else a for (j, tau, m), col in hid.items(): f1[col] += h * dmax(m, 'LJ', j, (tau, N)) for (j, tg, m), col in gid.items(): f1[col] += h * dmax(m, 'LG', j, (tg, N)) # ---- rows (jD, sigma, mD) ---- row_list = [(0, 0, 0)] for jD in range(1, MS + 1): for s_ in Hgrid[jD]: for mD in feas_max(jD, s_): row_list.append((jD, s_, mD)) rid = {r: n for n, r in enumerate(row_list)}; nr_ = len(row_list) T = np.zeros((nr_, nv)); I = np.zeros((nr_, nv)); DIAG = np.zeros((nr_, nv), dtype=bool) # c-part (independent of mD): computed per (jD, sigma) as in cont_hm, copied to every mD cpart = {} for jD in range(0, MS + 1): sig = [0] if jD == 0 else Hgrid[jD]; tt = np.array([N - s_ for s_ in sig]) Tc = np.zeros((len(sig), nc)); Ic = np.zeros((len(sig), nc)) for cl in classes: jp, kp, bp = cl; tgt = cid.get((jD + jp, kp, bp)) if tgt is None: continue vals = D[cl][tt]; sg = (-1) ** (jD + jp + kp); Tc[:, tgt] += sg * vals; Ic[:, tgt] += vals for wc in w_classes: jw, kw, bw = wc; sgn = (-1) ** (1 + jD + jw + kw) for key in Lam: jy, ky, byy = key; tgt = cid.get((jD + jw + jy, kw + ky, max(bw, byy))) if tgt is None: continue v = conv_w_lam(wc, key)[tt]; Tc[:, tgt] += sgn * v; Ic[:, tgt] += v for n_, s_ in enumerate(sig): cpart[(jD, s_)] = (Tc[n_], Ic[n_]) for r, (jD, s_, mD) in enumerate(row_list): T[r, :nc] = cpart[(jD, s_)][0]; I[r, :nc] = cpart[(jD, s_)][1] print(f"[{time.time()-t0:.1f}s] c-part of {nr_} rows", flush=True) # extra-parts kernels per maximum m: KH[m][D][sigma, tau] (signed, jD-independent part) and KG[m][D][sigma, tg]; unsigned KHu, KGu def build_K(m): R = RK[m]; Sm = R['S'] KH = {}; KHu = {}; KG = {}; KGu = {} sig_all = np.arange(na) for Dl in range(1, MS + 1): Ht = np.zeros((na, na)); Hu = np.zeros((na, na)) for s_ in range(na): taus = np.arange(s_, na); x = taus - s_ v = h * Sm[Dl][x] * Rsgn[N - taus]; Ht[s_, taus] += ((-1) ** Dl) * v; Hu[s_, taus] += h * Sm[Dl][x] * Runs[N - taus] for jw in range(0, Dl + 1): jy = Dl - jw; sg = (-1) ** (1 + jw) if jw == 0: vt = h * R['Ks'][Dl][x, N - s_]; vu = h * R['Ku'][Dl][x, N - s_] elif jy == 0: vt = h * Sm[jw][x] * K0sgn[N - taus]; vu = h * Sm[jw][x] * K0uns[N - taus] else: vt = np.zeros(len(taus)); vu = np.zeros(len(taus)) for i_, t_ in enumerate(taus): vv = np.arange(s_, t_ + 1) vt[i_] = h * h * np.dot(Sm[jw][vv - s_], R['Ks'][jy][t_ - vv, N - vv]) vu[i_] = h * h * np.dot(Sm[jw][vv - s_], R['Ku'][jy][t_ - vv, N - vv]) Ht[s_, taus] += sg * vt; Hu[s_, taus] += vu KH[Dl] = Ht; KHu[Dl] = Hu gt = np.arange(ihf + 1, ng) for Dl in range(1, JG + 1): Gt = np.zeros((na, ng)); Gu = np.zeros((na, ng)) for s_ in range(na): x = gt - s_ v = h * Sm[Dl][x] * R1[N - gt]; Gt[s_, gt] += ((-1) ** (Dl + 1)) * v; Gu[s_, gt] += v for jw in range(0, Dl + 1): jy = Dl - jw if jy >= 1: if jw == 0: vt = h * R['LGT'][jy][x, N - s_] else: vt = np.zeros(len(gt)) for i_, t_ in enumerate(gt): vv = np.arange(s_, t_ + 1) vt[i_] = h * h * np.dot(Sm[jw][vv - s_], R['LGT'][jy][t_ - vv, N - vv]) Gt[s_, gt] += ((-1) ** (1 + jw)) * vt; Gu[s_, gt] += vt if jy >= 2: base = R['LAST'][jy] if jw == 0 else conv(Sm[jw], R['LAST'][jy]) vt = h * R1[N - gt] * base[x] Gt[s_, gt] += ((-1) ** jw) * vt; Gu[s_, gt] += vt KG[Dl] = Gt; KGu[Dl] = Gu return KH, KHu, KG, KGu KK = {m: build_K(m) for m in mgrid} print(f"[{time.time()-t0:.1f}s] extra-part kernels built", flush=True) def Kexact(m, which, Dl): # kernel with the extra parts' maximum EXACTLY m: K(<= m) - K(<= m-1) a = KK[m][which][Dl] return a - KK[m - 1][which][Dl] if m - 1 >= ie else a # assemble the H- and G-parts of every row for r, (jD, s_, mD) in enumerate(row_list): sgD = (-1) ** jD if jD >= 1: # diagonal: H_jD(sigma, mD), all-rough completions (free and extended) col = hid[(jD, s_, mD)] T[r, col] += sgD * (Rsgn[N - s_] - K0sgn[N - s_]); I[r, col] += Runs[N - s_] + K0uns[N - s_]; DIAG[r, col] = True for (J, tau, M), col in hid.items(): if J <= jD or tau < s_: continue Dl = J - jD if M < mD: continue if jD == 0: kt = Kexact(M, 0, Dl)[0, tau]; ku = Kexact(M, 1, Dl)[0, tau] elif M == mD: kt = KK[mD][0][Dl][s_, tau]; ku = KK[mD][1][Dl][s_, tau] else: kt = Kexact(M, 0, Dl)[s_, tau]; ku = Kexact(M, 1, Dl)[s_, tau] T[r, col] += sgD * kt; I[r, col] += ku for (J, tg, M), col in gid.items(): if J <= jD: continue Dl = J - jD if Dl > JG or M < mD: continue if jD == 0: kt = Kexact(M, 2, Dl)[0, tg]; ku = Kexact(M, 3, Dl)[0, tg] elif M == mD: kt = KK[mD][2][Dl][s_, tg]; ku = KK[mD][3][Dl][s_, tg] else: kt = Kexact(M, 2, Dl)[s_, tg]; ku = Kexact(M, 3, Dl)[s_, tg] T[r, col] += sgD * kt; I[r, col] += ku print(f"[{time.time()-t0:.1f}s] rows assembled", flush=True) _cE = cid[(0, 2, NB)] # rec6e: the constant of E = (1/2, 1/2) read as two rough components print(f" EINFO: class (0, 2, {NB}) column {_cE}; T0 coefficient {T[0, _cE]:.15g}; f1 coefficient {f1[_cE]:.3g}; other twisted rows with a nonzero entry {int((np.abs(T[1:, _cE]) > 0).sum())} (max {np.abs(T[1:, _cE]).max():.2e})", flush=True) if os.environ.get("CONT_EINFO_ONLY"): sys.exit(0) Hc = np.array(sorted(set(hid.values()))); Gc = np.array(sorted(set(gid.values())), dtype=int) offH = np.where(DIAG[:, Hc], 0.0, I[:, Hc]) print(f" Type I check: c-block {np.abs(I[:, :nc]).max():.3e}; H diagonal {np.abs(I[DIAG]).max():.3e}; H integrated/h {np.abs(offH).max()/h:.3e};" f" G/h {np.abs(I[:, Gc]).max()/h if len(Gc) else 0:.3e}; T0 row: c {np.abs(I[0, :nc]).max():.3e}, H/h {np.abs(I[0, Hc]).max()/h:.3e}," f" G/h {np.abs(I[0, Gc]).max()/h if len(Gc) else 0:.3e}", flush=True) _ti = dict(c=float(np.abs(I[:, :nc]).max()), Hd=float(np.abs(I[DIAG]).max()), Hi_h=float(np.abs(offH).max() / h), G_h=float(np.abs(I[:, Gc]).max() / h) if len(Gc) else 0.0) if HC == 'closed': _fail = [k_ for k_ in ('c', 'Hd', 'Hi_h', 'G_h') if not _ti[k_] <= 1e-9] print(f" GATE (closed-1/2 exactness): {'PASS' if not _fail else 'FAIL ' + str(_fail)} {_ti}", flush=True) if os.environ.get('CONT_GATEONLY'): sys.exit(3 if _fail else 0) else: _fail = [k_ for k_, ok_ in (('G_h', _ti['G_h'] <= 1e-9), ('c', _ti['c'] <= 10 * h), ('Hd', _ti['Hd'] <= 2 * h)) if not ok_] import subprocess as _sp _env = dict(os.environ, CONT_HALFCLOSE='closed', CONT_GATEONLY='1') _r = _sp.run([sys.executable, '-B', os.path.abspath(__file__)] + sys.argv[1:], env=_env, capture_output=True, text=True) _gl = [l_ for l_ in _r.stdout.splitlines() if 'GATE' in l_] if _r.returncode != 0: _fail.append('closed-1/2 exactness (exit %d): %s' % (_r.returncode, _gl[-1].strip() if _gl else _r.stderr[-300:])) print(f" GATE (trapezoid: G/h <= 1e-9, c <= 10h, H diag <= 2h; closed-1/2 exactness <= 1e-9): {'PASS' if not _fail else 'FAIL ' + str(_fail)}; " f"{_ti}; closed: {_gl[-1].strip() if _gl else 'no output'}", flush=True) if _fail and not os.environ.get('CONT_NOGATE'): sys.exit(f'cont_hms_rec5: GATE FAILED {_fail}') if os.environ.get('CONT_GATEONLY'): sys.exit(0) if '--check' in sys.argv: sys.exit() # ---- optional diagnostics ---- TDEAD = None if os.environ.get('DEAD_SVD'): # diagnostic: spectrum of the dead rows' G-block (and H-block size), then exit dead_ = [r for r, (jD_, s_, m_) in enumerate(row_list) if jD_ >= 1 and s_ >= N - 2 * ia + 2] gc_ = np.array(sorted(set(gid.values())), dtype=int) hc_ = np.array(sorted(set(hid.values())), dtype=int) Bd = T[np.ix_(dead_, gc_)] U_, sv, _ = np.linalg.svd(Bd, full_matrices=False) rk = int((sv > 1e-9 * sv[0]).sum()) NL = U_[:, rk:] # left null space of the dead G-block (dead-row combinations G cannot touch) Cd = T[np.ix_(dead_, np.arange(nc))] if NL.shape[1]: Mc = NL.T @ Cd; svc = np.linalg.svd(Mc, compute_uv=False) touched = int((np.abs(Cd).max(axis=0) > 1e-12).sum()) print(f" DEAD_NULL: G-rank {rk} of {len(dead_)} dead rows -> {NL.shape[1]} dead-row combinations G cannot touch; on the class constants they have rank " f"{(svc > 1e-9 * svc[0]).sum() if svc[0] > 0 else 0} (largest sv {svc[0]:.2e}); constants touched by dead rows {touched} of {nc}", flush=True) else: print(f" DEAD_NULL: G-block has full row rank {rk} = {len(dead_)}: no G-free dead-row combination", flush=True) ks = [k for k in (1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024) if k < len(sv)] print(f" DEAD_SVD s_max: dead rows {len(dead_)} x G cols {len(gc_)}; sv0 {sv[0]:.3e}; sv/sv0 at k={ks}: " + ", ".join(f"{sv[k]/sv[0]:.1e}" for k in ks) + f"; #(>1e-3 sv0) {(sv > 1e-3*sv[0]).sum()}, #(>1e-6) {(sv > 1e-6*sv[0]).sum()}, #(>1e-9) {(sv > 1e-9*sv[0]).sum()}; H-block max {np.abs(T[np.ix_(dead_, hc_)]).max():.2e}", flush=True) sys.exit(0) if os.environ.get('CONT_DEADEPS'): # relax the dead rows to |T_dead x| <= eps (they make the equality LP numerically degenerate) dead = [r for r, (jD_, s_, m_) in enumerate(row_list) if jD_ >= 1 and s_ >= N - 2 * ia] live = [r for r in range(nr_) if r not in set(dead)] TDEAD = T[dead].copy(); T = T[live]; nr_ = len(live) print(f" CONT_DEADEPS = {os.environ['CONT_DEADEPS']}: {len(dead)} dead rows relaxed to |T x| <= eps", flush=True) if os.environ.get('CONT_DROPDEAD'): keep = [r for r, (jD_, s_, m_) in enumerate(row_list) if jD_ == 0 or s_ < N - 2 * ia] print(f" CONT_DROPDEAD: keeping {len(keep)} of {nr_} rows", flush=True) T = T[keep]; nr_ = len(keep) # ---- positivity (sparse) ---- from scipy.sparse import coo_matrix, csr_matrix as _csr GCAP = float(os.environ.get('CONT_GCAP', '50')); MSUB = int(os.environ.get('CONT_MSUB', '1')) lb = np.full(nv, -np.inf); ub = np.full(nv, np.inf); lb[:nc] = -1.0; ub[:nc] = PHIMAX nvac = 0 for (j, tau, m), col in hid.items(): lb[col] = -GCAP; ub[col] = GCAP # rec6 (2): the H columns of the interval of Lemma 8.1 are free too for col in gid.values(): lb[col] = -GCAP; ub[col] = GCAP if os.environ.get('CONT_ECAP'): # rec6e diagnostic: cap the constant of E's (rough, rough) reading ub[cid[(0, 2, NB)]] = float(os.environ['CONT_ECAP']); print(f" CONT_ECAP: c(0, 2, {NB}) <= {ub[cid[(0, 2, NB)]]}", flush=True) if os.environ.get('CONT_ELO'): # rec6e2 diagnostic: lower bound of that constant (CONT_ELO=0 CONT_ECAP=0 fixes it at 0, i.e. T0' x = 0) lb[cid[(0, 2, NB)]] = float(os.environ['CONT_ELO']); print(f" CONT_ELO: c(0, 2, {NB}) >= {lb[cid[(0, 2, NB)]]}", flush=True) if os.environ.get('CONT_NOGREC'): for col in gid.values(): lb[col] = 0.0; ub[col] = 0.0 print(f" CONT_NOGREC: {len(set(gid.values()))} G columns fixed to 0", flush=True) if os.environ.get('CONT_LBC'): sys.exit('cont_hms_rec4: CONT_LBC is not supported (focused read M1)') LBC = os.environ.get('CONT_LBC'); low_cls = [] if LBC is not None: LBC = float(LBC) for cl_ in classes: if cl_[0] >= 1 and (cl_[1] >= 2 or (cl_[1] == 1 and cl_[0] >= 2 and not os.environ.get('CONT_NOGREC'))): lb[cid[cl_]] = LBC; low_cls.append(cl_) hid_arr = -np.ones((MS + 1, na, na), dtype=int) for (j, tau, m), col in hid.items(): hid_arr[j, tau, m] = col gid_arr = -np.ones((JG + 1, N + 1, na), dtype=int) for (j, tg, m), col in gid.items(): gid_arr[j, tg, m] = col # cumulative kernels as arrays over the maximum: LJc[jy][m, tau_y, a], LGc[jy][m, tau_y, a], LASc[jy][m, a] mi = np.array(mgrid); NM = len(mgrid) LJc = {jy: np.array([RK[m]['LJ'][jy] for m in mgrid]) for jy in range(1, MS + 1)} LGc = {jy: np.array([RK[m]['LG'][jy] for m in mgrid]) for jy in range(1, JG + 1)} LASc = {jy: np.array([RK[m]['LAS'][jy] for m in mgrid]) for jy in range(2, JG + 1)} def exact_from(cum, mb): """rows over M = mb..ia of the 'maximum exactly M, or <= mb at M = mb' kernel: cum has a leading axis over mgrid (18:0x: callers now pass cum already sliced at the a-index, so this copies small arrays only)""" k0 = mb - ie out = cum[k0:].copy() out[1:] -= cum[k0:-1] return out # out[0] = cum(<= mb); out[i] = cum(<= mb+i) - cum(<= mb+i-1) LJa = {jy: np.ascontiguousarray(np.moveaxis(LJc[jy], 2, 0)) for jy in LJc} # [a][m, tau_y] LGa = {jy: np.ascontiguousarray(np.moveaxis(LGc[jy], 2, 0)) for jy in LGc} LASa = {jy: np.ascontiguousarray(LASc[jy].T) for jy in LASc} # [a][m] from array import array as _arr Rr, Rc, Rv = _arr('q'), _arr('q'), _arr('d'); nrow = 0; desc = [] def emit(entries): global nrow for c_, v_ in entries.items(): if v_ != 0.0: Rr.append(nrow); Rc.append(c_); Rv.append(v_) nrow += 1 def emit_arrays(base_dict, cols, vals): d = dict(base_dict) for c_, v_ in zip(cols, vals): d[c_] = d.get(c_, 0.0) + v_ emit(d) # free rows for cl in classes: j, k, b = cl if not in_sup(support(j, k, b), 1.0) or j == 0 or k == 0: continue supR = support(0, k, b) for (jj, tau, m), col in hid.items(): if jj == j and in_sup(supR, 1 - tau * h) and lb[col] != ub[col]: emit({cid[cl]: 1.0, col: 1.0}); desc.append(('free', cl)) if k == 1: seen = set() for (jj, tg, m), col in gid.items(): if jj == j and in_sup(supR, 1 - tg * h) and col not in seen: seen.add(col); emit({cid[cl]: 1.0, col: 1.0}); desc.append(('freeG', cl)) # every free grid shape; classes with a shape carrying no active added term keep c >= -1 _fs_cls, _fs_h, _fs_g = [], [], [] def _parts(total, lower, prefix): if total == 0: yield prefix; return for p_ in range(lower, min(ihf - 1, total) + 1): if total == p_ or total - p_ >= p_: yield from _parts(total - p_, p_, prefix + (p_,)) _pure = set() for _sh in _parts(N, ie, ()): _sm = [q for q in _sh if q <= ia]; _ro = [q for q in _sh if q > ia] _j, _k, _tau = len(_sm), len(_ro), sum(_sm) _b = 0 if not _ro else next(i_ for i_ in range(1, len(iedges)) if max(_ro) <= iedges[i_]) _cl = (_j, _k, _b) if _cl not in cid: continue _hc = hid.get((_j, _tau, max(_sm))) if (_sm and _tau <= ia) else None _gc = gid.get((_j, _tau, max(_sm))) if (_sm and _k == 1) else None _act = [cc_ for cc_ in (_hc, _gc) if cc_ is not None and not (lb[cc_] == 0 and ub[cc_] == 0)] if not _act: _pure.add(_cl) _fs_cls.append(cid[_cl]); _fs_h.append(_hc if _hc is not None else -1); _fs_g.append(_gc if _gc is not None else -1) _fs_cls = np.array(_fs_cls); _fs_h = np.array(_fs_h); _fs_g = np.array(_fs_g) _restored = [] for _cl in sorted(_pure): if lb[cid[_cl]] < -1: lb[cid[_cl]] = -1.0; _restored.append(_cl) _cn_cls, _cn_h, _cn_g, _cn_bare = [], [], [], [] # rec4 (3): closed-interval free nodes, as band() weights them for _cl in classes: _j, _k, _b = _cl if _k == 0: _taus = [N] if (_j >= 1 and _j * ie <= N <= _j * ia) else [] else: _sR = support(0, _k, _b); _taus = [t_ for t_ in range(_j * ie, min(_j * ia, N) + 1) if in_sup(_sR, 1 - t_ * h)] if _j >= 1 else ([0] if in_sup(_sR, 1.0) else []) for _t in _taus: for _m in (feas_max(_j, _t) if _j >= 1 else [None]): _hc = hid.get((_j, _t, _m)) if (_j >= 1 and _t <= ia) else None _gc = gid.get((_j, _t, _m)) if (_j >= 1 and _k == 1) else None _hc = _hc if (_hc is not None and not (lb[_hc] == 0 and ub[_hc] == 0)) else None _gc = _gc if (_gc is not None and not (lb[_gc] == 0 and ub[_gc] == 0)) else None _cn_cls.append(cid[_cl]); _cn_h.append(-1 if _hc is None else _hc); _cn_g.append(-1 if _gc is None else _gc) _cn_bare.append(_hc is None and _gc is None) _cn_cls = np.array(_cn_cls); _cn_h = np.array(_cn_h); _cn_g = np.array(_cn_g); _cn_bare = np.array(_cn_bare) print(f" REC4: {len(_cn_cls)} closed-interval free nodes (class, tau, m), {int(_cn_bare.sum())} bare (constant only); constants lower bound min {lb[:nc].min():g}", flush=True) print(f" REC3: {len(_fs_cls)} free grid shapes; classes with a pure-constant free shape: {len(_pure)}; lower bound restored to -1 on {len(_restored)}: {_restored}", flush=True) npure = 0 if LBC is not None: for cl_ in low_cls: j_, k_, b_ = cl_ if k_ != 1: continue missing = False for rho in range(max(iedges[b_ - 1], ia + 1), min(iedges[b_], ihf - 1) + 1): tg = N - rho if not (j_ * ie <= tg <= j_ * ia): continue for m_ in feas_max(j_, tg): if (j_, tg, m_) not in gid: missing = True; break if missing: break if missing: emit({cid[cl_]: 1.0}); desc.append(('pure_c', cl_)); npure += 1 print(f" CONT_LBC = {LBC}: lowered {len(low_cls)} class constants; restored c >= -1 rows: {npure}", flush=True) nfree = nrow alist = list(range(ihf, N, ASUB)) for bc in w_classes: jb, kb, bb = bc if bc == (0, 0, 0): continue supR = support(0, kb, bb) for ai in alist: a = ai * h; tb = N - ai if not in_sup(support(jb, kb, bb), tb * h): continue base = {} for key, L in Lam.items(): jy, ky, bp = key; tgt = cid.get((jb + jy, kb + ky, max(bb, bp))) if tgt is not None: base[tgt] = base.get(tgt, 0.0) + a * L[ai] if jb == 0: cols = []; vals = [] for jy in range(1, MS + 1): # fragments carry the whole small part: H(jy, tau_y, M), M = their maximum ex = exact_from(LJa[jy][ai], ie) # (NM, na): maximum exactly M for k0, M in enumerate(mgrid): cc = hid_arr[jy, :, M]; ok = cc >= 0 if ok.any(): cols += list(cc[ok]); vals += list(h * a * ex[k0][ok]) if kb == 1: for jy in range(2, JG + 1): ex = exact_from(LASa[jy][ai], ie) for k0, M in enumerate(mgrid): cg = gid_arr[jy, ai, M] if ai <= N else -1 if cg >= 0: cols.append(cg); vals.append(a * ex[k0]) if bc == (0, 1, NB) and ai == ihf: continue # rec6 (1): no positivity row at E = (1/2, 1/2) emit_arrays(base, cols, vals); desc.append(('ext', bc)); continue if kb: cands = [t_ for t_ in range(jb * ie, min(jb * ia, tb) + 1, SSUB) if in_sup(supR, (tb - t_) * h)] else: cands = [tb] if jb * ie <= tb <= jb * ia else [] done_const = False for tbeta in cands: mbs = feas_max(jb, tbeta) if tbeta <= ia else [None] if tbeta > ia and kb >= 2: if not done_const: emit(dict(base)); desc.append(('ext', bc)); done_const = True continue if tbeta > ia: mbs = feas_max(jb, tbeta) for mb in mbs[::MSUB]: cols = []; vals = [] if tbeta <= ia: c0 = hid_arr[jb, tbeta, mb] if jb <= MS else -1 if c0 >= 0: cols.append(c0); vals.append(a * Lam0[ai]) for jy in range(1, MS - jb + 1): ex = exact_from(LJa[jy][ai], mb) # (ia - mb + 1, na) for k0 in range(ex.shape[0]): M = mb + k0; taus_y = np.arange(jy * ie, ia - tbeta + 1) if len(taus_y) == 0: continue cc = hid_arr[jb + jy, tbeta + taus_y, M]; ok = cc >= 0 if ok.any(): cols += list(cc[ok]); vals += list(h * a * ex[k0][taus_y[ok]]) if kb == 0: for jy in range(1, JG - jb + 1): ex = exact_from(LGa[jy][ai], mb) taus_y = np.arange(jy * ie, ng); tt_ = tbeta + taus_y; okt = tt_ <= N taus_y, tt_ = taus_y[okt], tt_[okt] for k0 in range(ex.shape[0]): M = mb + k0; cc = gid_arr[jb + jy, tt_, M]; ok = cc >= 0 if ok.any(): cols += list(cc[ok]); vals += list(h * a * ex[k0][taus_y[ok]]) elif kb == 1: for jy in range(2, JG - jb + 1): ex = exact_from(LASa[jy][ai], mb) tt_ = tbeta + ai if tt_ > N: continue for k0 in range(len(ex)): cg = gid_arr[jb + jy, tt_, mb + k0] if cg >= 0: cols.append(cg); vals.append(a * ex[k0]) emit_arrays(base, cols, vals); desc.append(('ext', bc)) Gm = coo_matrix((Rv, (Rr, Rc)), shape=(nrow, nv)).tocsr() print(f"[{time.time()-t0:.1f}s] positivity rows: free {nfree}, extended {nrow - nfree}; nnz {Gm.nnz}; vacuous H fixed {nvac}; |L| <= {GCAP}", flush=True) # ---- control: tie L(j, tau, m) across m (must reproduce cont_hmg) ---- Aeq = _csr(T) if os.environ.get('CONT_TIEM'): tr, tc, tv = [], [], []; nt = 0 for store in (hid, gid): groups = {} for (j, tau, m), col in store.items(): groups.setdefault((j, tau), []).append(col) for cols in groups.values(): cols = sorted(set(cols)) for c1, c2 in zip(cols[:-1], cols[1:]): tr += [nt, nt]; tc += [c1, c2]; tv += [1.0, -1.0]; nt += 1 Aeq = _csr(np.vstack([T, coo_matrix((tv, (tr, tc)), shape=(nt, nv)).toarray()])) if nt else Aeq print(f" CONT_TIEM: {nt} tie rows", flush=True) NOUP = bool(os.environ.get('CONT_NOUPPER')) from scipy.sparse import vstack as _svs Aub = -Gm if NOUP else _svs([-Gm, Gm]).tocsr(); bub = np.ones(nrow) if NOUP else np.concatenate([np.ones(nrow), np.full(nrow, PHIMAX)]) _RAW = dict(T=T.copy(), f1=f1.copy(), Gm=Gm.copy(), TDEAD=(TDEAD.copy() if TDEAD is not None else None)) TINY = float(os.environ.get('CONT_TINY', '1e-9')) # remove coefficients below CONT_TINY from T, the dead rows, the positivity rows and the objective; this perturbs the rows and the objective, and the residuals reported below refer to the modified matrices T[np.abs(T) < TINY] = 0.0; f1[np.abs(f1) < TINY] = 0.0 Gm.data[np.abs(Gm.data) < TINY] = 0.0; Gm.eliminate_zeros() Aeq = _csr(T) if not os.environ.get('CONT_TIEM') else Aeq if os.environ.get('CONT_TIEM'): Aeq = Aeq.tocsr(); Aeq.data[np.abs(Aeq.data) < TINY] = 0.0; Aeq.eliminate_zeros() Aub = -Gm if NOUP else _svs([-Gm, Gm]).tocsr() if TDEAD is not None: TDEAD[np.abs(TDEAD) < TINY] = 0.0; epsd = float(os.environ['CONT_DEADEPS']) DS = float(os.environ.get('CONT_DEADSCALE', '1')) Aub = _svs([Aub, _csr(DS * TDEAD), _csr(-DS * TDEAD)]).tocsr(); bub = np.concatenate([bub, np.full(2 * TDEAD.shape[0], DS * epsd)]) print(f" sanity: finite T {np.isfinite(T).all()}, f1 {np.isfinite(f1).all()}, G {np.isfinite(Gm.data).all()}; max|T| {np.abs(T).max():.3e}, max|G| {np.abs(Gm.data).max():.3e}, max|f1| {np.abs(f1).max():.3e}; min|nonzero T| {np.abs(T[T != 0]).min():.3e}", flush=True) if os.environ.get('CONT_FEASCHECK'): r0 = linprog(np.zeros(nv), A_ub=Aub, b_ub=bub, A_eq=Aeq, b_eq=np.zeros(Aeq.shape[0]), bounds=np.column_stack([lb, ub]), method='highs', options={'presolve': True}) print(f" feasibility LP (zero objective): status {r0.status} ({r0.message})", flush=True) if os.environ.get('CONT_NOX'): # bundled HiGHS, IPM, run_crossover off; primal value + residuals + a float dual bound from scipy.optimize._highspy import _core as _hc from scipy.sparse import vstack as _vs2 from types import SimpleNamespace as _NS hh_ = _hc._Highs() for k_, v_ in (('output_flag', bool(os.environ.get('CONT_DISP'))), ('solver', 'ipm'), ('run_crossover', 'off'), ('presolve', 'on' if int(os.environ.get('CONT_PRESOLVE', '0')) else 'off'), ('ipm_optimality_tolerance', float(os.environ.get('CONT_OTOL', '1e-8'))), ('primal_feasibility_tolerance', float(os.environ.get('CONT_PTOL', '1e-7'))), ('dual_feasibility_tolerance', float(os.environ.get('CONT_PTOL', '1e-7'))), ('time_limit', float(os.environ.get('CONT_TLIM', '1e9')))): hh_.setOptionValue(k_, v_) INF_ = hh_.getInfinity() A_ = _vs2([Aeq, Aub]).tocsc(); A_.sort_indices(); neq_ = Aeq.shape[0]; m_ = A_.shape[0] rl_ = np.concatenate([np.zeros(neq_), np.full(Aub.shape[0], -INF_)]); ru_ = np.concatenate([np.zeros(neq_), np.asarray(bub, dtype=np.float64)]) hh_.addRows(m_, rl_, ru_, 0, np.zeros(m_, dtype=np.int32), np.zeros(0, dtype=np.int32), np.zeros(0)) lbh_ = np.where(np.isinf(lb), -INF_, lb); ubh_ = np.where(np.isinf(ub), INF_, ub) hh_.addCols(nv, np.asarray(f1, dtype=np.float64), lbh_, ubh_, A_.nnz, A_.indptr[:-1].astype(np.int32), A_.indices.astype(np.int32), A_.data.astype(np.float64)) print(f"[{time.time()-t0:.1f}s] NOX: model {hh_.getNumRow()} rows x {hh_.getNumCol()} cols, nnz {hh_.getNumNz()}; running IPM (no crossover)", flush=True) hh_.run() st_ = hh_.modelStatusToString(hh_.getModelStatus()); info_ = hh_.getInfo(); sol_ = hh_.getSolution() xx_ = np.asarray(sol_.col_value, dtype=np.float64); yy_ = np.asarray(sol_.row_dual, dtype=np.float64) pv_ = float(f1 @ xx_) if len(xx_) == nv else float('nan') if len(xx_) == nv: eqr_ = float(np.abs(Aeq @ xx_).max()) if neq_ else 0.0; ubv_ = float(np.maximum(Aub @ xx_ - bub, 0).max()) bdv_ = float(max(np.maximum(lb - xx_, 0).max(), np.maximum(xx_ - ub, 0).max())) else: eqr_ = ubv_ = bdv_ = float('nan') # float weak-duality bound: y free on equality rows, y <= 0 on the <= rows (clipped), d = c - A^T y, every column boxed db_ = float('nan') if len(yy_) == m_: y_ = yy_.copy(); y_[neq_:] = np.minimum(y_[neq_:], 0.0) d_ = np.asarray(f1, dtype=np.float64) - A_.T @ y_ if np.isfinite(lb).all() and np.isfinite(ub).all(): db_ = float(y_[neq_:] @ np.asarray(bub, dtype=np.float64) + np.sum(np.where(d_ >= 0, d_ * lb, d_ * ub))) print(f" NOX: status {st_}; ipm iters {info_.ipm_iteration_count}, crossover iters {info_.crossover_iteration_count}; HiGHS objective " f"{info_.objective_function_value:.9f}; primal value f1.x = {pv_:.9f} (max |Aeq x| {eqr_:.1e}, max ub violation {ubv_:.1e}, " f"max bound violation {bdv_:.1e}); float dual bound {db_:.9f}; HiGHS max primal/dual infeas {info_.max_primal_infeasibility:.1e} / " f"{info_.max_dual_infeasibility:.1e}", flush=True) if os.environ.get('CONT_SAVEX'): np.save(os.environ['CONT_SAVEX'], xx_) if len(xx_) == nv and len(yy_) == m_: # residuals, acceptance, net-multiplier dual bounds (pruned and raw) DS_ = float(os.environ.get('CONT_DEADSCALE', '1')); epsd_ = float(os.environ.get('CONT_DEADEPS', '0')) nd_ = TDEAD.shape[0] if TDEAD is not None else 0; npos_ = Gm.shape[0] live_p = float(np.abs(Aeq @ xx_).max()) if neq_ else 0.0 dead_p = float(np.abs(TDEAD @ xx_).max()) if nd_ else 0.0 pos_p = float((Gm @ xx_).min()); boxv = float(max(np.maximum(lb - xx_, 0).max(), np.maximum(xx_ - ub, 0).max())) live_r = float(max(np.abs(_RAW['T'] @ xx_).max(), (np.abs(Aeq[_RAW['T'].shape[0]:] @ xx_).max() if neq_ > _RAW['T'].shape[0] else 0.0))) if neq_ else 0.0 dead_r = float(np.abs(_RAW['TDEAD'] @ xx_).max()) if nd_ else 0.0 pos_r = float((_RAW['Gm'] @ xx_).min()); obj_r = float(_RAW['f1'] @ xx_) acc_ = (st_ == 'Optimal') and (live_p <= 1e-7) and (dead_p <= epsd_ + 1e-10) and (pos_p >= -1 - 1e-7) and (boxv <= 1e-7) y_ = yy_.copy(); y_[neq_:neq_ + npos_] = np.minimum(y_[neq_:neq_ + npos_], 0.0) if nd_: yp_, ym_ = y_[neq_ + npos_:neq_ + npos_ + nd_].copy(), y_[neq_ + npos_ + nd_:].copy(); lam_ = yp_ - ym_ y_[neq_ + npos_:neq_ + npos_ + nd_] = np.minimum(lam_, 0.0); y_[neq_ + npos_ + nd_:] = np.minimum(-lam_, 0.0) bfull_ = np.concatenate([np.zeros(neq_), np.asarray(bub, dtype=np.float64)]) def _bound(Amat, cvec): d__ = np.asarray(cvec, dtype=np.float64) - Amat.T @ y_ terms = np.where(d__ >= 0, d__ * lb, d__ * ub); val = float(bfull_ @ y_ + terms.sum()) allow = 1e-13 * (float(np.abs(bfull_ * y_).sum()) + float(np.abs(terms).sum())) return val - allow Lp_ = _bound(A_, f1) nT_ = _RAW['T'].shape[0] parts = [_csr(_RAW['T'])] + ([Aeq[nT_:]] if neq_ > nT_ else []) + [-_RAW['Gm']] + ([_csr(DS_ * _RAW['TDEAD']), _csr(-DS_ * _RAW['TDEAD'])] if nd_ else []) Lr_ = _bound(_vs2(parts).tocsr(), _RAW['f1']) _fv = xx_[_fs_cls] + np.where(_fs_h >= 0, xx_[np.maximum(_fs_h, 0)], 0.0) + np.where(_fs_g >= 0, xx_[np.maximum(_fs_g, 0)], 0.0) _fvc = xx_[_cn_cls] + np.where(_cn_h >= 0, xx_[np.maximum(_cn_h, 0)], 0.0) + np.where(_cn_g >= 0, xx_[np.maximum(_cn_g, 0)], 0.0) print(f" FREEMIN_CLOSED: min f over {len(_fvc)} closed-interval free nodes = {float(_fvc.min()):.12f}; over bare nodes = {float(_fvc[_cn_bare].min()) if _cn_bare.any() else float('nan'):.12f}", flush=True) print(f" FREEMIN: min f over all {len(_fs_cls)} free grid shapes = {float(_fv.min()):.12f}", flush=True) print(f" REC: status {st_}; objective pruned {pv_:.9f} raw {obj_r:.9f}; float dual lower bound pruned {Lp_:.9f} raw {Lr_:.9f}; " f"pruned residuals live {live_p:.2e} dead {dead_p:.2e} (eps {epsd_:g}, scale {DS_:g}) min positivity {pos_p:.12f} box viol {boxv:.1e}; " f"raw residuals live {live_r:.2e} dead {dead_r:.2e} min positivity {pos_r:.12f}; ACCEPT {acc_}; rows: live {neq_}, positivity {npos_}, dead {nd_}", flush=True) _cE = cid[(0, 2, NB)]; print(f" EOPT: c(0, 2, {NB}) = {xx_[_cE]:.9f}; T0 x = {float(T[0] @ xx_):.3e}; T0' x (without that term) = {float(T[0] @ xx_ - T[0, _cE] * xx_[_cE]):.9f}", flush=True) if os.environ.get('CONT_SAVEX'): np.savez_compressed(os.environ['CONT_SAVEX'] + '.rec.npz', x=xx_, y=yy_, lb=lb, ub=ub) res = _NS(status=0 if st_ == 'Optimal' else 1, message=f'NOX {st_}', x=xx_ if st_ == 'Optimal' else None, fun=pv_ if st_ == 'Optimal' else None, nit=info_.ipm_iteration_count) else: res = linprog(f1, A_ub=Aub, b_ub=bub, A_eq=Aeq, b_eq=np.zeros(Aeq.shape[0]), bounds=np.column_stack([lb, ub]), method=os.environ.get('CONT_METHOD', 'highs-ipm'), options=dict(presolve=bool(int(os.environ.get('CONT_PRESOLVE', '0'))), **({'primal_feasibility_tolerance': float(os.environ['CONT_PTOL']), 'dual_feasibility_tolerance': float(os.environ['CONT_PTOL'])} if os.environ.get('CONT_PTOL') else {}), **({'ipm_optimality_tolerance': float(os.environ['CONT_OTOL'])} if os.environ.get('CONT_OTOL') else {}), **({'time_limit': float(os.environ['CONT_TLIM'])} if os.environ.get('CONT_TLIM') else {}), **({'disp': True} if os.environ.get('CONT_DISP') else {}))) print(f"[{time.time()-t0:.1f}s] LP status {res.status} ({res.message})", flush=True) print(f" result: fun {getattr(res, 'fun', None)}; nit {getattr(res, 'nit', None)}", flush=True) # night2 if res.status != 0 and getattr(res, 'x', None) is not None: # report the returned nonoptimal point and its residuals; feasibility is not asserted xx_ = np.asarray(res.x); print(f" NONOPT POINT: f1.x = {float(f1 @ xx_):.6f}; max eq residual {float(np.abs(Aeq @ xx_).max()):.2e}; max ub violation {float(np.maximum(Aub @ xx_ - bub, 0).max()):.2e}", flush=True) np.save(os.environ.get('CONT_SAVEX', 'nonopt_x.npy'), xx_) if res.status == 0: x = res.x; z = f1 @ x print(f" min f(1) = {z:.6f} -> {'PRIME-FREE after rescaling, margin %.4f' % (1 - 1/abs(z)) if z < -1 else 'NOT prime-free, C- >= %.6f' % (1 + z)}") hv = x[Hc]; gv = x[Gc] if len(Gc) else np.zeros(1) print(f" H in [{hv.min():.3f}, {hv.max():.3f}] (at cap {int((np.abs(hv) > GCAP - 1e-6).sum())}); G in [{gv.min():.3f}, {gv.max():.3f}] (at cap {int((np.abs(gv) > GCAP - 1e-6).sum())}); |T x| {np.abs(T @ x).max():.1e}") ~~~~~~~~~~~~ #### file: compare_gexact.py - kind: script - as run: SHA-256 1ed6fad57ca2a44a9724f175cb4323e673509e6bfdc2167f8b4dc31139555d1f - as shipped: SHA-256 48564dc30952ff5d39c5c3fc174af5eaa18c169451df258ecfdd903a20ded22a - identity: identical, after dropping docstrings and comments, to the file as run - trailing newline: yes ~~~~~~~~~~~~ python #!/usr/bin/env python3 """compare_gexact.py: every shipped G and richer-family record that rec6's FFT-sign test restricts, rerun with cont_hms_rec6g.py (G support by exact integer feasibility). For each: the rec6 value, the rec6g value, the change, the number of restored G keys, and section 8's acceptance checks on the rec6g solve (check_sec8_values.check, unchanged). Usage: python3 -B compare_gexact.py REC6_RAW_DIR GEXACT_RAW_DIR Exit 0 only if every rerun passes the checks.""" import os, re, sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from check_sec8_values import check, QUOTED PAIRS = { # table key: rec6g raw stem (run_rec6g.sh names) 'G25_u4': 'G_n72_e36_a25_u4_g50', 'G25_u1000': 'G_n72_e36_a25_u1000_g1000', 'M72_24_u4': 'M_n72_e36_a24_u4_g50', 'M72_25_u4': 'M_n72_e36_a25_u4_g50', 'M72_25_u50': 'M_n72_e36_a25_u50_g50', 'M72_25_u1000': 'M_n72_e36_a25_u1000_g1000', 'M72_24_u1000': 'M_n72_e36_a24_u1000_g1000', 'M72_26_u2000': 'M_n72_e36_a26_u2000_g2000', 'M72_27_u2000': 'M_n72_e36_a27_u2000_g2000', 'M90_31': 'M_n90_e45_a31_u4_g50', 'M90_32': 'M_n90_e45_a32_u4_g50', 'M96_34': 'M_n96_e48_a34_u4_g50', } VAL = re.compile(r'value ([-+0-9.]+) \(raw') GX = re.compile(r'GEXACT: (\d+) G keys') if __name__ == '__main__': d6, dg = sys.argv[1], sys.argv[2] ok_all = True; dmax = 0.0 for k, stem in PAIRS.items(): l6, ok6 = check([d6], k, QUOTED[k]); lg, okg = check([dg], k, stem) v6 = float(VAL.search(l6).group(1)); m = VAL.search(lg) vg = float(m.group(1)) if m else float('nan') fg = lg.split()[1] if m else '-' nk = GX.search(open(os.path.join(dg, fg)).read()).group(1) if m else '?' dv = vg - v6; dmax = max(dmax, abs(dv)) if m else float('inf'); ok_all &= ok6 and okg print(f"{k:14s} rec6 {v6:+.9f} rec6g {vg:+.9f} change {dv:+.2e} restored keys {nk:>3s} rec6g checks {'PASS' if okg else 'FAIL'} ({fg})") if not okg: print(' ' + lg) print(f"VERDICT: {len(PAIRS)} records; max |change| {dmax:.2e}; every rerun passes the checks: {ok_all}") sys.exit(0 if ok_all else 1) ~~~~~~~~~~~~ #### file: e_report.py - kind: script - as run: SHA-256 f5348328c0d41ee0c29bc2ce70387ba9b1a2b4d42b850fc0013dc134bbd9e132 - as shipped: SHA-256 726329c0007b265cc7f3b02157f0aa8e0f7d203cc6e2a1a83345dbcaa1dad56d - identity: identical, after dropping docstrings and comments, to the file as run - trailing newline: yes ~~~~~~~~~~~~ python #!/usr/bin/env python3 """e_report.py: at every optimum of record (the accepted solve that check_sec8_values.py selects), the constant c(0, 2, top bin) of E = (1/2, 1/2) read as two rough components, the box it sits in, T_0' x = T_0 x - c_E/144 (computed from the saved x and the equality T_0 x = 0, which holds to the printed live residual; the coefficient is taken from the EINFO lines of cont_hms_rec6e.py when given, else 1/144 at N = 72), and the first-order change of f(1) if c_E were fixed at 0 (T_0' x = 0), -y_T0 * coef * (0 - c_E), from the saved row dual of T_0 (row 0). Usage: e_report.py REC_DIR SWEEP_DIR [einfo.out]""" import os, re, sys, numpy as np sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from check_sec8_values import check, QUOTED, CONTROLS coef = {} if len(sys.argv) > 3: for l in open(sys.argv[3]): m = re.search(r'N=(\d+) eta=([0-9/]+) alpha=([0-9/]+) edges=\[([^\]]*)\].*EINFO: class \(0, 2, (\d+)\) column (\d+); T0 coefficient ([-0-9.e]+)', l) if m: coef[(m.group(1), m.group(2), m.group(3), m.group(4))] = (int(m.group(6)), float(m.group(7))) rows = [] for k, stem in list(QUOTED.items()) + list(CONTROLS.items()): line, ok = check(sys.argv[1:3], k, stem) fn = line.split()[1] raw = next(os.path.join(d, fn) for d in sys.argv[1:3] if os.path.exists(os.path.join(d, fn))) t = open(raw).read() m = re.search(r"N=(\d+) eta=([0-9/]+) alpha=([0-9/]+) edges=\[([^\]]*)\]", t) NB = len(m.group(4).split(',')) - 1 key = m.groups() cid, c0 = coef.get(key, (2 * NB - 1, 1 / 144 if m.group(1) == '72' else float('nan'))) assert cid == 2 * NB - 1 z = np.load(raw[:-4] + '.rec.npz'); x, y, lb, ub = z['x'], z['y'], z['lb'], z['ub'] cE = x[cid]; T0p = -c0 * cE # T_0 x = 0 up to the live residual d = -y[0] * c0 # reduced cost of c_E (f1 coefficient 0; only T_0 carries it) at = 'lower bound -1' if abs(cE + 1) < 1e-6 else ('upper bound U' if abs(cE - ub[cid]) < 1e-6 * max(1, ub[cid]) else 'interior') est = d * (0 - cE) print(f"{k:14s} {'quoted' if k in QUOTED else 'control':7s} c_E {cE:+.6f} in [{lb[cid]:g}, {ub[cid]:g}] ({at}); T0' x {T0p:+.6f} " f"(coef {c0:.6g}); y_T0 {y[0]:+.4e}; first-order change of f(1) with c_E fixed at 0: {est:+.2e} ({fn})") ~~~~~~~~~~~~ #### file: check_example.py - kind: script - as run: SHA-256 6b23addef4b1b890c1e46cc2ce94667152fd60f7eb8a11aa1ad689e758353a78 - as shipped: SHA-256 6b23addef4b1b890c1e46cc2ce94667152fd60f7eb8a11aa1ad689e758353a78 - identity: identical, after dropping docstrings and comments, to the file as run - trailing newline: yes ~~~~~~~~~~~~ python """Check one shipped primal/dual example of the family programme (example_F_n72_a25.json, example_G_n72_a25.json, example_M_n72_a25.json), as the README describes. Usage, from the directory that holds the example files, after `python unpack.py OUTDIR`: python OUTDIR/check_example.py example_F_n72_a25.json OUTDIR The script reruns the program named in the example's `how` field (taken from OUTDIR) with the environment given in `common` and `how` and the arguments given in `how`, and keeps the matrices that program assembles: A_ (the equality rows stacked over the <= rows), bub (the right-hand sides of the <= rows; the equality rows have right-hand side 0), f1 (the objective q), lb and ub. On those matrices it checks the example against the run's printed `REC:` line in the shipped output named at the end of `how`: x: equality rows to 1e-7; rows of Lemma 8.1, unscaled, to eps + 1e-10; every positivity row >= -1 - 1e-7; the box lb <= x <= ub to 1e-7; q.x within 1e-9 of the printed pruned objective; y: positivity-row duals clipped to <= 0; each dead-row pair replaced by lambda = y(+ copy) - y(- copy), placed as min(lambda, 0) and min(-lambda, 0); with d = q - A^T y and t_j = min(d_j lb_j, d_j ub_j), S = b.y + sum_j t_j; then S - 1e-13 (sum_i |b_i y_i| + sum_j |t_j|), the program's own heuristic rounding allowance, within 1e-9 of the printed pruned float dual lower bound. Two negative controls follow on the same matrices: x with one entry changed by 1e-3 (the column with the largest objective coefficient) and y with one entry changed by 1e-3 (the first equality row); each must FAIL. The bound is flat in some directions of y: for instance it does not depend at all on the dual of a row whose columns are all fixed (lb = ub), such as a row tying two columns that are fixed to zero, so a change there is not detected; the control therefore uses the first equality row, where the bound is not flat. The exit status is 0 when the example passes and both controls fail, 1 otherwise. These are floating-point checks of solver output, not certificates. """ import sys, os, re, glob, json, time import numpy as np if len(sys.argv) != 3: sys.exit(__doc__) ex_path, outdir = sys.argv[1], sys.argv[2] ex = json.load(open(ex_path)) prog = ex['how'].split()[0] out_name = re.search(r'(\S+\.out)\s*$', ex['how']).group(1) env = dict(re.findall(r'(CONT_[A-Z]+)=([^\s;]+)', ex['common'] + ' ' + ex['how'])) a = re.search(r'args ([^;]+);', ex['how']).group(1).split() print('example %s (family %s): program %s, output %s' % (os.path.basename(ex_path), ex['family'], prog, out_name)) print('environment %s; arguments %s' % (' '.join('%s=%s' % kv for kv in env.items()), ' '.join(a)), flush=True) # the printed line of the original run, from the shipped output prefix = 'N=%s eta=1/%s alpha=%s/%s edges=%s U=%s LBC=%s GCAP=%s presolve=%s:' % ( a[0], a[2], a[3], a[4], a[5], a[6], env.get('CONT_LBC', 'none'), env['CONT_GCAP'], env['CONT_PRESOLVE']) cands = glob.glob(os.path.join(outdir, '**', out_name), recursive=True) if len(cands) != 1: sys.exit('REFUSED: expected one file named %s under the unpacked directory, found %d' % (out_name, len(cands))) orig = [l for l in open(cands[0]) if prefix in l and 'REC: status' in l] if len(orig) != 1: sys.exit('REFUSED: expected one printed REC line for "%s" in %s, found %d' % (prefix, out_name, len(orig))) orig_rec = re.search(r'REC: status.*', orig[0].rstrip('\n')).group(0) orig_rec = re.sub(r' RSS_kB .*$', '', orig_rec) # resource figures appended by the run's recipe o_obj = float(re.search(r'objective pruned (\S+)', orig_rec).group(1)) o_db = float(re.search(r'float dual lower bound pruned (\S+)', orig_rec).group(1)) print('printed by the original run: pruned objective %.9f, pruned float dual lower bound %.9f' % (o_obj, o_db), flush=True) class Tee: def __init__(self, s): self.s, self.buf = s, [] def write(self, t): self.buf.append(t) return self.s.write(t) def flush(self): self.s.flush() for k in [k for k in os.environ if k.startswith('CONT_') or k == 'DEAD_SVD']: del os.environ[k] os.environ.update(env) script = os.path.join(outdir, prog) sys.argv = [script] + a ns = {'__name__': '__main__', '__file__': script} print('---- rerun of %s ----' % prog, flush=True) real, tee = sys.stdout, Tee(sys.stdout) sys.stdout = tee t0 = time.process_time() # CPU time: the wall clock can step backwards try: exec(compile(open(script).read(), script, 'exec'), ns) except SystemExit as e: print('(the program called sys.exit(%r); its variables are kept)' % (e.code,)) finally: sys.stdout = real print('---- end of rerun (%.0f s of CPU time) ----' % (time.process_time() - t0), flush=True) rerun_rec = [m.group(0) for m in re.finditer(r'REC: status[^\n]*', ''.join(tee.buf))] print('INFO: rerun printed the same REC line as the original run: %s' % ('YES' if rerun_rec == [orig_rec] else 'NO (%d REC lines; not part of the verdict)' % len(rerun_rec))) A = ns['A_'].tocsr(); bub = np.asarray(ns['bub'], dtype=np.float64); q = np.asarray(ns['f1'], dtype=np.float64) lb = np.asarray(ns['lb'], dtype=np.float64); ub = np.asarray(ns['ub'], dtype=np.float64); neq = int(ns['neq_']) npos = ns['Gm'].shape[0]; nd = ns['TDEAD'].shape[0] if ns.get('TDEAD') is not None else 0 DS = float(env.get('CONT_DEADSCALE', '1')); eps = float(env.get('CONT_DEADEPS', '0')) x0 = np.asarray(ex['x'], dtype=np.float64); y0 = np.asarray(ex['y'], dtype=np.float64) print('matrices: %d rows (equality %d, positivity %d, dead %d + %d) x %d columns; example x %d, y %d' % (A.shape[0], neq, npos, nd, nd, A.shape[1], len(x0), len(y0))) if A.shape != (len(y0), len(x0)) or A.shape[0] != neq + npos + 2 * nd: sys.exit('REFUSED: the example does not fit the rebuilt matrices') same_bounds = np.array_equal(np.asarray(ex['lb'], dtype=np.float64), lb) and np.array_equal(np.asarray(ex['ub'], dtype=np.float64), ub) b = np.concatenate([np.zeros(neq), bub]) def check(x, y): fails = [] if not same_bounds: fails.append('the example lb/ub differ from the rebuilt ones') Ax = A @ x eq = float(np.abs(Ax[:neq]).max()) if neq else 0.0 dead = float(np.abs(Ax[neq + npos:neq + npos + nd]).max() / DS) if nd else 0.0 pos = float((-Ax[neq:neq + npos]).min()) box = float(max(np.maximum(lb - x, 0).max(), np.maximum(x - ub, 0).max())) obj = float(q @ x) if not eq <= 1e-7: fails.append('equality residual %.2e > 1e-7' % eq) if not dead <= eps + 1e-10: fails.append('dead-row residual %.2e > %.1e' % (dead, eps + 1e-10)) if not pos >= -1 - 1e-7: fails.append('positivity %.12f < -1 - 1e-7' % pos) if not box <= 1e-7: fails.append('box violation %.1e > 1e-7' % box) if not abs(obj - o_obj) <= 1e-9: fails.append('q.x = %.12f is %.1e from the printed objective' % (obj, abs(obj - o_obj))) yy = y.copy() wrong = int(np.count_nonzero(yy[neq:] > 0)) yy[neq:neq + npos] = np.minimum(yy[neq:neq + npos], 0.0) if nd: lam = yy[neq + npos:neq + npos + nd] - yy[neq + npos + nd:] yy[neq + npos:neq + npos + nd] = np.minimum(lam, 0.0) yy[neq + npos + nd:] = np.minimum(-lam, 0.0) d = q - A.T @ yy t = np.minimum(d * lb, d * ub) S = float(b @ yy + t.sum()) allow = 1e-13 * (float(np.abs(b * yy).sum()) + float(np.abs(t).sum())) L = S - allow if not abs(L - o_db) <= 1e-9: fails.append('dual bound %.12f is %.1e from the printed bound' % (L, abs(L - o_db))) print(' x: equality residual %.2e, dead-row residual %.2e, min positivity %.12f, box violation %.1e, q.x %.12f' % (eq, dead, pos, box, obj)) print(' y: %d wrong-sign duals clipped; S %.12f, allowance %.2e, S - allowance %.12f' % (wrong, S, allow, L)) return fails print('THE EXAMPLE:') f0 = check(x0, y0) print(' -> %s' % ('PASS' if not f0 else 'FAIL: ' + '; '.join(f0))) j = int(np.argmax(np.abs(q))) x1 = x0.copy(); x1[j] += 1e-3 print('NEGATIVE CONTROL 1: x[%d] changed by 1e-3 (largest |q_j|):' % j) f1c = check(x1, y0) print(' -> %s' % ('FAIL, as required: ' + '; '.join(f1c) if f1c else 'PASS: the control was NOT refused')) i = 0 y2 = y0.copy(); y2[i] += 1e-3 print('NEGATIVE CONTROL 2: y[%d] changed by 1e-3 (first equality row):' % i) f2c = check(x0, y2) print(' -> %s' % ('FAIL, as required: ' + '; '.join(f2c) if f2c else 'PASS: the control was NOT refused')) ok = (not f0) and bool(f1c) and bool(f2c) print('VERDICT %s: %s' % (ex['family'], 'PASS (example passes; both negative controls fail)' if ok else 'NOT PASSED')) sys.exit(0 if ok else 1) ~~~~~~~~~~~~ #### file: agg_lp7.py - kind: script - as run: SHA-256 92c0dc4a667929892aea43fc34643a5d59cf68a732091038e3fd0163fea17034 - as shipped: SHA-256 798710f2f42c4f28213d3c293b7844d261f4379768a503214dc5be93f415756b - identity: identical, after dropping docstrings and comments, to the file as run with 1 declared change(s) of a machine-specific path, which now read: `sys.path.insert(0, __import__('os').path.dirname(__import__('os').path.abspath(__file__)))` - trailing newline: yes ~~~~~~~~~~~~ python """agg_lp7.py: the lever test of agg_lp5.py (the free twisted shape program restricted to a family phi(u) = g[class(u)], one value per class of shapes with equal features), with the class keys computed per family from the shape tuples and an option ROWMAXLEN = L that keeps only the rows whose modulus v has at most L parts (a relaxation: a positive value shows that the family is forced, a zero is inconclusive). Its function features(u, K, thT) gives the features of a shape u (parts in decreasing order, grid units): small parts are <= thT, rough parts lie strictly between thT and K/2, large parts are >= K/2; j and tau are the number and total of the small parts, k the number of rough parts, rmax the largest rough part, L the large part (0 if none), and S the small parts in decreasing order. verify_fake_v2.py uses it as an implementation of the class map separate from famrk.py's. Floats (HiGHS); not certified. Usage: agg_lp7.py K m0 thI thT cap fam1 [fam2 ...] (environment ETA, ALPHA, THETA_RULE as twist_otf.py).""" import math import os import sys import time from collections import Counter from fractions import Fraction as F import numpy as np from scipy.optimize import linprog from scipy.sparse import coo_matrix, csr_matrix sys.path.insert(0, __import__('os').path.dirname(__import__('os').path.abspath(__file__))) import modm_otf_lowmem2 as B0 # noqa: E402 from twist_otf import TwistRows # noqa: E402 time.time = time.monotonic def features(u, K, thT): sm = [x for x in u if x <= thT] ro = [x for x in u if thT < x < K / 2] la = [x for x in u if x >= K / 2] return {"j": len(sm), "tau": sum(sm), "k": len(ro), "rmax": ro[0] if ro else 0, "L": la[0] if la else 0, "smax": sm[0] if sm else 0, "s2": sm[1] if len(sm) > 1 else 0, "S": tuple(sm), "R": tuple(ro), "top3": tuple(sm[:3]), "top4": tuple(sm[:4]), "top5": tuple(sm[:5]), "top6": tuple(sm[:6]), "bot2": tuple(sm[-2:]), "nsm_ge": tuple(sum(1 for x in sm if x >= t) for t in (4, 8, 12, 16))} FAMS = {"ct": ("j", "tau", "k", "rmax", "L"), "ct+smax": ("j", "tau", "k", "rmax", "L", "smax"), "ct+s2": ("j", "tau", "k", "rmax", "L", "smax", "s2"), "S": ("S", "k", "rmax", "L"), "R": ("j", "tau", "R", "L"), "ct+s3": ("j", "tau", "k", "rmax", "L", "top3"), "ct+s4": ("j", "tau", "k", "rmax", "L", "top4"), "ct+s5": ("j", "tau", "k", "rmax", "L", "top5"), "ct+s6": ("j", "tau", "k", "rmax", "L", "top6"), "ct+s2+bot2": ("j", "tau", "k", "rmax", "L", "smax", "s2", "bot2"), "ct+cnt": ("j", "tau", "k", "rmax", "L", "nsm_ge")} def main(): K = int(sys.argv[1]); m0 = int(sys.argv[2]); thI = int(sys.argv[3]); thT = int(sys.argv[4]); cap = float(sys.argv[5]) fams = sys.argv[6:] eta = F(os.environ["ETA"]); alpha = F(os.environ["ALPHA"]); rule = os.environ.get("THETA_RULE", "half") thI_def = {"half": F(K, 2), "below_half": F((K - 1) // 2)}[rule] assert eta * K == m0 and alpha * K == thT and thI_def == thI, "PARAMS MISMATCH" log = print ts = time.time() log(f"PARAMS (checked): K = {K}; eta = {eta} -> m0 = {m0}; Type I thI = {thI} ({rule}); alpha = {alpha} -> thT = {thT}; " f"cap x <= {cap:g}; small <= {thT} < rough < {K / 2} <= large (grid units)", flush=True) U = B0.Universe(K, m0, thI, log=log) T = TwistRows(U, thT) shapes = list(B0.partitions_min(K, m0)) assert len(shapes) == U.S and shapes[U.P] == (K,) feats = None # v6: no per-shape feature dicts nrows = U.nI + T.nT DROP_DEAD = os.environ.get("DROP_DEAD", "0") == "1" ROWMAXLEN = int(os.environ["ROWMAXLEN"]) if os.environ.get("ROWMAXLEN") else None sizes_T = np.array([sum(v) for v in U.vrows[:T.nT]], dtype=np.int64) dead_T = (sizes_T >= K - 2 * thT) & (sizes_T <= thT) log(f" dead twisted rows (|v| in [{K - 2 * thT}, {thT}]): {int(dead_T.sum())} of {T.nT}; " f"{'DROPPED (control run)' if DROP_DEAD else 'enforced'}", flush=True) log(f" {U.S} shapes, {U.nI} Type I + {T.nT} twisted rows; features {time.time() - ts:.0f} s; RSS {B0.rss_gb():.2f} GB", flush=True) for fam in fams: t1 = time.time() if fam == "free": cls = np.arange(U.S, dtype=np.int64) else: keyf = FAMS[fam] index = {} cls = np.empty(U.S, dtype=np.int64) for i, u in enumerate(shapes): if i == U.P: key = ("PRIME",) else: ft = features(u, K, thT) key = tuple(ft[n] for n in keyf) cls[i] = index.setdefault(key, len(index)) del index nC = int(cls.max()) + 1 cP = int(cls[U.P]) assert np.sum(cls == cP) == 1 # v7: rows to build (a relaxation keeps a subset; the filter is applied DURING the build so dropped rows cost no memory) lens_all = np.array([len(v) for v in U.vrows], dtype=np.int64) bI = np.ones(U.nI, dtype=bool); bT = np.ones(T.nT, dtype=bool) if ROWMAXLEN is not None: bI &= lens_all <= ROWMAXLEN; bT &= lens_all[:T.nT] <= ROWMAXLEN if DROP_DEAD: bT &= ~dead_T A = csr_matrix((nrows, nC)) for r0, r1 in U.chunks(6_000_000): if not bI[r0:r1].any() and not (r0 < T.nT and bT[r0:min(r1, T.nT)].any()): continue p0, p1 = U.rowptr[r0], U.rowptr[r1] ids = U.ids[p0:p1] rows = np.repeat(np.arange(r0, r1, dtype=np.int64), U.rowlen[r0:r1]) w = U.coef_slice(p0, p1).astype(np.float64) * U.Bf[ids] mI = bI[rows] if mI.any(): A = A + coo_matrix((w[mI], (rows[mI], cls[ids[mI]])), shape=(nrows, nC)).tocsr() if r0 < T.nT: r1t = min(r1, T.nT); nt = int(U.rowptr[r1t] - p0) mT = bT[rows[:nt]] if mT.any(): A = A + coo_matrix((w[:nt][mT] * T.lam[ids[:nt][mT]], (U.nI + rows[:nt][mT], cls[ids[:nt][mT]])), shape=(nrows, nC)).tocsr() A = A.tocsr() A.eliminate_zeros() # row equilibration (rows are homogeneous equalities, so scaling them changes nothing): row r divided by max|A[r, :]| rowmax = np.asarray(abs(A).max(axis=1).todense()).ravel() keep_r = rowmax > 0 if DROP_DEAD: # control: drop the twisted rows with |v| in [K - 2 thT, thT] keep_r[U.nI + np.nonzero(dead_T)[0]] = False if ROWMAXLEN is not None: # v7 relaxation: only rows with len v <= ROWMAXLEN lens_v = np.array([len(v) for v in U.vrows], dtype=np.int64) keep_r[:U.nI] &= lens_v <= ROWMAXLEN keep_r[U.nI:] &= lens_v[:T.nT] <= ROWMAXLEN A = csr_matrix((1.0 / np.where(keep_r, rowmax, 1.0), (np.arange(nrows), np.arange(nrows))), shape=(nrows, nrows)) @ A A = A[np.nonzero(keep_r)[0]].tocsc() nrows_eff = A.shape[0] # column scaling: g_c = s_c * h_c with s_c = 1 / max|A[:, c]| colmax = np.asarray(abs(A).max(axis=0).todense()).ravel() s = np.where(colmax > 0, 1.0 / np.maximum(colmax, 1e-4), 1.0) # scale capped at 1e4 (v5) As = A @ csr_matrix((s, (np.arange(nC), np.arange(nC))), shape=(nC, nC)) lo = -np.ones(nC) / s hi = np.full(nC, cap - 1.0) / s hi[cP] = np.inf c = np.zeros(nC); c[cP] = s[cP] build = time.time() - t1 from scipy.optimize._highspy import _core Asc = As.tocsc(); Asc.sort_indices() meth = None; xsol = None for mode in os.environ.get("AGG_MODES", "ipm,simplex").split(","): hh = _core._Highs() for k_, v_ in (("output_flag", False), ("presolve", "off"), ("primal_feasibility_tolerance", 1e-10), ("dual_feasibility_tolerance", 1e-10), ("time_limit", float(os.environ.get("AGG_TL", "900")))): hh.setOptionValue(k_, v_) if mode == "ipm": hh.setOptionValue("solver", "ipm"); hh.setOptionValue("run_crossover", "off") else: hh.setOptionValue("solver", "simplex"); hh.setOptionValue("simplex_strategy", 1) INF = hh.getInfinity() hh.addRows(nrows_eff, np.zeros(nrows_eff), np.zeros(nrows_eff), 0, np.zeros(nrows_eff, dtype=np.int32), np.zeros(0, dtype=np.int32), np.zeros(0)) hh.addCols(nC, c, lo, np.where(np.isinf(hi), INF, hi), Asc.nnz, Asc.indptr[:-1].astype(np.int32), Asc.indices.astype(np.int32), Asc.data.astype(np.float64)) hh.run() st = hh.modelStatusToString(hh.getModelStatus()) if st == "Optimal": xsol = np.asarray(hh.getSolution().col_value, dtype=np.float64); meth = f"core-{mode}" break log(f" family {fam}: core {mode} status {st}; next", flush=True) if xsol is None: log(f"FAMILY {fam}: solver failed (all modes)", flush=True) continue class _R: pass r = _R(); r.x = xsol g_raw = r.x * s glo = np.full(nC, -1.0); ghi = np.full(nC, cap - 1.0); ghi[cP] = np.inf g = np.clip(g_raw, glo, ghi) # v5: judge the clipped point clip = float(np.max(np.abs(g - g_raw))) if nC else 0.0 val = 1.0 + float(g[cP]) res = float(np.max(np.abs(A @ g))) if nrows_eff else 0.0 # on the row-equilibrated matrix, at the clipped point comp = np.arange(nC) != cP log(f"FAMILY {fam}{' [DEAD ROWS DROPPED]' if DROP_DEAD else ''}{f' [RELAXED: rows with len v <= {ROWMAXLEN}, {nrows_eff} rows]' if ROWMAXLEN is not None else ''} (key {FAMS.get(fam, 'free')}): {nC} classes; C- = {val:.9f}{' ZERO (a fake in this family)' if (val <= 1e-7 and res <= 1e-9) else (' (value ~0 but residual above 1e-9: NOT a zero)' if val <= 1e-7 else '')}; clip {clip:.1e}; " f"phi in [{g.min():.6f}, {g[comp].max():.6f}]; max |row residual| {res:.2e}; build {build:.0f} s, solve {time.time() - t1 - build:.0f} s " f"({meth}); RSS {B0.rss_gb():.2f} GB", flush=True) if val <= 1e-7 and res <= 1e-9 and fam != "free": np.savez(f"agg_K{K}_m0{m0}_thI{thI}_thT{thT}_cap{cap:g}_{fam}{'_dropdead' if DROP_DEAD else ''}.npz", cls=cls, g=g) if __name__ == "__main__": main() ~~~~~~~~~~~~ #### file: verify_fake_v2.py - kind: script - as run: SHA-256 4972ae5a676dfcaef8831a10c81398ef0b15b5ef12c770dd168c656fb4892b2b - as shipped: SHA-256 ca7386df4cc08c0416d73c5858be5bb4388b661413fcc2757a8c5b4b3171cba0 - identity: identical, after dropping docstrings and comments, to the file as run with 2 declared change(s) of a machine-specific path, which now read: `sys.path.insert(0, __import__('os').path.dirname(__import__('os').path.abspath(__file__)))`; `sys.path.insert(0, __import__('os').path.dirname(__import__('os').path.abspath(__file__)))` - trailing newline: yes ~~~~~~~~~~~~ python """verify_fake_v2.py: checks a family solution of famrk.py with value zero (a fake of the family program, saved as .npz) against every row of the free twisted shape program, built with the stored (u, v) pairs of modm_otf_lowmem2.py's Universe as in twist_otf.py. The classes of the shapes are assigned with agg_lp7.py's features(), an implementation separate from famrk.py's, padded to famrk.py's class key. First prints the SHA-256 of every module it imports and of itself. Reports the bounds of phi, phi at the prime shape and, for each row, the residual sum_u c(u, v) B(u) [lam(u)] phi(u), absolute and relative to sum_u c(u, v) B(u) |phi(u)|. Floats. Usage: verify_fake_v2.py fake.npz K m0 thI thT cap r""" import sys, time import numpy as np sys.path.insert(0, __import__('os').path.dirname(__import__('os').path.abspath(__file__))) sys.path.insert(0, __import__('os').path.dirname(__import__('os').path.abspath(__file__))) import modm_otf_lowmem2 as B0 from twist_otf import TwistRows from agg_lp7 import features import famrk as FR import hashlib for _mod in (B0, sys.modules["twist_otf"], sys.modules["agg_lp7"], FR): print(f"module {_mod.__name__}: {_mod.__file__} sha256 {hashlib.sha256(open(_mod.__file__, 'rb').read()).hexdigest()}", flush=True) print(f"script {__file__} sha256 {hashlib.sha256(open(__file__, 'rb').read()).hexdigest()}", flush=True) fn = sys.argv[1]; K, m0, thI, thT = (int(a) for a in sys.argv[2:6]); cap = float(sys.argv[6]); r = int(sys.argv[7]) z = np.load(fn); cs = z["ccodes"]; g = z["g"] enc = FR.encoder(K, m0, thT, r) U = B0.Universe(K, m0, thI) T = TwistRows(U, thT) shapes = list(B0.partitions_min(K, m0)) def key(u): f = features(u, K, thT) top = tuple(f["S"][:r]) + (0,) * (r - min(r, f["j"])) return (f["j"], f["tau"], f["k"], f["rmax"], f["L"]) + top codes = enc(np.array([key(u) for u in shapes], dtype=np.int64)) ci = np.searchsorted(cs, codes); assert np.all(cs[np.minimum(ci, len(cs) - 1)] == codes), "a shape's class is missing" phi = g[ci] comp = U.lens >= 2 print(f"{fn}: {U.S} shapes, {len(cs)} classes; phi(prime) = {phi[U.P]:.12f}; composite phi in [{phi[comp].min():.9f}, {phi[comp].max():.9f}] " f"(bounds [-1, {cap - 1:g}]); phi >= -1 everywhere: {bool(np.all(phi >= -1.0))}", flush=True) res = np.zeros(U.nI); sc = np.zeros(U.nI); resT = np.zeros(T.nT); scT = np.zeros(T.nT) for q0, q1 in U.chunks(): p0, p1 = U.rowptr[q0], U.rowptr[q1] ids = U.ids[p0:p1] rows = np.repeat(np.arange(q0, q1, dtype=np.int64), U.rowlen[q0:q1]) w = U.coef_slice(p0, p1).astype(np.float64) * U.Bf[ids] res += np.bincount(rows, weights=w * phi[ids], minlength=U.nI) sc += np.bincount(rows, weights=w * np.abs(phi[ids]), minlength=U.nI) if q0 < T.nT: q1t = min(q1, T.nT); nt = int(U.rowptr[q1t] - p0) resT += np.bincount(rows[:nt], weights=w[:nt] * T.lam[ids[:nt]] * phi[ids[:nt]], minlength=U.nI)[:T.nT] scT += np.bincount(rows[:nt], weights=w[:nt] * np.abs(phi[ids[:nt]]), minlength=U.nI)[:T.nT] rel = np.abs(res) / np.maximum(sc, 1e-300); relT = np.abs(resT) / np.maximum(scT, 1e-300) print(f"VERIFY: Type I {U.nI} rows: max |residual| {np.abs(res).max():.2e}, max relative {rel.max():.2e}; twisted {T.nT} rows: max |residual| " f"{np.abs(resT).max():.2e}, max relative {relT.max():.2e}; v = () row residual {res[U.vrow0]:.2e}", flush=True) ~~~~~~~~~~~~ #### file: famrk.py - kind: script - as run: SHA-256 0a34eca9a07d00498e2b7f5539a92f549adb61529f225446fa36a555585c2126 - as shipped: SHA-256 55278b5fbc88fc604f208d2f5042db956c45a69f9790ac0652aa521d4bd1c91a - identity: identical, after dropping docstrings and comments, to the file as run with 1 declared change(s) of a machine-specific path, which now read: `sys.path.insert(0, __import__('os').path.dirname(__import__('os').path.abspath(__file__)))` - trailing newline: yes ~~~~~~~~~~~~ python """famrk.py: the lever-family LP of agg_lp7.py (ct+s_r families), built by EXACT ROW-KEY DEDUPLICATION from feature distributions, with no stored (u, v) pairs. Why. agg_lp6/7 aggregate the ~20k free-LP rows over classes; at K = 72 that matrix is nearly dense (> 5.6 GB for ct+s2). But for a family whose class key is ct+s_r = (j, tau, k, rmax, L, top_r(S)) (small <= thT < rough < K/2 <= large, grid units; top_r = the r largest small parts, padded with 0), row v of the family LP is Type I: sum_u B(u) c(u, v) g[cls(u)] = B(v) * sum_{w |- K - |v|, parts >= m0} B(w) g[cls(v + w)] twisted: (-1)^{len v} sum_u B(u) c(u, v) (-1)^{len u - len v} g[cls(u)] = B(v) * sum_w B(w) (-1)^{len w} g[cls(v + w)] (B(u) c(u, v) = B(v) B(u - v), the bijection u <-> w = u - v), and cls(v + w) depends on v only through the ROW KEY rho(v) = (n = K - |v|, j_v, tau_v, k_v, rmax_v, L_v, top_r(S_v)), because j, tau, k add, rmax and L are maxima, and top_r(S_v + S_w) = top_r(top_r(S_v) + top_r(S_w)). So rows with equal keys are equal up to the nonzero scalar B(v): keeping one per key gives the SAME feasible set (an exact reduction, not a relaxation). Each kept row is sum_omega D_n(omega) g[merge(rho, omega)] (Type I) and sum_omega Dp_n(omega) g[merge(rho, omega)] (twisted), with D_n(omega) = sum of B(w) and Dp_n(omega) = sum of B(w) (-1)^{len w} over w |- n (parts >= m0) with features omega (omega = (j, tau, k, rmax, L, top_r) of w). D and Dp come from a product DP: small parts (DP over sizes thT..m0, descending, so the first r parts appended are the top r), rough parts and large parts enumerated (B and the sign factor over the three disjoint size ranges). The classes are the omega of n = K (the v = () row); the prime (K) is its own class (j = k = 0, L = K). Rows: Type I for every v (parts >= m0) with |v| <= thI and K - |v| = 0 or >= m0 (as modm_otf_lowmem2.Universe); twisted for |v| <= thT. Dead twisted rows: |v| in [K - 2 thT, thT] (DROP_DEAD=1 drops them, a control). LP: min g_P, rows = 0, g in [-1, cap - 1] on composites, g_P >= -1; C- = 1 + min. Solved as agg_lp7 (row equilibration; column scaling 1/max|col| (capped 1e4); HiGHS core IPM without crossover, then the dual simplex; the point is CLIPPED into the bounds and the residual on the row-equilibrated matrix is recomputed there). Optional extra scaling passes (env RUIZ=n: n rounds of sqrt row/col equilibration). A family ZERO is (value <= 1e-7, residual <= 1e-9): a family fake, hence a free-LP fake, honest to solver tolerance (floats). Positive values are float diagnostics, as agg_lp7's. VERIFY=1: expand g to every shape and recompute the residual over ALL original free-LP rows with the stored pairs (modm_otf_lowmem2 Universe), row-equilibrated as agg_lp7: the cross-check of the deduplication (small K only, or with memory to spare). Usage: famrk.py K m0 thI thT cap fam1 [fam2 ...] fam in ct, ct+smax, ct+s2, ct+s3, ct+s4, ct+s5, ct+s6 (env ETA, ALPHA, THETA_RULE as agg_lp7, checked; AGG_MODES (ipm,simplex), AGG_TL (900), DROP_DEAD, RUIZ (0), VERIFY (0), SAVE (1)) """ import math import os import sys import time from collections import defaultdict from fractions import Fraction as F import numpy as np from scipy.sparse import coo_matrix, csc_matrix, csr_matrix sys.path.insert(0, __import__('os').path.dirname(__import__('os').path.abspath(__file__))) import modm_otf_lowmem2 as B0 # noqa: E402 time.time = time.monotonic FAMR = {"ct": 0, "ct+smax": 1, "ct+s2": 2, "ct+s3": 3, "ct+s4": 4, "ct+s5": 5, "ct+s6": 6} def feat(parts, K, thT, r): """the agg_lp7 class key of a multiset (any order), as a tuple (j, tau, k, rmax, L, top_1..top_r padded with 0).""" p = sorted(parts, reverse=True) sm = [x for x in p if x <= thT] ro = [x for x in p if thT < x and 2 * x < K] la = [x for x in p if 2 * x >= K] top = tuple(sm[:r]) + (0,) * (r - min(r, len(sm))) return (len(sm), sum(sm), len(ro), ro[0] if ro else 0, la[0] if la else 0) + top class Dist: """D_n(omega), Dp_n(omega) for n in [nmin, K] (product DP).""" def __init__(self, K, m0, thT, r, nmin, jmax=None): """jmax: keep only small states with j <= jmax (for restrictions to classes with j <= jmax: merged j = j_v + j_w >= j_w).""" self.K, self.m0, self.thT, self.r = K, m0, thT, r JM = K if jmax is None else jmax # small parts: sizes thT .. m0 descending st = {(0, 0, ()): (1.0, 1.0)} for x in range(thT, m0 - 1, -1): new = defaultdict(lambda: [0.0, 0.0]) for (tau, j, top), (w, wp) in st.items(): m = 0; fac = 1.0 while tau + m * x <= K and j + m <= JM: key = (tau + m * x, j + m, top + (x,) * min(m, r - len(top))) a = new[key]; a[0] += w * fac; a[1] += wp * fac * (-1) ** m m += 1; fac /= (x * m) st = {k_: (v_[0], v_[1]) for k_, v_ in new.items()} self.nsmall = len(st) # rough parts: sizes in (thT, K/2); large parts: sizes in [K/2, K] rough_sizes = [x for x in range(thT + 1, K + 1) if 2 * x < K] large_sizes = [x for x in range(m0, K + 1) if 2 * x >= K] def multisets(sizes, cap): out = [] def rec(i, tot, parts): out.append((tot, tuple(parts))) for t in range(i, len(sizes)): x = sizes[t] if tot + x <= cap: rec(t, tot + x, parts + [x]) rec(0, 0, []) return out def bw(parts): d = 1.0 for x in parts: d *= x for mlt in {x: parts.count(x) for x in parts}.values(): d *= math.factorial(mlt) return 1.0 / d, (-1.0) ** len(parts) RL = [] for tr, pr in multisets(rough_sizes, K): for tl, pl in multisets(large_sizes, K - tr): b1, s1 = bw(list(pr)); b2, s2 = bw(list(pl)) RL.append((tr + tl, len(pr), max(pr) if pr else 0, max(pl) if pl else 0, b1 * b2, s1 * s2)) self.nRL = len(RL) # combine: D[(n, j, tau, k, rmax, L, top)] for n in [nmin, K] S_tau = np.array([k_[0] for k_ in st], dtype=np.int64) S_j = np.array([k_[1] for k_ in st], dtype=np.int64) S_top = np.array([k_[2] + (0,) * (r - len(k_[2])) for k_ in st], dtype=np.int64).reshape(len(st), r) S_w = np.array([v_[0] for v_ in st.values()]); S_wp = np.array([v_[1] for v_ in st.values()]) # vectorised (v2.1): per (k, rmax, L) combo, the entries (n, j, tau, k, rmax, L, top) with n = tau + tot; codes in a mixed radix rad = [K + 1, K // m0 + 1, K + 1, max(1, K // (thT + 1)) + 1, K // 2 + 1, K + 1] + [thT + 1] * r assert math.prod(rad) < 2 ** 62, "omega code overflow" mult = np.array([math.prod(rad[i + 1:]) for i in range(len(rad))], dtype=np.int64) base_small = S_j * mult[1] + S_tau * mult[2] + (S_top @ mult[6:] if r else 0) codes, ws, wps = [], [], [] for (tot, k, rmax, L, b, sg) in RL: n = S_tau + tot sel = np.nonzero((n >= nmin) & (n <= K))[0] if len(sel) == 0: continue codes.append(n[sel] * mult[0] + base_small[sel] + k * mult[3] + rmax * mult[4] + L * mult[5]) ws.append(S_w[sel] * b); wps.append(S_wp[sel] * (b * sg)) codes = np.concatenate(codes); ws = np.concatenate(ws); wps = np.concatenate(wps) uc, inv = np.unique(codes, return_inverse=True) Dw = np.bincount(inv, weights=ws); Dpw = np.bincount(inv, weights=wps) del codes, ws, wps, inv arr = np.empty((len(uc), 6 + r), dtype=np.int64) rem = uc.copy() for i in range(6 + r): arr[:, i] = rem // mult[i]; rem = rem % mult[i] self.byn = {} for n in range(nmin, K + 1): sel = arr[:, 0] == n self.byn[n] = (arr[sel, 1:], Dw[sel], Dpw[sel]) self.nD = len(uc) def encoder(K, m0, thT, r): kmax = max(1, K // (thT + 1)) rad = [K // m0 + 1, K + 1, kmax + 1, K // 2 + 1, K + 1] + [thT + 1] * r assert math.prod(rad) < 2 ** 62, "class code overflow" mult = np.array([math.prod(rad[i + 1:]) for i in range(len(rad))], dtype=np.int64) return lambda M: M @ mult def merge(rho, Om, r): """class keys of v + w for the row key rho = (j, tau, k, rmax, L, top...) and the omega rows Om (n_omega x (5 + r)).""" out = np.empty_like(Om) out[:, 0] = Om[:, 0] + rho[0]; out[:, 1] = Om[:, 1] + rho[1]; out[:, 2] = Om[:, 2] + rho[2] out[:, 3] = np.maximum(Om[:, 3], rho[3]); out[:, 4] = np.maximum(Om[:, 4], rho[4]) if r: tops = np.concatenate([Om[:, 5:], np.broadcast_to(np.array(rho[5:], dtype=np.int64), (len(Om), r))], axis=1) tops = -np.sort(-tops, axis=1) out[:, 5:] = tops[:, :r] return out def build_family(K, m0, thI, thT, r, V, drop_dead, log, jmax=None, rowjmax=None): """the deduplicated family LP: returns dict(A = CSR rows (Type I keys, then twisted keys) x classes, csorted, mass, cP, enc, ...). jmax (RESTRICTION): only classes with j <= jmax (all others fixed at g = 0) and rows with j_v <= jmax; the dropped rows meet only classes with j >= j_v > jmax, so they vanish identically: a zero of this LP is a family fake. rowjmax (RELAXATION): only rows with j_v <= rowjmax, all classes: its duals give a valid lower bound for the full family.""" t1 = time.time() Dd = Dist(K, m0, thT, r, K - thI, jmax) enc = encoder(K, m0, thT, r) Om_K, D_K, _ = Dd.byn[K] if jmax is not None: assert Om_K[:, 0].max() <= jmax ccode = enc(Om_K) order = np.argsort(ccode); csorted = ccode[order] assert np.all(csorted[1:] != csorted[:-1]) nC = len(ccode) mass = D_K[order] # class c (sorted order) mass = sum of B(u) over its shapes Pkey = np.array([[0, 0, 0, 0, K] + [0] * r], dtype=np.int64) cP = int(np.searchsorted(csorted, enc(Pkey)[0])); assert csorted[cP] == enc(Pkey)[0] assert abs(mass[cP] - 1.0 / K) < 1e-15 rkI, rkT = {}, {} RJ = min(x for x in (jmax, rowjmax, K) if x is not None) for v in V: key = (K - sum(v),) + feat(v, K, thT, r) if key[1] > RJ: continue rkI.setdefault(key, v) if sum(v) <= thT: dead = K - 2 * thT <= sum(v) <= thT if not (drop_dead and dead): rkT.setdefault(key, v) C_, V_, L_ = [], [], [] # CSR built row by row (int32 column indices: 12 bytes per entry) for typ, rk in (("I", rkI), ("T", rkT)): for key in rk: n = key[0]; rho = key[1:] Om, Dw, Dpw = Dd.byn[n] Mg = merge(rho, Om, r) if jmax is not None: okj = Mg[:, 0] <= jmax Mg, Dw, Dpw = Mg[okj], Dw[okj], Dpw[okj] mc = enc(Mg) cls = np.searchsorted(csorted, mc) assert np.all(csorted[np.minimum(cls, nC - 1)] == mc), f"merged class missing for row key {key}" u_, inv = np.unique(cls, return_inverse=True) vals = np.bincount(inv, weights=Dw if typ == "I" else Dpw) nzm = vals != 0.0 C_.append(u_[nzm].astype(np.int32)); V_.append(vals[nzm]); L_.append(int(nzm.sum())) nrow = len(L_) indptr = np.concatenate([[0], np.cumsum(L_)]).astype(np.int64) A = csr_matrix((np.concatenate(V_), np.concatenate(C_), indptr), shape=(nrow, nC)) del C_, V_, L_ r0 = list(rkI).index((K,) + feat((), K, thT, r)) assert r0 == 0 and np.allclose(A[0].toarray().ravel(), mass, rtol=1e-12, atol=0), "row 0 is not v = () = the class mass" keys = [("I",) + k_ for k_ in rkI] + [("T",) + k_ for k_ in rkT] # row order of A: (type, n, j, tau, k, rmax, L, top...) return dict(A=A, csorted=csorted, mass=mass, cP=cP, enc=enc, nC=nC, nrI=len(rkI), nrT=len(rkT), Dd=Dd, build=time.time() - t1, keys=keys, cfeat=Om_K[order]) # class features (j, tau, k, rmax, L, top...) in A's column order def scale(A, cap, mode="max"): """row equilibration in place (rows are homogeneous: no change to the LP), CSC, then column scaling sc_ (g = sc_ * h): mode "max": agg_lp7's 1/max|col| capped at 1e4; mode "ruiz:N": N rounds of Ruiz equilibration (rows and columns by 1/sqrt(max)), the row factors folded into A (A stays a row-scaled copy of the true rows: residuals are judged on it, with rows max-equilibrated afterwards so that the 1e-9 residual test keeps its meaning) and the column factors returned in sc_ (uncapped).""" rowmax = np.maximum.reduceat(np.abs(A.data), A.indptr[:-1]) assert np.all(np.diff(A.indptr) > 0), "an empty row" A.data /= np.repeat(rowmax, np.diff(A.indptr)) A = A.tocsc(); A.sort_indices() nC = A.shape[1] nzc = np.diff(A.indptr) > 0 def colmax_of(M): cm = np.zeros(nC); cm[nzc] = np.maximum.reduceat(np.abs(M.data), M.indptr[:-1][nzc]); return cm colmax = colmax_of(A) if mode == "max": sc_ = np.where(colmax > 0, 1.0 / np.maximum(colmax, 1e-4), 1.0) return (A, A), sc_, colmax assert mode.startswith("ruiz"), f"unknown scaling {mode}" nit = int(mode[4:].lstrip(":")) sc_ = np.ones(nC) W = A.copy() # W = D_r A diag(sc_): HiGHS gets A_hi = D_r A (its rows only rescaled) rtot = np.ones(A.shape[0]) for _ in range(nit): cm = colmax_of(W); cf = np.where(cm > 0, 1.0 / np.sqrt(cm), 1.0) W.data *= np.repeat(cf, np.diff(W.indptr)); sc_ *= cf Wr = W.tocsr(); rm = np.maximum.reduceat(np.abs(Wr.data), Wr.indptr[:-1]); rf = 1.0 / np.sqrt(rm) W.data *= rf[W.indices]; rtot *= rf A_hi = A.copy(); A_hi.data *= rtot[A_hi.indices] return (A, A_hi), sc_, colmax_of(A) def solve_family(A, sc_, colmax, cP, cap, modes, log, tag, smv=1e-12, ptol=1e-10, tl=900.0, ccut=0.0, polish_tl=1800.0, mass=None, mcut=0.0): """solve min g_P on the scaled family LP; returns dict(val, res, lb, g, meth). A: row-equilibrated CSC (the TRUE rows, for residuals). ccut > 0: classes with colmax < ccut are FIXED at g = 0 (a restriction: a zero stays a family fake). lb: the float Lagrangian bound of the FULL family LP (all classes, true matrix) from the solver's row duals: valid for any duals. mcut > 0: classes with mass (sum of B(u) over the class) < mcut are also FIXED at 0 (a restriction).""" from scipy.optimize._highspy import _core A, A_hi = A # (true rows max-equilibrated: residuals; row-rescaled copy for the solver) nre, nC = A.shape act = colmax >= ccut if mcut > 0: act &= mass >= mcut act[cP] = True ia = np.nonzero(act)[0] Asub = A_hi[:, ia] if not act.all() else A_hi sa = sc_[ia] Aw = csc_matrix((Asub.data * np.repeat(sa, np.diff(Asub.indptr)), Asub.indices, Asub.indptr), shape=Asub.shape) lo = -np.ones(len(ia)) / sa hi = np.full(len(ia), cap - 1.0) / sa jP = int(np.searchsorted(ia, cP)); assert ia[jP] == cP hi[jP] = np.inf c = np.zeros(len(ia)); c[jP] = sa[jP] glo = np.full(nC, -1.0); ghi = np.full(nC, cap - 1.0); ghi[cP] = np.inf out = None for mode in modes: hh = _core._Highs() opts = [("output_flag", False), ("presolve", "on" if "+pre" in mode else "off"), ("primal_feasibility_tolerance", ptol), ("dual_feasibility_tolerance", ptol), ("time_limit", tl), ("small_matrix_value", smv)] if mode.startswith("ipm"): opts += [("solver", "ipm"), ("run_crossover", "on" if mode.startswith("ipmx") else "off")] elif mode.startswith("psimplex"): opts += [("solver", "simplex"), ("simplex_strategy", 4)] else: opts += [("solver", "simplex"), ("simplex_strategy", 1)] for k_, v_ in opts: assert str(hh.setOptionValue(k_, v_)).endswith("kOk"), f"HiGHS option {k_} = {v_} rejected" INF = hh.getInfinity() hh.addRows(nre, np.zeros(nre), np.zeros(nre), 0, np.zeros(nre, dtype=np.int32), np.zeros(0, dtype=np.int32), np.zeros(0)) hh.addCols(len(ia), c, lo, np.where(np.isinf(hi), INF, hi), Aw.nnz, Aw.indptr[:-1].astype(np.int32), Aw.indices.astype(np.int32), Aw.data.astype(np.float64)) t0 = time.time() hh.run() st = hh.modelStatusToString(hh.getModelStatus()) if st != "Optimal": log(f" {tag}: {mode} status {st} ({time.time() - t0:.0f} s); next", flush=True) continue sol = hh.getSolution() h = np.asarray(sol.col_value, dtype=np.float64) y = np.asarray(sol.row_dual, dtype=np.float64) g = np.zeros(nC); g[ia] = np.clip(h * sa, glo[ia], ghi[ia]) clip = float(np.max(np.abs(g[ia] - h * sa))) val = 1.0 + float(g[cP]); res = float(np.max(np.abs(A @ g))) meth = mode if val <= 1e-7 and res > 1e-9 and mode.startswith("ipm") and polish_tl > 0: hh.clearSolver(); hh.setOptionValue("solver", "simplex"); hh.setOptionValue("simplex_strategy", 1) hh.setOptionValue("time_limit", polish_tl); hh.run() stp = hh.modelStatusToString(hh.getModelStatus()) if stp == "Optimal": hp = np.asarray(hh.getSolution().col_value, dtype=np.float64) gp = np.zeros(nC); gp[ia] = np.clip(hp * sa, glo[ia], ghi[ia]) valp = 1.0 + float(gp[cP]); resp = float(np.max(np.abs(A @ gp))) log(f" {tag}: simplex polish value {valp:.3e}, residual {resp:.2e}", flush=True) if valp <= 1e-7 and resp <= 1e-9: g, val, res, clip, meth = gp, valp, resp, float(np.max(np.abs(gp[ia] - hp * sa))), mode + "+polish" else: log(f" {tag}: simplex polish status {stp}", flush=True) # Lagrangian bound of the FULL family LP (g-space: rows = A (row-equilibrated, true), all classes): for any row duals y, # min g_P >= sum_c min over [glo_c, ghi_c] of d_c g_c, d = e_P - A^T y; d_P < 0 is repaired on row 0 (the mass row, A[0, c] > 0). # The solver's duals live in the column-scaled problem: A_w = A diag(sc), so y is also a dual vector for A. d = -(A_hi.T @ y); d[cP] += 1.0 row0 = np.asarray(A_hi[0, :].todense()).ravel() if d[cP] < 0: d = d + (-d[cP] / row0[cP]) * row0 lbt = np.where(d >= 0, d * glo, d * ghi); lbt[cP] = d[cP] * glo[cP] lb = 1.0 + float(np.sum(lbt)) out = dict(val=val, res=res, lb=lb, g=g, clip=clip, meth=meth, nact=len(ia)) break return out def main(): K = int(sys.argv[1]); m0 = int(sys.argv[2]); thI = int(sys.argv[3]); thT = int(sys.argv[4]); cap = float(sys.argv[5]) fams = sys.argv[6:] eta = F(os.environ["ETA"]); alpha = F(os.environ["ALPHA"]); rule = os.environ.get("THETA_RULE", "half") thI_def = {"half": F(K, 2), "below_half": F((K - 1) // 2)}[rule] assert eta * K == m0 and alpha * K == thT and thI_def == thI, "PARAMS MISMATCH" DROP_DEAD = os.environ.get("DROP_DEAD", "0") == "1" VERIFY = os.environ.get("VERIFY", "0") == "1" SAVE = os.environ.get("SAVE", "1") == "1" MODES = os.environ.get("AGG_MODES", "ipm,simplex").split(",") SMV = float(os.environ.get("SMV", "1e-12")); PTOL = float(os.environ.get("PTOL", "1e-10")); TL = float(os.environ.get("AGG_TL", "900")) CCUT = float(os.environ.get("CCUT", "0")); MCUT = float(os.environ.get("MCUT", "0")); POLISH_TL = float(os.environ.get("POLISH_TL", "1800")) log = print log(f"famrk PARAMS (checked): K = {K}; eta = {eta} -> m0 = {m0}; Type I thI = {thI} ({rule}); alpha = {alpha} -> thT = {thT}; " f"cap x <= {cap:g}; small <= {thT} < rough < {K / 2} <= large (grid units){'; DEAD ROWS DROPPED' if DROP_DEAD else ''}; " f"solver modes {MODES}, small_matrix_value {SMV:g}, tolerances {PTOL:g}, class cuts colmax {CCUT:g} mass {MCUT:g} (classes below are fixed at 0)", flush=True) ks = [k for k in range(0, thI + 1) if (k == 0 or k >= m0) and (K - k == 0 or K - k >= m0)] V = [v for k in ks for v in B0.partitions_min(k, m0)] nI = len(V); nT = sum(1 for v in V if sum(v) <= thT) log(f" {nI} Type I rows, {nT} twisted rows (dead: |v| in [{K - 2 * thT}, {thT}]) before deduplication", flush=True) U = None for fam in fams: t1 = time.time() r = FAMR[fam] JMAX = int(os.environ["JMAX"]) if os.environ.get("JMAX") else None ROWJMAX = int(os.environ["ROWJMAX"]) if os.environ.get("ROWJMAX") else None bf = build_family(K, m0, thI, thT, r, V, DROP_DEAD, log, JMAX, ROWJMAX) if JMAX is not None or ROWJMAX is not None: log(f" family {fam}: {'RESTRICTION to classes with j <= ' + str(JMAX) + ' (a zero is a family fake)' if JMAX is not None else ''}" f"{'; ' if JMAX is not None and ROWJMAX is not None else ''}{'RELAXATION to rows with j_v <= ' + str(ROWJMAX) + ' (the Lagrangian bound is valid; a zero is inconclusive)' if ROWJMAX is not None else ''}", flush=True) A, nC, cP, csorted, enc, Dd = bf["A"], bf["nC"], bf["cP"], bf["csorted"], bf["enc"], bf["Dd"] log(f" family {fam}: {Dd.nsmall} small states, {Dd.nRL} rough/large combos, {Dd.nD} omega-entries; {nC} classes; " f"deduplicated rows: {bf['nrI']} Type I (of {nI}) + {bf['nrT']} twisted (of {nT}); nnz {A.nnz} " f"({A.nnz / max(1, A.shape[0] * nC):.3f} dense); build {bf['build']:.0f} s; RSS {B0.rss_gb():.2f} GB", flush=True) if os.environ.get("BUILD_ONLY", "0") == "1": continue A, sc_, colmax = scale(A, cap, os.environ.get("SCALE", "max")) nre = A[0].shape[0] o = solve_family(A, sc_, colmax, cP, cap, MODES, log, f"family {fam}", smv=SMV, ptol=PTOL, tl=TL, ccut=CCUT, polish_tl=POLISH_TL, mass=bf["mass"], mcut=MCUT) if o is None: log(f"FAMILY {fam}: solver failed (all modes)", flush=True) continue g, val, res = o["g"], o["val"], o["res"] comp = np.arange(nC) != cP zero = val <= 1e-7 and res <= 1e-9 log(f"FAMILY {fam}{' [DEAD ROWS DROPPED]' if DROP_DEAD else ''} (famrk, key j,tau,k,rmax,L,top{r}): {nC} classes ({o['nact']} free), " f"{nre} rows; C- = {val:.9f}{' ZERO (a fake in this family)' if zero else (' (value ~0 but residual above 1e-9: NOT a zero)' if val <= 1e-7 else '')}; " f"Lagrangian bound (float, {'of this RESTRICTED LP only' if JMAX is not None else 'full family'}) {o['lb']:.9f}; clip {o['clip']:.1e}; phi in [{g.min():.6f}, {g[comp].max():.6f}]; " f"max |row residual| {res:.2e}; build {bf['build']:.0f} s, solve {time.time() - t1 - bf['build']:.0f} s ({o['meth']}); " f"RSS {B0.rss_gb():.2f} GB", flush=True) if SAVE: np.savez(f"famrk_K{K}_m0{m0}_thI{thI}_thT{thT}_cap{cap:g}_{fam}{'_dropdead' if DROP_DEAD else ''}" f"{'_J' + str(JMAX) if JMAX is not None else ''}{'_RJ' + str(ROWJMAX) if ROWJMAX is not None else ''}.npz", ccodes=csorted, g=g, val=val, res=res, lb=o["lb"], r=r) if VERIFY: from twist_otf import TwistRows if U is None: U = B0.Universe(K, m0, thI, log=log) Tw = TwistRows(U, thT) shapes = list(B0.partitions_min(K, m0)) codes = enc(np.array([feat(u, K, thT, r) for u in shapes], dtype=np.int64)) ci = np.searchsorted(csorted, codes); assert np.all(csorted[ci] == codes) phi = g[ci] dead_T = np.array([K - 2 * thT <= sum(v) <= thT for v in U.vrows[:Tw.nT]]) resI = np.zeros(U.nI); resT = np.zeros(Tw.nT); rmI = np.zeros(U.nI); rmT = np.zeros(Tw.nT) for q0, q1 in U.chunks(): p0, p1 = U.rowptr[q0], U.rowptr[q1] ids = U.ids[p0:p1] rows = np.repeat(np.arange(q0, q1, dtype=np.int64), U.rowlen[q0:q1]) w = U.coef_slice(p0, p1).astype(np.float64) * U.Bf[ids] resI += np.bincount(rows, weights=w * phi[ids], minlength=U.nI) Mi = coo_matrix((w, (rows, ci[ids])), shape=(U.nI, nC)).tocsr() # the row's class aggregation (for its max) rmI = np.maximum(rmI, np.asarray(abs(Mi).max(axis=1).todense()).ravel()) if q0 < Tw.nT: q1t = min(q1, Tw.nT); nt = int(U.rowptr[q1t] - p0) wt = w[:nt] * Tw.lam[ids[:nt]] resT += np.bincount(rows[:nt], weights=wt * phi[ids[:nt]], minlength=U.nI)[:Tw.nT] Mt = coo_matrix((wt, (rows[:nt], ci[ids[:nt]])), shape=(U.nI, nC)).tocsr() rmT = np.maximum(rmT, np.asarray(abs(Mt).max(axis=1).todense()).ravel()[:Tw.nT]) eqI = np.abs(resI) / np.where(rmI > 0, rmI, 1.0) eqT = np.abs(resT) / np.where(rmT > 0, rmT, 1.0) if DROP_DEAD: eqT = eqT[~dead_T] log(f" VERIFY {fam}: family point expanded to all {U.S} shapes; max row-equilibrated residual over ALL original rows: " f"Type I {eqI.max():.2e} ({U.nI} rows), twisted {eqT.max() if len(eqT) else 0:.2e} ({len(eqT)} rows); " f"phi(prime) + 1 = {1.0 + phi[U.P]:.9f}", flush=True) if __name__ == "__main__": main() ~~~~~~~~~~~~ #### file: rigfam.py - kind: script - as run: SHA-256 1a212ad4192861e1142ba232b1d04b180e32c306c819baa365201f1c928ee545 - as shipped: SHA-256 8e70b862363eb671b91e37d271fc43dd701cc2f2b29f0ea919be90013f2bfa80 - identity: identical, after dropping docstrings and comments, to the file as run with 3 declared change(s) of a machine-specific path or of names in a message, which now read: `sys.path.insert(0, __import__('os').path.dirname(__import__('os').path.abspath(__file__)))`; `sys.path.insert(0, __import__('os').path.dirname(__import__('os').path.abspath(__file__)))`; `(continuations trusted saved state; review defects D1, D2, D4)` - trailing newline: yes ~~~~~~~~~~~~ python """rigfam.py: a computer-assisted proof that a lever-family programme of famrk.py has an exactly prime-free point near a floating-point one. The feature distributions are computed exactly: every B(w) = 1/d(w) with d(w) = prod_x x^{m_x} m_x!, and with Q a common multiple of all d(w) (|w| <= K, parts >= m0, Q found by a knapsack over prime valuations) the product dynamic programme of famrk.Dist is redone in Python integers, so that D_ex(n, omega) = Q D_n(omega) exactly. The floating-point point is rounded to the 2^-60 grid with the prime value set to -1 and its residual computed exactly in integers; a basis is chosen; a verified solve with a priori gamma_n bounds encloses the exact correction on the basis; and the corrected point is checked exactly against the bounds. Usage: python rigfam.py verify K m0 thI thT cap r J (J = full for the whole family), with the environment variables FAKE (a floating-point starting point) and BASIS (a basis hint), both untrusted. On success it writes rigfam_cert__.npz. certify_core is an internal function. An independent review did not establish this file as a sound general verifier; the two certificates of section 8 rest on the separately written checkers v4_exact_grid.py, v4_independent_K36_bound.py and v4_independent_K72.py.""" import math import sys from collections import defaultdict from fractions import Fraction import numpy as np sys.path.insert(0, __import__('os').path.dirname(__import__('os').path.abspath(__file__))) sys.path.insert(0, __import__('os').path.dirname(__import__('os').path.abspath(__file__))) def vp(n, p): e = 0 while n % p == 0: n //= p; e += 1 return e def vp_fact(m, p): e, q = 0, p while q <= m: e += m // q; q *= p return e def common_Q(K, m0): primes = [p for p in range(2, K + 1) if all(p % q for q in range(2, int(p ** 0.5) + 1))] Q = 1 for p in primes: f = [0] * (K + 1) # f[t] = max v_p(d(w)) over multisets with total exactly t (or -1) f = [-1] * (K + 1); f[0] = 0 for x in range(m0, K + 1): g = f[:] for t in range(K + 1): if f[t] < 0: continue m = 1 while t + m * x <= K: val = f[t] + m * vp(x, p) + vp_fact(m, p) if val > g[t + m * x]: g[t + m * x] = val m += 1 f = g Q *= p ** max(f) return Q class DistExact: """exact integer version of famrk.Dist: byn[n] = (Om (n_omega x (5 + r)) int64, D_ex list of ints, Dp_ex list of ints), Q.""" def __init__(self, K, m0, thT, r, nmin, jmax=None, Q=None): self.K, self.m0, self.thT, self.r = K, m0, thT, r self.Q = Q if Q is not None else common_Q(K, m0) JM = K if jmax is None else jmax st = {(0, 0, ()): (self.Q, self.Q)} for x in range(thT, m0 - 1, -1): new = defaultdict(lambda: [0, 0]) for (tau, j, top), (w, wp) in st.items(): m = 0; den = 1 while tau + m * x <= K and j + m <= JM: assert w % den == 0 and wp % den == 0 key = (tau + m * x, j + m, top + (x,) * min(m, r - len(top))) a = new[key]; a[0] += w // den; a[1] += (wp // den) * (-1) ** m m += 1; den *= x * m st = {k_: (v_[0], v_[1]) for k_, v_ in new.items()} rough_sizes = [x for x in range(thT + 1, K + 1) if 2 * x < K] large_sizes = [x for x in range(m0, K + 1) if 2 * x >= K] def multisets(sizes, cap): out = [] def rec(i, tot, parts): out.append((tot, tuple(parts))) for t in range(i, len(sizes)): if tot + sizes[t] <= cap: rec(t, tot + sizes[t], parts + [sizes[t]]) rec(0, 0, []) return out def dden(parts): d = math.prod(parts) if parts else 1 for x in set(parts): d *= math.factorial(parts.count(x)) return d RL = [] for tr, pr in multisets(rough_sizes, K): for tl, pl in multisets(large_sizes, K - tr): RL.append((tr + tl, len(pr), max(pr) if pr else 0, max(pl) if pl else 0, dden(list(pr)) * dden(list(pl)), (-1) ** (len(pr) + len(pl)))) D = defaultdict(lambda: [0, 0]) bytau = defaultdict(list) # small states bucketed by their total (only n = tau + tot in range is used) for (tau, j, top), (w, wp) in st.items(): bytau[tau].append((j, top + (0,) * (r - len(top)), w, wp)) for (tot, k, rmax, L, den, sg) in RL: for tau in range(max(0, nmin - tot), K - tot + 1): for (j, topp, w, wp) in bytau.get(tau, ()): assert w % den == 0 and wp % den == 0 a = D[(tau + tot, j, tau, k, rmax, L) + topp] a[0] += w // den; a[1] += (wp // den) * sg self.byn = {} keys = sorted(D.keys()) arr = np.array(keys, dtype=np.int64).reshape(len(keys), 6 + r) for n in range(nmin, K + 1): idx = np.nonzero(arr[:, 0] == n)[0] self.byn[n] = (arr[idx, 1:], [D[keys[i]][0] for i in idx], [D[keys[i]][1] for i in idx]) self.nD = len(keys) def rounded(self, n): """correctly rounded binary64 D and Dp for row length n (Python int / int true division rounds correctly).""" Om, De, Dpe = self.byn[n] return Om, np.array([d / self.Q for d in De]), np.array([d / self.Q for d in Dpe]) U_ROUND = 2.0 ** -53 UNDERFLOW = 2.0 ** -1074 def gam(n): """gamma_n = n u / (1 - n u) for round-to-nearest binary64 (n u < 1/2 enforced).""" assert 0 <= n < 2 ** 40 return n * U_ROUND / (1.0 - n * U_ROUND) # ============================================================================ verification (v2) # REPAIRS (the numbered defects of an independent review): # D1/D2: the certificate is decided by certify_core(), which checks EVERY class in exact rational arithmetic: the prime coordinate is # exactly -1 and not in the basis; every off-basis class lies in [-1, cap - 1]; every basis class's outward endpoints # g + x~ -/+ eps (Fractions) lie in [-1, cap - 1]. No entry point can skip it. # D3: endpoints and eps are exact Fractions (eps = zmax / (1 - nE) with zmax, nE rigorous float upper bounds). # D4: the continuation entry points basis_from / minnorm_from and the PREP_OUT certificate path are REMOVED. The only certifying # entry point is `verify K m0 thI thT cap r J`, which recomputes the exact DP, the rows, the exact residual and all enclosures # from the parameters. A basis may be given as a HINT (env BASIS, e.g. from select_from); it only selects columns and is # validated (count, uniqueness, range, prime excluded). The float starting point (env FAKE or the LP solve) is untrusted input. # A complete certificate file is written (parameters, Q, the dyadic point, the exact residual integers, the basis, x~, eps, # nE, source shas). # D5: the parameter domain is enforced (check_domain); all floating inputs must be finite; power-of-two row scaling must stay in # the normal range; an absolute underflow allowance TINY is added to every a priori bound; gam() asserts n u < 1/2. # D6: degenerate cases (no candidates, singular or invalid basis, failed inversion) return NOT VERIFIED instead of crashing. # The minimum-norm path is disabled (it was never used for a result). # ARITHMETIC CONTRACT: IEEE binary64, round-to-nearest-even, subnormals preserved (environment logged); a priori bounds # |fl(x.y) - x.y| <= gam(n) |x|.|y| + n 2^-1074 for length-n dot products (any summation order, FMA or not); scaling by powers of two # is exact when results stay normal (asserted). def check_domain(K, m0, thI, thT, cap, r, J): """the supported parameter domain (D5). Returns None if supported, else the reason.""" for name, x in (("K", K), ("m0", m0), ("thI", thI), ("thT", thT), ("cap", cap), ("r", r)): if int(x) != x: return f"{name} = {x} is not an integer" if not (2 <= K <= 200): return "K outside [2, 200]" if not (1 <= m0 <= thT): return "need 1 <= m0 <= thT (small sizes [m0, thT] nonempty; rough sizes start at thT + 1 > m0 - 1)" if not (2 * thT < K): return "need 2 thT < K (small and large size ranges disjoint)" if not (thT <= thI and 2 * thI <= K): return "need thT <= thI <= K/2 (twisted moduli are a subset of the Type I moduli; Type I level at most 1/2)" if not (1 <= cap <= 2 ** 53): return "need 1 <= cap <= 2^53 (g = 0 must satisfy the bounds of omitted classes; cap - 1 must be an exact binary64 integer)" if not (0 <= r <= 6): return "need 0 <= r <= 6" if J is not None and not (int(J) == J and J >= 0): return "J must be None (full family) or an integer >= 0" return None def certify_core(Bs, dBs, rs, drs, g, S, cP, cap, log=print): """INTERNAL (not a public entry point; the supported interface is the `verify` command). Validates and converts every argument, then runs _certify_core_checked; any exception on the certifying path is returned as NOT VERIFIED. Accepted inputs: Bs, dBs scipy sparse matrices of dtype float64 (square, same shape, finite, dBs >= 0); rs, drs, g numpy float64 ndarrays of rank 1 (finite, drs >= 0); S a numpy ndarray of an integer dtype and rank 1 or a list of Python ints (converted to Python ints, range-checked); cP and cap Python or numpy integers (not bool), 1 <= cap <= 2^53. Everything else (float or numpy integer residuals, object arrays, NaN caps, ...) is refused. Exactness: all decisions use Fractions; the floating bound computations are a priori gamma_n-inflated round-to-nearest evaluations whose every result is checked finite (an overflow or NaN refuses), with an absolute underflow allowance.""" import numbers import scipy.sparse as sp try: for name, v in (("cap", cap), ("cP", cP)): if isinstance(v, bool) or not isinstance(v, (int, np.integer)): return f"NOT VERIFIED (malformed input: {name} must be an integer, got {type(v).__name__})", {} cap = int(cap); cP = int(cP) if not (1 <= cap <= 2 ** 53): return "NOT VERIFIED (cap must be an integer in [1, 2^53])", {} for name, M in (("B", Bs), ("dB", dBs)): if not sp.issparse(M) or M.dtype != np.float64 or M.ndim != 2: return f"NOT VERIFIED (malformed input: {name} must be a scipy sparse float64 matrix)", {} Bs = sp.csr_matrix(Bs, dtype=np.float64, copy=True); dBs = sp.csr_matrix(dBs, dtype=np.float64, copy=True) for name, v in (("r", rs), ("dr", drs), ("g", g)): if not isinstance(v, np.ndarray) or v.dtype != np.float64 or v.ndim != 1: return f"NOT VERIFIED (malformed input: {name} must be a rank-1 float64 ndarray)", {} rs = np.array(rs, dtype=np.float64, copy=True); drs = np.array(drs, dtype=np.float64, copy=True) g = np.array(g, dtype=np.float64, copy=True) if isinstance(S, np.ndarray): if S.ndim != 1 or not np.issubdtype(S.dtype, np.integer): return "NOT VERIFIED (malformed input: S must be a rank-1 integer array)", {} Sl = [int(x) for x in S.tolist()] elif isinstance(S, (list, tuple)) and all(isinstance(x, int) and not isinstance(x, bool) for x in S): Sl = [int(x) for x in S] else: return "NOT VERIFIED (malformed input: S)", {} nC = len(g); m = Bs.shape[0] if not (0 <= cP < nC) or any(not (0 <= x < nC) for x in Sl) or len(set(Sl)) != len(Sl) or cP in set(Sl): return "NOT VERIFIED (invalid basis or prime index)", {} if not (0 < m < 2 ** 30): return "NOT VERIFIED (dimension out of the supported range)", {} S = np.array(Sl, dtype=np.int64) # values < nC < 2^62: exact return _certify_core_checked(Bs, dBs, rs, drs, g, S, cP, cap, log) except Exception as e: # every exception on the certifying path refuses return f"NOT VERIFIED (exception on the certifying path: {type(e).__name__})", {} def _certify_core_checked(Bs, dBs, rs, drs, g, S, cP, cap, log=print): """The certifying step. Bs (m x m, CSR) +- dBs (entrywise radii) encloses the exact basis matrix B_ex (row-scaled by exact powers of two); rs +- drs encloses the exact (equally row-scaled) residual r = A_ex g of the dyadic point g (all nC classes); S = the basis columns. Proves, if it returns "VERIFIED EXACT FAKE": B_ex is invertible, delta_S = -B_ex^{-1} r satisfies |delta_S - x~| <= eps, and g* = g + delta (delta = 0 off S) satisfies A_ex g* = 0, g*_P = -1 exactly, -1 <= g* <= cap - 1 on all composites (checked exactly). Returns (verdict, info).""" from fractions import Fraction as Fr import scipy.linalg as sla info = {} if int(cap) != cap or not (1 <= int(cap) <= 2 ** 53): # v3 (review defect 1): cap - 1 exact in binary64 return "NOT VERIFIED (cap must be an integer in [1, 2^53])", info g = np.asarray(g); S = np.asarray(S); rs = np.asarray(rs); drs = np.asarray(drs) if Bs.ndim != 2 or g.ndim != 1 or S.ndim != 1 or rs.ndim != 1 or drs.ndim != 1: # v3 (defect 2): shapes return "NOT VERIFIED (malformed input: wrong array rank)", info if g.dtype != np.float64 or not np.issubdtype(S.dtype, np.integer): return "NOT VERIFIED (malformed input: wrong dtype)", info m = Bs.shape[0]; nC = len(g) S = S.astype(np.int64) if Bs.shape != (m, m) or dBs.shape != (m, m) or len(rs) != m or len(drs) != m or len(S) != m: return "NOT VERIFIED (inconsistent dimensions)", info if len(np.unique(S)) != m or (m and (S.min() < 0 or S.max() >= nC)) or np.isin(cP, S): return "NOT VERIFIED (invalid basis: duplicates, out of range, or the prime)", info for name, arr in (("B", Bs.data), ("dB", dBs.data), ("r", rs), ("dr", drs), ("g", g)): if not np.all(np.isfinite(arr)): return f"NOT VERIFIED (non-finite {name})", info if np.any(dBs.data < 0) or np.any(drs < 0): return "NOT VERIFIED (negative radius)", info # the prime and all off-basis classes, exactly (they keep their values: delta = 0 off S) if Fr(float(g[cP])) != -1: return f"NOT VERIFIED (the prime coordinate is {float(g[cP])!r}, not exactly -1)", info off = np.ones(nC, dtype=bool); off[S] = False; off[cP] = False lo_e, hi_e = Fr(-1), Fr(int(cap) - 1) # v3: EXACT comparisons for every off-basis class nb = sum(1 for x in g[off].tolist() if not (lo_e <= Fr(x) <= hi_e)) if nb: return f"NOT VERIFIED (an off-basis class violates its bounds: {nb} classes)", info TINY = (8 * m + 64) * UNDERFLOW # inverse by LAPACK getrf + getri in place (scipy.linalg.inv SEGFAULTS on an exactly singular matrix with SciPy 1.18: D6) try: lu, piv, inf1 = sla.lapack.dgetrf(Bs.toarray(order="F"), overwrite_a=True) if inf1 != 0 or not np.all(np.diag(lu) != 0): return "NOT VERIFIED (singular basis matrix in floating point)", info Cinv, inf2 = sla.lapack.dgetri(lu, piv, overwrite_lu=True) if inf2 != 0: return "NOT VERIFIED (inversion failed)", info except Exception as e: # D6 return f"NOT VERIFIED (inversion failed: {type(e).__name__})", info if not np.all(np.isfinite(Cinv)): return "NOT VERIFIED (non-finite inverse)", info xt = Cinv @ (-rs) if not np.all(np.isfinite(xt)): return "NOT VERIFIED (non-finite correction)", info absBs = abs(Bs) g1 = gam(m + 2); g2 = 1.0 + 2 * gam(m + 4) res_f = -rs - Bs @ xt res_rad = drs + dBs @ np.abs(xt) + g1 * (np.abs(rs) + absBs @ np.abs(xt)) + TINY res_abs = (np.abs(res_f) + res_rad * g2) * (1.0 + 2 * U_ROUND) + TINY rowsum1 = np.empty(m); BsT = Bs.T.tocsr() for i0 in range(0, m, 2048): # row blocks of C B i1 = min(m, i0 + 2048) blk = np.asarray((BsT @ Cinv[i0:i1].T).T) blk[np.arange(i1 - i0), np.arange(i0, i1)] -= 1.0 rowsum1[i0:i1] = np.abs(blk).sum(axis=1) del blk np.abs(Cinv, out=Cinv) # |C| from here on if not (np.all(np.isfinite(res_abs)) and np.all(np.isfinite(rowsum1))): return "NOT VERIFIED (non-finite residual or norm bound)", info z = (Cinv @ res_abs) * g2 + TINY if not np.all(np.isfinite(z)): return "NOT VERIFIED (non-finite correction bound)", info t2 = Cinv @ (absBs @ np.ones(m)); t3 = Cinv @ (dBs @ np.ones(m)) del Cinv rows = rowsum1 * (1.0 + 2 * U_ROUND) + (g1 * t2 + t3) * g2 + TINY nE = float(np.max(rows)) * g2 + TINY zmax = float(np.max(z)) if m else 0.0 info.update(nE=nE, zmax=zmax, xt=xt, maxdx=float(np.max(np.abs(xt))) if m else 0.0) if not (np.all(np.isfinite(rows)) and np.isfinite(zmax) and nE < 1.0): return f"NOT VERIFIED (||I - C B||_inf <= {nE:.3e} is not < 1)", info eps = Fr(zmax) / (1 - Fr(nE)) # exact rational (D3) info["eps"] = eps lo_b, hi_b = Fr(-1), Fr(int(cap) - 1) worst = None for i, x in zip(S.tolist(), xt.tolist()): c = Fr(float(g[i])) + Fr(x) lo, hi = c - eps, c + eps if lo < lo_b or hi > hi_b: return f"NOT VERIFIED (basis class {i}: exact interval [{float(lo)!r}, {float(hi)!r}] leaves the bounds)", info w = min(lo - lo_b, hi_b - hi) worst = w if worst is None or w < worst else worst info["minslack_basis"] = worst return "VERIFIED EXACT FAKE", info def verify(K, m0, thI, thT, cap, r, J, log=print, tl=3600.0, slack_min=1e-6): """THE certifying entry point: build the family LP (classes with j <= J, or the full family if J is None) from the EXACT distributions, take a float fake (env FAKE: a famrk .npz with ccodes and g, untrusted; else solve), round it to a dyadic point, compute its residual EXACTLY, choose a basis (env BASIS hint, validated; else LU with partial pivoting) and call certify_core.""" import os import time import scipy.linalg as sla from scipy.sparse import csr_matrix import famrk as FR import modm_otf_lowmem2 as B0 t0 = time.time() why = check_domain(K, m0, thI, thT, cap, r, J) if why: return dict(verdict=f"NOT VERIFIED (unsupported parameters: {why})") Q = common_Q(K, m0) DE = DistExact(K, m0, thT, r, K - thI, jmax=J, Q=Q) Jf = K if J is None else J enc = FR.encoder(K, m0, thT, r) Om_K, DK, _ = DE.byn[K] ccode = enc(Om_K); order = np.argsort(ccode); cs = ccode[order]; nC = len(cs) if nC < 2 or np.any(cs[1:] == cs[:-1]): return dict(verdict="NOT VERIFIED (degenerate class list)") pfeat = [0, 0, 0, 0, K] + [0] * r Pkey = enc(np.array([pfeat], dtype=np.int64))[0] cP = int(np.searchsorted(cs, Pkey)) if cP >= nC or cs[cP] != Pkey: return dict(verdict="NOT VERIFIED (no prime class)") # the prime class is the singleton {(K)}: features j = k = 0 and largest large part K force u = (K); its exact mass is 1/K ip = int(order[cP]) assert Q % K == 0 and DK[ip] == Q // K, "prime class mass is not exactly 1/K" log(f" exact distributions: Q {Q.bit_length()} bits, {DE.nD} omega entries, {nC} classes (j <= {Jf}); prime class mass exactly 1/K; " f"{time.time() - t0:.0f} s; RSS {B0.rss_gb():.2f} GB", flush=True) ks = [k for k in range(0, thI + 1) if (k == 0 or k >= m0) and (K - k == 0 or K - k >= m0)] rk = {} for k in ks: for v in B0.partitions_min(k, m0): key = (K - sum(v),) + FR.feat(v, K, thT, r) if key[1] > Jf: continue rk.setdefault(("I",) + key, v) if sum(v) <= thT: rk.setdefault(("T",) + key, v) rows = list(rk) rows.sort(key=lambda t: (t[0] != "I",)) m = len(rows) cache = {n: DE.rounded(n) for n in range(K - thI, K + 1)} C_, V_, S_, N_, L_ = [], [], [], [], [] for (typ, n, *rho) in rows: Om = DE.byn[n][0] Mg = FR.merge(rho, Om, r) okj = Mg[:, 0] <= Jf mc = enc(Mg[okj]) cls = np.searchsorted(cs, mc); assert np.all(cs[np.minimum(cls, nC - 1)] == mc) idx = np.nonzero(okj)[0] dfl = (cache[n][1] if typ == "I" else cache[n][2])[idx] u_, inv, cnt = np.unique(cls, return_inverse=True, return_counts=True) vals = np.bincount(inv, weights=dfl); absv = np.bincount(inv, weights=np.abs(dfl)) C_.append(u_.astype(np.int32)); V_.append(vals); S_.append(absv); N_.append(cnt); L_.append(len(u_)) if m == 0 or min(L_) == 0: return dict(verdict="NOT VERIFIED (an empty row)") indptr = np.concatenate([[0], np.cumsum(L_)]).astype(np.int64) A = csr_matrix((np.concatenate(V_), np.concatenate(C_), indptr), shape=(m, nC)) Aabs = csr_matrix((np.concatenate(S_), np.concatenate(C_), indptr), shape=(m, nC)) Acnt = csr_matrix((np.concatenate(N_).astype(np.float64), np.concatenate(C_), indptr), shape=(m, nC)) del C_, V_, S_, N_ # |A_ex - A~| <= (cnt + 2) u sum|D~| (1 + 1e-12) + (cnt + 2) 2^-1074 (rounded inputs, cnt - 1 additions, underflow) dA = Aabs.copy(); dA.data = (Acnt.data + 2.0) * (U_ROUND * Aabs.data * (1.0 + 1e-12) + UNDERFLOW) maxcnt = int(Acnt.data.max()); del Aabs, Acnt assert np.all(np.isfinite(A.data)) and np.all(np.isfinite(dA.data)) log(f" family LP: {m} rows ({sum(1 for t in rows if t[0] == 'I')} Type I keys, {sum(1 for t in rows if t[0] == 'T')} twisted), " f"{nC} classes, nnz {A.nnz}; max entry count {maxcnt}; {time.time() - t0:.0f} s; RSS {B0.rss_gb():.2f} GB", flush=True) Ae = A.copy(); rmx = np.maximum.reduceat(np.abs(Ae.data), Ae.indptr[:-1]); Ae.data /= np.repeat(rmx, np.diff(Ae.indptr)) if os.environ.get("FAKE"): # untrusted float starting point try: zf = np.load(os.environ["FAKE"], allow_pickle=False); zc = np.asarray(zf["ccodes"]); gf = np.array(zf["g"], dtype=np.float64) except Exception as e: return dict(verdict=f"NOT VERIFIED (unreadable FAKE file: {type(e).__name__})") if zc.shape != cs.shape or not np.array_equal(zc, cs) or gf.shape != (nC,): return dict(verdict="NOT VERIFIED (the FAKE file's classes or point shape differ)") o = dict(val=1.0 + float(gf[cP]), res=float(np.max(np.abs(Ae @ gf))), g=gf) log(f" float fake loaded from {os.environ['FAKE']} (untrusted)", flush=True) else: (Ar, Ahi), sc_, colmax = FR.scale(Ae, cap, "max") o = FR.solve_family((Ar, Ahi), sc_, colmax, cP, cap, ["ipm", "simplex"], log, "float fake", tl=tl, polish_tl=600.0) del Ar, Ahi if o is None: return dict(verdict="NOT VERIFIED (float solve failed)") del Ae log(f" float fake: C- = {o['val']:.3e}, residual {o['res']:.2e} (row-equilibrated); {time.time() - t0:.0f} s", flush=True) if o["val"] > 1e-7 and not os.environ.get("RIGFAM_TEST_NOPRIME"): return dict(verdict="NO FLOAT FAKE", val=o["val"]) if os.environ.get("RIGFAM_SAVE_FAKE"): # test support: save the float point (famrk format) np.savez(os.environ["RIGFAM_SAVE_FAKE"], ccodes=cs, g=o["g"]) # the dyadic point: multiples of 2^-60, clipped to the bounds, prime set to -1. TEST HOOKS (tests_rigfam.py only) skip the clip or # the prime assignment, to check that certify_core refuses such points. SH = 60 g = np.ldexp(np.round(np.ldexp(np.array(o["g"], dtype=np.float64), SH)), -SH) comp = np.arange(nC) != cP if not os.environ.get("RIGFAM_TEST_NOCLIP"): g[comp] = np.clip(g[comp], -1.0, cap - 1.0) if not os.environ.get("RIGFAM_TEST_NOPRIME"): g[cP] = -1.0 if not np.all(np.isfinite(g)): return dict(verdict="NOT VERIFIED (non-finite starting point)") M = [] for x in g.tolist(): # exact integers M_c = g_c 2^SH nu, de = float(x).as_integer_ratio() if (1 << SH) % de: return dict(verdict="NOT VERIFIED (starting point not on the 2^-60 grid)") M.append(nu * ((1 << SH) // de)) Mobj = np.array(M, dtype=object); del M Dobj = {n: (np.array(DE.byn[n][1], dtype=object), np.array(DE.byn[n][2], dtype=object)) for n in range(K - thI, K + 1)} Rex = [] for (typ, n, *rho) in rows: Om = DE.byn[n][0] Mg = FR.merge(rho, Om, r) okj = np.nonzero(Mg[:, 0] <= Jf)[0] cls = np.searchsorted(cs, enc(Mg[okj])) dd = Dobj[n][0 if typ == "I" else 1][okj] Rex.append(int(np.dot(dd, Mobj[cls])) if len(okj) else 0) den = Q << SH del Dobj, Mobj, cache, DE rt = np.array([R / den for R in Rex]) # correctly rounded (CPython int/int) dr = U_ROUND * np.abs(rt) + UNDERFLOW log(f" exact residual of the dyadic point: max |r| {np.abs(rt).max():.3e}; {time.time() - t0:.0f} s; RSS {B0.rss_gb():.2f} GB", flush=True) if os.environ.get("SELECT_PREP_OUT"): # input for the heuristic selector select_from (NOT a certificate) np.savez(os.environ["SELECT_PREP_OUT"], A_data=A.data, A_ind=A.indices, A_ptr=A.indptr, rmx=rmx, g=g, cP=cP, cap=cap, nC=nC, m=m) return dict(verdict="SELECTION INPUT WRITTEN (no certificate)") if os.environ.get("MINNORM"): return dict(verdict="NOT VERIFIED (the minimum-norm path is disabled in v2)") # exact power-of-two row scaling, kept in the normal range (D5) p2 = np.exp2(-np.round(np.log2(rmx))) if not (np.all(p2 >= 2.0 ** -500) and np.all(p2 <= 2.0 ** 500)): return dict(verdict="NOT VERIFIED (row scaling out of range)") A.data *= np.repeat(p2, np.diff(A.indptr)); dA.data *= np.repeat(p2, np.diff(dA.indptr)) nzA = np.abs(A.data[A.data != 0]) if nzA.size and nzA.min() < 2.0 ** -1000: return dict(verdict="NOT VERIFIED (scaled matrix entries leave the normal range)") rs = p2 * rt; drs = p2 * dr slack = np.minimum(g + 1.0, cap - 1.0 - g) cand = np.nonzero(comp & (slack >= slack_min))[0] if len(cand) < m: return dict(verdict=f"NOT VERIFIED (only {len(cand)} candidate classes for {m} rows)") As = A.tocsc(); del A if os.environ.get("BASIS"): # a HINT: validated in certify_core try: S = np.load(os.environ["BASIS"], allow_pickle=False) except Exception as e: return dict(verdict=f"NOT VERIFIED (unreadable basis hint: {type(e).__name__})") if not isinstance(S, np.ndarray): # e.g. an .npz archive (NpzFile) try: S.close() except Exception: pass return dict(verdict=f"NOT VERIFIED (malformed basis hint: a {type(S).__name__}, not an array)") if S.ndim != 1 or not np.issubdtype(S.dtype, np.integer): return dict(verdict=f"NOT VERIFIED (malformed basis hint: shape {S.shape}, dtype {S.dtype})") S = S.astype(np.int64) log(f" basis hint: {os.environ['BASIS']} ({len(S)} indices)", flush=True) else: CMAX = int(os.environ.get("CMAX", str(len(cand)))) if len(cand) > CMAX: cmx = np.zeros(nC); nzc = np.diff(As.indptr) > 0 cmx[nzc] = np.maximum.reduceat(np.abs(As.data), As.indptr[:-1][nzc]) cand = np.sort(cand[np.argsort(-cmx[cand], kind="stable")[:CMAX]]) Ad = As[:, cand].T.toarray(order="F") try: lu, piv, inf = sla.lapack.dgetrf(Ad, overwrite_a=True) except Exception as e: return dict(verdict=f"NOT VERIFIED (basis selection failed: {type(e).__name__})") perm = np.arange(len(cand)) for i, p in enumerate(piv[:m]): perm[i], perm[p] = perm[p], perm[i] S = cand[perm[:m]] del Ad, lu if inf != 0: return dict(verdict="NOT VERIFIED (no nonsingular basis among the candidates)") S = np.asarray(S, dtype=np.int64) if len(S) != m or len(np.unique(S)) != m or S.min() < 0 or S.max() >= nC: return dict(verdict="NOT VERIFIED (invalid basis)") Bs = As[:, np.sort(S)].tocsr() dBs = dA.tocsc()[:, np.sort(S)].tocsr() S = np.sort(S) del As, dA verdict, info = certify_core(Bs, dBs, rs, drs, g, S, cP, cap, log) del Bs, dBs out = dict(verdict=verdict, m=m, nC=nC, cs=cs, cP=cP, g=g, S=S, Rex=Rex, Q=Q, SH=SH, **info) if "eps" in info: log(f" certify_core: ||I - C B||_inf <= {info['nE']:.3e}; |delta_S - x~|_inf <= eps = {float(info['eps']):.3e} (exact rational); " f"max |x~| {info['maxdx']:.3e}; exact minimum basis slack {float(info.get('minslack_basis') or 0):.6e}; prime exactly -1 and every " f"off-basis class within [-1, {cap - 1:g}] (checked exactly)", flush=True) log(f" VERDICT: {verdict}; {time.time() - t0:.0f} s; RSS {B0.rss_gb():.2f} GB", flush=True) return out def _sha(path): import hashlib return hashlib.sha256(open(path, "rb").read()).hexdigest() if __name__ == "__main__" and len(sys.argv) > 1 and sys.argv[1] == "verify": import os import platform import scipy import famrk as FR import modm_otf_lowmem2 as B0 try: K, m0, thI, thT = (int(a) for a in sys.argv[2:6]); cap = int(sys.argv[6]); r = int(sys.argv[7]) J = None if sys.argv[8] == "full" else int(sys.argv[8]) except Exception as e: print(f"RESULT rigfam v4 (arguments {sys.argv[2:9]}): NOT VERIFIED (malformed arguments: {type(e).__name__})", flush=True) sys.exit(0) shas = {"rigfam": _sha(__file__), "famrk": _sha(FR.__file__), "modm_otf_lowmem2": _sha(B0.__file__)} print(f"rigfam v4 verify {K} {m0} {thI} {thT} {cap} {r} {sys.argv[8]}; sources {shas}; python {platform.python_version()}, numpy " f"{np.__version__}, scipy {scipy.__version__}; float info: eps {np.finfo(float).eps}, smallest subnormal {np.nextafter(0.0, 1.0)}; " f"hints: FAKE={os.environ.get('FAKE')} BASIS={os.environ.get('BASIS')} TEST hooks: " f"{[k for k in os.environ if k.startswith('RIGFAM_TEST_')]}", flush=True) try: out = verify(K, m0, thI, thT, cap, r, J) except Exception as e: # every exception on the certifying path refuses out = dict(verdict=f"NOT VERIFIED (exception: {type(e).__name__})") tag = f"K{K}_m0{m0}_thI{thI}_thT{thT}_cap{cap}_r{r}_{sys.argv[8]}_{shas['rigfam'][:8]}" if out["verdict"] == "VERIFIED EXACT FAKE" and not any(k.startswith("RIGFAM_TEST_") for k in os.environ): cf = f"rigfam_cert_{tag}.npz" np.savez(cf, params=np.array([K, m0, thI, thT, cap, r, -1 if J is None else J]), Qhex=hex(out["Q"]), SH=out["SH"], ccodes=out["cs"], cP=out["cP"], g=out["g"], S=out["S"], xt=out["xt"], eps=str(out["eps"]), nE=out["nE"], zmax=out["zmax"], Rex_hex=np.array([hex(x) for x in out["Rex"]]), sources=np.array([f"{k} {v}" for k, v in shas.items()])) print(f" certificate written: {cf} (sha256 {_sha(cf)})", flush=True) print(f"RESULT rigfam v4 K={K} m0={m0} thI={thI} thT={thT} cap={cap} ct+s{r} " f"{'full family' if J is None else 'classes j <= ' + str(J)}: {out['verdict']}", flush=True) elif __name__ == "__main__" and len(sys.argv) > 1 and sys.argv[1] == "select_from": # HEURISTIC basis selection (no certificate): float32 LU with partial pivoting over all composite classes with slack >= 1e-6 import scipy.linalg as sla from scipy.sparse import csr_matrix import modm_otf_lowmem2 as B0 z = np.load(sys.argv[2]) m, nC, cP, cap = int(z["m"]), int(z["nC"]), int(z["cP"]), float(z["cap"]) g = z["g"]; rmx = z["rmx"] p2 = np.exp2(-np.round(np.log2(rmx))) data = (z["A_data"] * np.repeat(p2, np.diff(z["A_ptr"]))).astype(np.float32) A32 = csr_matrix((data, z["A_ind"], z["A_ptr"]), shape=(m, nC)); del data slack = np.minimum(g + 1.0, cap - 1.0 - g) cand = np.nonzero((np.arange(nC) != cP) & (slack >= 1e-6))[0] Ad = A32.tocsc()[:, cand].T.toarray(order="F"); del A32 lu, piv, info = sla.lapack.sgetrf(Ad, overwrite_a=True) perm = np.arange(len(cand)) for i, p in enumerate(piv[:m]): perm[i], perm[p] = perm[p], perm[i] S = np.sort(cand[perm[:m]]); dg = np.abs(np.diag(lu[:m, :m])) np.save(sys.argv[3], S) print(f"select_from (heuristic hint, no certificate): {m} classes saved to {sys.argv[3]}; float32 |U| diagonal in [{dg.min():.3e}, " f"{dg.max():.3e}]; sgetrf info {info}; RSS {B0.rss_gb():.2f} GB", flush=True) elif __name__ == "__main__" and len(sys.argv) > 1 and sys.argv[1] in ("basis_from", "minnorm_from"): print(f"{sys.argv[1]}: REMOVED in rigfam v2 (continuations trusted saved state; review defects D1, D2, D4). Use `verify`.") sys.exit(2) elif __name__ == "__main__": # self-test: Q divisibility; exact D and Dp against brute-force Fractions at EVERY n in [K//2, K] import famrk as FR import modm_otf_lowmem2 as B0 K, m0, thT, r = int(sys.argv[1]), int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]) Q = common_Q(K, m0) nchk = 0 for n in range(1, K + 1): for w in B0.partitions_min(n, m0): d = math.prod(w) for x in set(w): d *= math.factorial(w.count(x)) assert Q % d == 0, w nchk += 1 print(f"Q ({Q.bit_length()} bits) divisible by d(w) for all {nchk} multisets with total <= {K}") DE = DistExact(K, m0, thT, r, K // 2) tot = 0 for n in range(K // 2, K + 1): acc = defaultdict(lambda: [Fraction(0), Fraction(0)]) for w in B0.partitions_min(n, m0): d = math.prod(w) for x in set(w): d *= math.factorial(w.count(x)) a = acc[FR.feat(w, K, thT, r)]; a[0] += Fraction(1, d); a[1] += Fraction((-1) ** len(w), d) Om_e, De, Dpe = DE.byn[n] assert len(Om_e) == len(acc), n for i in range(len(Om_e)): key = tuple(int(t) for t in Om_e[i]) assert acc[key][0] == Fraction(De[i], Q) and acc[key][1] == Fraction(Dpe[i], Q), (n, key) tot += len(Om_e) print(f"brute force at every n in [{K // 2}, {K}]: all {tot} omega entries EXACTLY equal (D and Dp)") ~~~~~~~~~~~~ #### file: indep_export.py - kind: script - as run: SHA-256 d85fbc4767ef32626964fdef6b685252b4f1565326528f5c163fc5095cac0923 - as shipped: SHA-256 8b02ed286d64db81e2296eb3fa66c665874fdaa069633bb46598428da276607c - identity: identical, after dropping docstrings and comments, to the file as run with 1 declared change(s) of a machine-specific path, which now read: `TW = __import__('os').path.dirname(__import__('os').path.abspath(__file__))` - trailing newline: yes ~~~~~~~~~~~~ python """indep_export.py: runs rigfam.py's verify unchanged on K = 72 and exports everything an independent checker needs to verify the contraction and correction bounds itself. The core's inputs (B, dB, r, dr, g, S, cP, cap) are saved exactly as it receives them; the floating-point inverse C is recomputed by the same LAPACK calls on the same input (dgetrf and dgetri on B.toarray(order='F'), one thread) and saved with x~ = C(-r), and the core's own x~ must be bitwise identical (asserted). numpy.exp2 is wrapped during verify to record p2, the exact power-of-two row scaling, so that B and r can be related to the exact family matrix. It also writes the row keys, class codes, the exact residual integers (Rex_hex.txt) and claims.json, which records the SHA-256 of rigfam.py, famrk.py, modm_otf_lowmem2.py and itself. Usage: indep_export.py OUTDIR (environment FAKE and BASIS as for rigfam.py verify).""" import hashlib import json import os import sys import numpy as np TW = __import__('os').path.dirname(__import__('os').path.abspath(__file__)) sys.path.insert(0, TW); sys.path.insert(0, os.path.dirname(TW)) import rigfam as R # noqa: E402 import famrk as FR # noqa: E402 import modm_otf_lowmem2 as B0 # noqa: E402 OUT = sys.argv[1] os.makedirs(OUT, exist_ok=True) K, m0, thI, thT, cap, r, J = 72, 2, 36, 26, 4, 3, None rec = {} orig_core = R._certify_core_checked def wrapped(Bs, dBs, rs, drs, g, S, cP, cap_, log=print): import scipy.linalg as sla np.save(os.path.join(OUT, "B_data.npy"), Bs.data); np.save(os.path.join(OUT, "B_indices.npy"), Bs.indices) np.save(os.path.join(OUT, "B_indptr.npy"), Bs.indptr) np.save(os.path.join(OUT, "dB_data.npy"), dBs.data); np.save(os.path.join(OUT, "dB_indices.npy"), dBs.indices) np.save(os.path.join(OUT, "dB_indptr.npy"), dBs.indptr) np.save(os.path.join(OUT, "r.npy"), rs); np.save(os.path.join(OUT, "dr.npy"), drs) np.save(os.path.join(OUT, "g.npy"), g); np.save(os.path.join(OUT, "S.npy"), S) rec.update(cP=int(cP), cap=int(cap_), m=int(Bs.shape[0]), nC=int(len(g)), B_format="CSR (data, indices, indptr), shape (m, m)", B_is_csr=bool(Bs.format == "csr"), dB_is_csr=bool(dBs.format == "csr")) lu, piv, inf1 = sla.lapack.dgetrf(Bs.toarray(order="F"), overwrite_a=True) C, inf2 = sla.lapack.dgetri(lu, piv, overwrite_lu=True) del lu np.save(os.path.join(OUT, "C.npy"), C) xt = C @ (-rs) np.save(os.path.join(OUT, "xt.npy"), xt) del C verdict, info = orig_core(Bs, dBs, rs, drs, g, S, cP, cap_, log) same = "xt" in info and np.array_equal(np.asarray(info["xt"]), xt) rec.update(core_verdict=verdict, core_xt_bitwise_equal_to_exported_xt=bool(same), getrf_info=int(inf1), getri_info=int(inf2)) assert same, "the exported C/x~ do not reproduce the core's x~ bitwise" if "eps" in info: rec.update(nE=info["nE"], zmax=info["zmax"], eps_fraction=str(info["eps"]), eps_float=float(info["eps"]), minslack_basis=str(info.get("minslack_basis"))) return verdict, info R._certify_core_checked = wrapped _exp2 = np.exp2 p2s = [] def rec_exp2(x, *a, **k): y = _exp2(x, *a, **k); p2s.append(np.array(y)); return y np.exp2 = rec_exp2 try: out = R.verify(K, m0, thI, thT, cap, r, J) finally: np.exp2 = _exp2 assert len(p2s) == 1 and p2s[0].shape == (rec["m"],), "expected exactly one exp2 call (the row scaling)" np.save(os.path.join(OUT, "p2.npy"), p2s[0]) np.save(os.path.join(OUT, "class_codes.npy"), out["cs"]) with open(os.path.join(OUT, "Rex_hex.txt"), "w") as f: # exact residual integers R_i = r_i Q 2^SH (rows in order) f.write("\n".join(hex(x) for x in out["Rex"]) + "\n") # row keys, rebuilt with the same code as rigfam v4 verify(): (type 0 = Type I / 1 = twisted, n = K - |v|, j, tau, k, rmax, L, top1..top3) ks = [k for k in range(0, thI + 1) if (k == 0 or k >= m0) and (K - k == 0 or K - k >= m0)] rk = {} for k in ks: for v in B0.partitions_min(k, m0): key = (K - sum(v),) + FR.feat(v, K, thT, r) rk.setdefault(("I",) + key, v) if sum(v) <= thT: rk.setdefault(("T",) + key, v) rows = sorted(rk, key=lambda t: (t[0] != "I",)) assert len(rows) == rec["m"] np.save(os.path.join(OUT, "row_keys.npy"), np.array([[0 if t[0] == "I" else 1] + list(t[1:]) for t in rows], dtype=np.int64)) shas = {"rigfam": hashlib.sha256(open(R.__file__, "rb").read()).hexdigest(), "famrk": hashlib.sha256(open(FR.__file__, "rb").read()).hexdigest(), "modm_otf_lowmem2": hashlib.sha256(open(B0.__file__, "rb").read()).hexdigest(), "indep_export": hashlib.sha256(open(__file__, "rb").read()).hexdigest()} rec.update(verdict=out["verdict"], params=dict(K=K, m0=m0, thI=thI, thT=thT, cap=cap, r=r, J="full"), Qhex=hex(out["Q"]), SH=out["SH"], sources=shas, FAKE=os.environ.get("FAKE"), BASIS=os.environ.get("BASIS")) json.dump(rec, open(os.path.join(OUT, "claims.json"), "w"), indent=1) print(f"indep_export: verdict {out['verdict']}; files in {OUT}; claims {rec.get('nE')} {rec.get('zmax')} {rec.get('eps_float')}", flush=True) ~~~~~~~~~~~~ #### file: v4_capture_K36.py - kind: script - as run: SHA-256 2ef412c9cb631655659cd47b9cd4f5a3c35811f5d7350c0b0317e3f82390eba1 - as shipped: SHA-256 582283b1a1980a61ac9aee4e2256a3d880e424f828760929158625022abcbce0 - identity: identical, after dropping docstrings and comments, to the file as run with 2 declared change(s) of a machine-specific path, which now read: `ROOT / "rigfam_cert_K36.npz"`; `ROOT / "rigfam.py"` - trailing newline: yes ~~~~~~~~~~~~ python """Rebuild only K36 and retain inputs to the exact same certify_core body for independent replay.""" import importlib.util import os from pathlib import Path import numpy as np import scipy.linalg as sla ROOT = Path(__file__).resolve().parent OUT = ROOT / "review_v4" OUT.mkdir(exist_ok=True) CERT = ROOT / "rigfam_cert_K36.npz" with np.load(CERT) as f: cert = {k: f[k] for k in f.files} np.save(OUT / "K36_basis_hint.npy", cert["S"]) os.environ["FAKE"] = str(CERT) os.environ["BASIS"] = str(OUT / "K36_basis_hint.npy") for key in list(os.environ): if key.startswith("RIGFAM_TEST_") or key in ("SELECT_PREP_OUT", "MINNORM"): del os.environ[key] spec = importlib.util.spec_from_file_location("rigfam_v4_review", ROOT / "rigfam.py") R = importlib.util.module_from_spec(spec) spec.loader.exec_module(R) original_core = R.certify_core original_getri = sla.lapack.dgetri def capture_inverse(*args, **kwargs): inverse, info = original_getri(*args, **kwargs) if info == 0: np.save(OUT / "K36_C.npy", inverse) return inverse, info def capture_core(Bs, dBs, rs, drs, g, S, cP, cap, log=print): np.savez(OUT / "K36_core_inputs.npz", B_data=Bs.data, B_ind=Bs.indices, B_ptr=Bs.indptr, dB_data=dBs.data, dB_ind=dBs.indices, dB_ptr=dBs.indptr, rs=rs, drs=drs, g=g, S=S, cP=cP, cap=cap, shape=np.array(Bs.shape)) return original_core(Bs, dBs, rs, drs, g, S, cP, cap, log) R.certify_core = capture_core sla.lapack.dgetri = capture_inverse try: out = R.verify(36, 1, 18, 13, 4, 3, None) finally: sla.lapack.dgetri = original_getri assert out["verdict"] == "VERIFIED EXACT FAKE" for key in ("g", "S", "xt"): assert np.array_equal(out[key], cert[key]), key assert out["eps"] == __import__("fractions").Fraction(str(cert["eps"])) assert out["nE"] == float(cert["nE"]) and out["zmax"] == float(cert["zmax"]) assert out["Rex"] == [int(str(x), 16) for x in cert["Rex_hex"]] print("Captured K36 basis enclosures and inverse; recomputed result equals saved certificate", flush=True) ~~~~~~~~~~~~ #### file: v4_exact_grid.py - kind: script - as run: SHA-256 6e15649b06ee1aafcb3f582fbbe43baa369794469c84dea298215edc15be0376 - as shipped: SHA-256 c6f697544bbdbea3da14d48b3815f306a92a350cc336e8fc35d76a73e9f8f502 - identity: identical, after dropping docstrings and comments, to the file as run with 1 declared change(s) of a machine-specific path, which now read: `root/'rigfam_cert_K36.npz'` - trailing newline: yes ~~~~~~~~~~~~ python """Independent exact shape enumeration and basis-enclosure checks. No modules of the producing code are imported. Shapes are enumerated directly (ascending parts), unlike rigfam's aggregated small/rough/large dynamic programme. """ from fractions import Fraction as F from collections import Counter import math import time import numpy as np def shapes(n, minimum): if n == 0: yield () return if n < minimum: return yield (n,) for first in range(minimum, n//2+1): for rest in shapes(n-first, first): yield (first,) + rest def feature(parts, K, thT, r): small = [x for x in parts if x <= thT] rough = [x for x in parts if thT < x and 2*x < K] large = [x for x in parts if 2*x >= K] top = sorted(small, reverse=True)[:r] return (len(small), sum(small), len(rough), max(rough, default=0), max(large, default=0), *(top+[0]*(r-len(top)))) def denominator(parts): answer = math.prod(parts) for count in Counter(parts).values(): answer *= math.factorial(count) return answer def radix(K, m0, thT, r): bases = [K//m0+1, K+1, max(1,K//(thT+1))+1, K//2+1, K+1]+[thT+1]*r assert math.prod(bases) < 2**62 return np.array([math.prod(bases[i+1:]) for i in range(len(bases))], dtype=np.int64) def merged(features, rho, r): out = np.empty_like(features) out[:,:3] = features[:,:3] + np.array(rho[:3], dtype=np.int64) out[:,3:5] = np.maximum(features[:,3:5], np.array(rho[3:5], dtype=np.int64)) if r: tails = np.column_stack([features[:,5:], np.tile(rho[5:], (len(features),1))]) out[:,5:] = np.sort(tails, axis=1)[:,-r:][:,::-1] return out def enclosed(num, den, mid, rad): a,b = float(mid).as_integer_ratio() c,d = float(rad).as_integer_ratio() return c >= 0 and abs(num*b-a*den)*d <= c*den*b def canonical(M): assert M.dtype == np.float64 and M.format == 'csr' assert M.indptr.shape == (M.shape[0]+1,) and M.indptr[0] == 0 assert M.indptr[-1] == len(M.data) == len(M.indices) assert np.all(np.diff(M.indptr) >= 0) assert np.all((M.indices >= 0) & (M.indices < M.shape[1])) assert np.all(np.isfinite(M.data)) for lo,hi in zip(M.indptr[:-1],M.indptr[1:]): assert np.all(np.diff(M.indices[lo:hi]) > 0) def check_grid(c, B, dB, rs, drs, p2_input=None, row_keys_input=None, label='grid'): began = time.monotonic() K,m0,thI,thT,cap,r,J = map(int,c['params']) assert J == -1 and 1 <= m0 <= thT <= thI <= K//2 and 2*thT 0 and SH == 60 g,S,cs = c['g'],c['S'],c['ccodes'] cP = int(c['cP']); m,nC = len(S),len(g) assert B.shape == dB.shape == (m,m) and rs.shape == drs.shape == (m,) canonical(B); canonical(dB) assert np.all(dB.data>=0) and np.all(drs>=0) assert g.dtype == rs.dtype == drs.dtype == np.float64 assert np.all(np.isfinite(g)) and np.all(np.isfinite(rs)) and np.all(np.isfinite(drs)) assert np.all(np.diff(S)>0) and S[0]>=0 and S[-1]0) assert tuple(map(int,om[order[cP]]))==(0,0,0,0,K,*([0]*r)) assert int(weights[order[cP]])*K==Q # The persisted row order is total increasing, descending-lexicographic # moduli within each total, first occurrence of each feature key, I before T. keys={}; original={'I':0,'T':0} for total in range(thI+1): if (total and total0 assert scale.numerator & (scale.numerator-1)==0 assert scale.denominator & (scale.denominator-1)==0 assert enclosed(Rex[i]*scale.numerator,(Q<=0; coefs={} for j,w in zip(pos[cls[inS]],dd[inS]): j=int(j); coefs[j]=coefs.get(j,0)+int(w) lo,hi=B.indptr[i:i+2]; mids=dict(zip(B.indices[lo:hi].tolist(),B.data[lo:hi].tolist())) lo,hi=dB.indptr[i:i+2]; radii=dict(zip(dB.indices[lo:hi].tolist(),dB.data[lo:hi].tolist())) for j in coefs.keys()|mids.keys()|radii.keys(): assert enclosed(coefs.get(j,0)*scale.numerator,Q*scale.denominator,mids.get(j,0.),radii.get(j,0.)),(i,j,'B radius') checked+=1 if (i+1)%3000==0: print(f'{label}: exact residual/basis enclosures {i+1}/{m}; {time.monotonic()-began:.1f}s',flush=True) result=dict(shapes_enumerated=count,shapes_at_K=count_at_K,feature_entries=sum(len(t[0]) for t in distributions.values()), rows=m,classes=nC,original_rows=original,basis_positions_checked=checked,seconds=time.monotonic()-began) print(f'{label}: ALL exact grid and coefficient/residual enclosure checks PASS {result}',flush=True) return result if __name__=='__main__': from pathlib import Path from scipy.sparse import csr_matrix root=Path(__file__).resolve().parent with np.load(root/'rigfam_cert_K36.npz') as z: c={k:z[k] for k in z.files} with np.load(root/'review_v4/K36_core_inputs.npz') as z: d={k:z[k] for k in z.files} m=len(d['rs']) B=csr_matrix((d['B_data'],d['B_ind'],d['B_ptr']),shape=(m,m)) dB=csr_matrix((d['dB_data'],d['dB_ind'],d['dB_ptr']),shape=(m,m)) check_grid(c,B,dB,d['rs'],d['drs'],label='independent K36') ~~~~~~~~~~~~ #### file: v4_independent_K36_bound.py - kind: script - as run: SHA-256 22a98639982f6318b3715042720e2a2a19fdbb1ea531e164bc1d08ea3a4124a4 - as shipped: SHA-256 ca19e3bacb5de4d268ec6c6d7fdddf0898fb2da857c885d15606a371e4c5fcb2 - identity: identical, after dropping docstrings and comments, to the file as run with 1 declared change(s) of a machine-specific path, which now read: `ROOT / "rigfam_cert_K36.npz"` - trailing newline: yes ~~~~~~~~~~~~ python """Independent outward-rounded Rump bounds for the captured K36 certificate. The inputs B +- dB and r +- dr are validated against exact coefficients by v4_exact_grid.py and v4_audit_K36.py. This script treats C and xt as arbitrary exact dyadics. It uses exact gamma constants, explicit outward scalar operations, and a separate dot-error bound including m^2*eta in the matrix infinity norm. """ from fractions import Fraction as F from pathlib import Path import json import numpy as np from scipy.sparse import csr_matrix ROOT = Path(__file__).resolve().parent with np.load(ROOT / "review_v4/K36_core_inputs.npz") as z: data = {k: z[k] for k in z.files} with np.load(ROOT / "rigfam_cert_K36.npz") as z: cert = {k: z[k] for k in z.files} m = len(data["rs"]) B = csr_matrix((data["B_data"], data["B_ind"], data["B_ptr"]), shape=(m, m)) dB = csr_matrix((data["dB_data"], data["dB_ind"], data["dB_ptr"]), shape=(m, m)) assert B.has_canonical_format and dB.has_canonical_format C = np.load(ROOT / "review_v4/K36_C.npy") assert C.shape == (m, m) and np.all(np.isfinite(C)) xt = cert["xt"] u, eta = F(1, 1 << 53), F(1, 1 << 1074) def up_fraction(x): f = float(x) if F(f) < x: f = float(np.nextafter(f, np.inf)) assert F(f) >= x return f def down_fraction(x): f = float(x) if F(f) > x: f = float(np.nextafter(f, -np.inf)) assert F(f) <= x return f def up(x): return np.nextafter(x, np.inf) def add(a, b): return up(np.asarray(a) + np.asarray(b)) def mul(a, b): return up(np.asarray(a) * np.asarray(b)) def gamma(n): return n*u / (1-n*u) def underflow_bound(n): return n*eta / (1-n*u) def upper_nonnegative_dot(A, x): n = A.shape[1] value = A @ x assert np.all(np.isfinite(value)) and np.all(value >= 0) numerator = add(value, up_fraction(underflow_bound(n))) denominator = down_fraction(1-gamma(n)) return up(numerator / denominator) def upper_sum_rows(M): n = M.shape[1] return up(add(np.sum(M, axis=1), up_fraction(underflow_bound(n))) / down_fraction(1-gamma(n))) absC, absB = np.abs(C), abs(B) ones = np.ones(m) t2 = upper_nonnegative_dot(absC, upper_nonnegative_dot(absB, ones)) t3 = upper_nonnegative_dot(absC, upper_nonnegative_dot(dB, ones)) row_defect = np.empty(m) for start in range(0, m, 256): end = min(m, start+256) cb = np.asarray((B.T @ C[start:end].T).T) absolute = np.abs(cb) for local, global_i in enumerate(range(start, end)): absolute[local, global_i] = up_fraction(abs(F(float(cb[local, global_i]))-1)) row_defect[start:end] = upper_sum_rows(absolute) row_error = add(mul(up_fraction(gamma(m)), t2), up_fraction(m*underflow_bound(m))) norm_rows = add(add(row_defect, row_error), t3) nE = float(np.max(norm_rows)) assert 0 <= nE < 1 p = B @ xt center = np.array([up_fraction(abs(-F(float(r))-F(float(pi)))) for r, pi in zip(data["rs"], p)]) matrix_uncertainty = upper_nonnegative_dot(dB, np.abs(xt)) product_error = add(mul(up_fraction(gamma(m)), upper_nonnegative_dot(absB, np.abs(xt))), up_fraction(underflow_bound(m))) residual = add(add(add(center, data["drs"]), matrix_uncertainty), product_error) zmax = float(np.max(upper_nonnegative_dot(absC, residual))) eps = F(zmax)/(1-F(nE)) assert F(nE) <= F(float(cert["nE"])) assert F(zmax) <= F(float(cert["zmax"])) assert eps <= F(str(cert["eps"])) g, S, cP, cap = data["g"], data["S"], int(data["cP"]), int(data["cap"]) assert F(float(g[cP])) == -1 and cP not in S off = np.ones(len(g), dtype=bool); off[S] = False; off[cP] = False assert all(-1 <= F(float(x)) <= cap-1 for x in g[off]) slack = min(min(F(float(g[i]))+F(float(x))+1-eps, F(cap-1)-F(float(g[i]))-F(float(x))-eps) for i, x in zip(S, xt)) assert slack >= 0 result = dict(nE=nE, zmax=zmax, eps=str(eps), minslack=str(slack), stored_nE=float(cert["nE"]), stored_zmax=float(cert["zmax"]), stored_eps=str(cert["eps"]), verdict="K36 VERIFIED BY INDEPENDENT OUTWARD BOUNDS") (ROOT / "review_v4/K36_independent_bounds.json").write_text(json.dumps(result, indent=2)+"\n") print(result["verdict"], flush=True) print(f"nE <= {nE:.17g} (stored {float(cert['nE']):.17g}); zmax <= {zmax:.17g} " f"(stored {float(cert['zmax']):.17g}); eps <= {float(eps):.17g}; exact minslack={float(slack):.17g}", flush=True) print("Every upper-bound scalar operation rounds outward; exact Fraction endpoints passed for every class; " "dot products use the standard IEEE error bound with exact gamma and explicit absolute underflow allowance.", flush=True) ~~~~~~~~~~~~ #### file: v4_independent_K72.py - kind: script - as run: SHA-256 6c6cd6a55aa76e27ad563481ab8c3937f3d8aaca71b18c9e0f97ce989039d059 - as shipped: SHA-256 f27a394c98d3039797b7ecea6019c8f29821d48b3f622fbb977034026bf90f12 - identity: identical, after dropping docstrings and comments, to the file as run with 6 declared change(s) of a machine-specific path, which now read: `D=ROOT/'K72_indep'`; `ROOT/'rigfam_cert_K72.npz'`; `('rigfam','rigfam.py')`; `('famrk','famrk.py')`; `('modm_otf_lowmem2','modm_otf_lowmem2.py')`; `('indep_export','indep_export.py')` - trailing newline: yes ~~~~~~~~~~~~ python """Independent exact-grid plus outward-rounded proof of every K72 README premise. Imports none of the producing code. Numerical dot products are enclosed with exact gamma constants and explicit absolute underflow allowances. Every bound operation is rounded outward with nextafter; all final intervals use exact Fractions. """ from fractions import Fraction as F from pathlib import Path import hashlib import json import time import sys import numpy as np import scipy from scipy.sparse import csr_matrix from v4_exact_grid import check_grid, canonical ROOT=Path(__file__).resolve().parent D=ROOT/'K72_indep' started=time.monotonic() def say(s): print(f'[{time.monotonic()-started:.1f}s] {s}',flush=True) for line in (D/'READY').read_text().splitlines(): expected,name=line.split() assert Path(name).name==name h=hashlib.sha256() with (D/name).open('rb') as f: while b:=f.read(8*1024*1024): h.update(b) assert h.hexdigest()==expected, name say('Every frozen READY hash independently rechecked before use') claims=json.loads((D/'claims.json').read_text()) with np.load(ROOT/'rigfam_cert_K72.npz',allow_pickle=False) as f: cert={k:f[k] for k in f.files} say('Certificate fields: '+repr({k:(v.shape,str(v.dtype)) for k,v in cert.items()})) assert list(map(int,cert['params']))==[72,2,36,26,4,3,-1] assert claims['params']==dict(K=72,m0=2,thI=36,thT=26,cap=4,r=3,J='full') assert int(cert['Qhex'].item(),16)==int(claims['Qhex'],16) and int(cert['SH'])==claims['SH']==60 for key,name in [('rigfam','rigfam.py'),('famrk','famrk.py'),('modm_otf_lowmem2','modm_otf_lowmem2.py'),('indep_export','indep_export.py')]: actual=hashlib.sha256((ROOT/name).read_bytes()).hexdigest() assert actual==claims['sources'][key] # The certificate's three producing-source hashes, checked against local pins. cert_sources=dict(str(v).split() for v in cert['sources']) assert cert_sources=={k:claims['sources'][k] for k in ('rigfam','famrk','modm_otf_lowmem2')} load=lambda name: np.load(D/(name+'.npy'),allow_pickle=False) m,nC=claims['m'],claims['nC'] assert m==15282 and nC==68615 B=csr_matrix((load('B_data'),load('B_indices'),load('B_indptr')),shape=(m,m)) dB=csr_matrix((load('dB_data'),load('dB_indices'),load('dB_indptr')),shape=(m,m)) canonical(B);canonical(dB) rs,drs,g,S,xt=map(load,['r','dr','g','S','xt']) cs,p2,keys=map(load,['class_codes','p2','row_keys']) for k,v in [('g',g),('S',S),('xt',xt),('ccodes',cs)]: assert np.array_equal(cert[k],v),k assert list(map(lambda v:int(str(v),16),cert['Rex_hex']))==[int(s,16) for s in (D/'Rex_hex.txt').read_text().splitlines()] assert int(cert['cP'])==claims['cP']==1 assert float(cert['nE'])==claims['nE'] and float(cert['zmax'])==claims['zmax'] assert F(str(cert['eps']))==F(claims['eps_fraction']) assert F(claims['eps_fraction'])==F(claims['zmax'])/(1-F(claims['nE'])) assert 0<=claims['nE']<1 and claims['zmax']>=0 for v in [rs,drs,g,xt,p2]: assert v.dtype==np.float64 and v.ndim==1 and np.all(np.isfinite(v)) assert xt.shape==(m,) say('Parameters, all export/certificate values and source hashes agree; exact epsilon identity passed') exact=check_grid(cert,B,dB,rs,drs,p2_input=p2,row_keys_input=keys,label='independent K72') say('README P1 and P2 proved from independent shape enumeration and exact integer sums') C=np.load(D/'C.npy',mmap_mode='r',allow_pickle=False) assert C.shape==(m,m) and C.dtype==np.float64 for start in range(0,m,256): assert np.all(np.isfinite(C[start:start+256])) say(f'IEEE environment: NumPy {np.__version__}, SciPy {scipy.__version__}; C is finite binary64') small=np.nextafter(np.float64(0),np.float64(1)) assert small==2.0**-1074 and small*np.float64(1)==small and small+small>small assert np.float64(1)+np.float64(2.0**-53)==1.0 # Treat both C and xt as arbitrary exact dyadics: inversion accuracy is not assumed. assert np.array_equal(C@(-rs),xt),'export assertion of bitwise correction reproducibility failed' say('Stored xt also agrees bitwise with C @ (-r); this is not a proof premise') u,eta=F(1,1<<53),F(1,1<<1074) def upF(q): a=float(q) if F(a)=q return a def downF(q): a=float(q) if F(a)>q:a=float(np.nextafter(a,-np.inf)) assert np.isfinite(a) and F(a)<=q return a def upper(a): ans=np.nextafter(a,np.inf) assert np.all(np.isfinite(ans)) return ans def add(a,b):return upper(np.asarray(a)+np.asarray(b)) def mul(a,b):return upper(np.asarray(a)*np.asarray(b)) def gam(n):return n*u/(1-n*u) def uf(n):return n*eta/(1-n*u) def udot(A,x): n=A.shape[1];v=A@x assert np.all(np.isfinite(v)) and np.all(v>=0) return upper(add(v,upF(uf(n)))/downF(1-gam(n))) def usum(A): n=A.shape[1];v=np.sum(A,axis=1) assert np.all(np.isfinite(v)) and np.all(v>=0) return upper(add(v,upF(uf(n)))/downF(1-gam(n))) absB=abs(B); absC=np.abs(C) ones=np.ones(m) t2=udot(absC,udot(absB,ones));t3=udot(absC,udot(dB,ones)) rowdefect=np.empty(m) say('Nonnegative matrix/radius products enclosed outward') for start in range(0,m,256): end=min(m,start+256) cb=np.asarray((B.T@C[start:end].T).T) assert np.all(np.isfinite(cb)) a=np.abs(cb) for local,global_i in enumerate(range(start,end)): a[local,global_i]=upF(abs(F(float(cb[local,global_i]))-1)) rowdefect[start:end]=usum(a) if start%2048==0 or end==m:say(f'Outward contraction product: {end}/{m} rows') del cb,a nE=float(np.max(add(add(rowdefect,add(mul(upF(gam(m)),t2),upF(m*uf(m)))),t3))) assert 0<=nE<1 and F(nE)<=F(claims['nE']) say(f'README 1 PASS: N <= {nE:.17g} <= claimed {claims["nE"]:.17g} < 1') p=B@xt assert np.all(np.isfinite(p)) center=np.array([upF(abs(-F(float(a))-F(float(b)))) for a,b in zip(rs,p)]) product_error=add(mul(upF(gam(m)),udot(absB,np.abs(xt))),upF(uf(m))) residual=add(add(add(center,drs),udot(dB,np.abs(xt))),product_error) zmax=float(np.max(udot(absC,residual))) assert F(zmax)<=F(claims['zmax']) say(f'README 2 PASS: Z <= {zmax:.17g} <= claimed {claims["zmax"]:.17g}') eps=F(zmax)/(1-F(nE));claimed_eps=F(claims['eps_fraction']) assert eps<=claimed_eps say(f'README 3 PASS: exact epsilon identity and correction radius {float(eps):.17g} <= claimed {float(claimed_eps):.17g}') cP,cap=claims['cP'],claims['cap'] assert F(float(g[cP]))==-1 and cP not in S mask=np.ones(nC,dtype=bool);mask[S]=False;mask[cP]=False assert all(F(-1)<=F(float(v))<=F(cap-1) for v in g[mask]) slack=min(min(F(float(g[i]))+F(float(x))+1-claimed_eps,F(cap-1)-F(float(g[i]))-F(float(x))-claimed_eps) for i,x in zip(S,xt)) assert slack>=0 and slack==F(claims['minslack_basis']) say(f'README 4 PASS: exact endpoints of every class, minimum basis slack {float(slack):.17g}') assert not any(s in sys.modules for s in ['rigfam','famrk','modm_otf_lowmem2']) result=dict(verdict='K72 VERIFIED HERE: independent exact grid and outward numerical inequalities',exact_grid=exact,nE=nE,zmax=zmax,eps=str(eps),stored_nE=claims['nE'],stored_zmax=claims['zmax'],stored_eps=str(claimed_eps),minslack=str(slack),seconds=time.monotonic()-started) (ROOT/'review_v4/K72_independent_bounds.json').write_text(json.dumps(result,indent=2)+'\n') say('README 5 follows: rational exact solution of all key rows, every full shape row, all bounds, prime exactly -1') say(result['verdict']) ~~~~~~~~~~~~ #### file: exact_windows.py - kind: script - as run: SHA-256 5af8578dc2ecc204e5b351006ec0ff29297a97c4b5d5bfa29e00faf968c262af - as shipped: SHA-256 5af8578dc2ecc204e5b351006ec0ff29297a97c4b5d5bfa29e00faf968c262af - identity: identical, after dropping docstrings and comments, to the file as run - trailing newline: yes ~~~~~~~~~~~~ python """Independent Fraction references for the P5 windows, including the half-point. Run: python3 -B exact_windows.py [path/to/p5rig.py] [output.json] Exit status 1 means at least one enclosure refused the independent reference. Transcendental references use rational log brackets, never floating quadrature. """ from audit_common import HERE, SUPP, load_module from fractions import Fraction as F import json import sys import numpy as np M = 120 ie, ia, ih = M // 20, M // 5, M // 2 source = sys.argv[1] if len(sys.argv) > 1 else str(SUPP / 'sep_p5/p5rig.py') outpath = sys.argv[2] if len(sys.argv) > 2 else 'exact_original.json' p = load_module(source) C = p.Cert5(M, {}, Q=1, log=lambda *args: None) C.setup() # R1 and R2 cell bounds made only from r=1/u on (1/5,1/2). # Each exact step convolution is linear between nodes. Its extrema on a cell # are its two endpoint values. This is the first-order enclosure of the paper. rlo = [F(M, i + 1) if ia <= i < ih else F(0) for i in range(M)] rhi = [F(M, i) if ia <= i < ih else F(0) for i in range(M)] def step_convolution(a, b): return [sum((a[i] * b[n - 1 - i] for i in range(max(0, n-M), min(n, M))), F(0)) / M for n in range(M + 1)] nl, nh = step_convolution(rlo, rlo), step_convolution(rhi, rhi) exact_R = {1: (rlo, rhi), 2: ([min(nl[i], nl[i+1]) / 2 for i in range(M)], [max(nh[i], nh[i+1]) / 2 for i in range(M)])} def exact_P(br, m, nwin, grid, upper): a, b = exact_R[br][upper], exact_R[m][upper] return sum((a[i] * b[grid-i-1] for i in range(ih-nwin, ih) if 0 <= grid-i-1 < M), F(0)) / M def log_bracket(q): """log(q) via 2 sum z^(2j+1)/(2j+1), with an exact geometric tail.""" assert q > 0 if q < 1: a, b = log_bracket(1/q) return -b, -a z = (q-1)/(q+1) n = 64 s = 2 * sum((z**(2*j+1)/F(2*j+1) for j in range(n)), F(0)) tail = 2*z**(2*n+1)/(F(2*n+1)*(1-z*z)) return s, s+tail def P11(x, sigma): a = max(F(1,2)-sigma, F(1,5), x-F(1,2)) b = min(F(1,2), x-F(1,5)) if b <= a: return F(0), F(0) low, high = log_bracket(b*(x-a)/(a*(x-b))) return low/x, high/x def R2(x): a, b = max(F(1,5), x-F(1,2)), min(F(1,2), x-F(1,5)) if b <= a: return F(0), F(0) low, high = log_bracket(b*(x-a)/(a*(x-b))) return low/(2*x), high/(2*x) def exact_Z(k, x, sigma): inside = F(1,2)-sigma <= x < F(1,2) if k == 1: v = -1/x if inside and F(1,5) < x < F(1,2) else F(0) return v, v if k == 2: low, high = P11(x, sigma) if inside: a, b = R2(x) low, high = low-b, high-a return low, high # All tested Z3 half-edge points are below 3/5: P12=P21=0. assert x < F(3,5) return F(0), F(0) records = [] def check(label, interval, reference, **where): lo, hi = map(lambda x: F(float(x)), interval) a, b = reference slack = min(a-lo, hi-b) records.append(dict(label=label, where=where, lower=float(lo), upper=float(hi), reference_lower=float(a), reference_upper=float(b), slack=float(slack), slack_exact=str(slack), ok=slack >= 0)) win = C.Win(C) chosen = {ie, 2*ie, 3*ie, ia-1} for l in range(ia): if l in chosen: # Exact rational bounds for the nonnegative step integrals of all three # Wk operands, including both sides of R1's switch at 1/2 in x-u. nodes = sorted(set([ih-2, ih-1, ih, ih+1, ih+2, M-l-2, M-l-1, M-l, ih + ih-l-1, ih+ih-l, ih+ih-l+1])) for key in C.Wk: for n in nodes: if not 0 <= n <= M: continue result = win.comp(key, np.array([n]), np.array([n])) ref = (exact_P(*key, l, n, 0), exact_P(*key, l+1, n, 1)) check('exact_step_P', (result[0][0], result[1][0]), ref, key=key, l=l, n=n) # Direct integral of the paper's r, with an exact rational log bracket. for n in nodes: if not 0 <= n <= M: continue result = win.comp((1,1), np.array([n]), np.array([n])) for sig in (F(l,M), F(2*l+1,2*M), F(l+1,M)): check('paper_P11', (result[0][0], result[1][0]), P11(F(n,M),sig), l=l,n=n,sigma=str(sig)) # The exact _pts/cells used by pass_C and far_strata. Test both the # closed endpoints and interior rational points independently. for aidx in (ih+l-2, ih+l-1, ih+l, ih+l+1): c0, c1 = aidx-l-1, aidx-l pts = [np.array([n]) for n in (c0,c0+1,c1+1)] cells = (np.array([c0]), np.array([c1])) for k in (1,2,3): got = C.Zk(win,k,pts,cells,l) for tick in (0,1,2,3,4): x = F(2*c0+tick,2*M) for sig in (F(l,M), F(2*l+1,2*M), F(l+1,M)): check('paper_Z_closed' if tick in (0,4) else 'paper_Z_interior', (got[0][0],got[1][0]),exact_Z(k,x,sig), l=l,k=k,aidx=aidx,c0=c0,c1=c1,x=str(x),sigma=str(sig)) win.step() groups = {} for name in sorted({r['label'] for r in records}): rows = [r for r in records if r['label']==name] groups[name] = dict(count=len(rows), failures=sum(not r['ok'] for r in rows), smallest_slack=min(rows,key=lambda r:F(r['slack_exact']))) report = dict(M=M, source=source, groups=groups, failures=[r for r in records if not r['ok']], total=len(records), passed=sum(r['ok'] for r in records)) (HERE/outpath).write_text(json.dumps(report,indent=2)+'\n') print(json.dumps({k:v for k,v in report.items() if k!='failures'},indent=2)) raise SystemExit(1 if report['failures'] else 0) ~~~~~~~~~~~~ #### file: audit_common.py - kind: script - as run: SHA-256 6ace64030551affc666cf5f977a0ee1650addae55378cba5e6912a3b04e486b2 - as shipped: SHA-256 536c4a6ec3e88d9c1da60b89c9e3ac1b269d57e466dd1eb03c5704d16f5947df - identity: identical, after dropping docstrings and comments, to the file as run with 4 declared change(s) of a machine-specific path, which now read: `SNAP = Path('.')`; `SUPP = SNAP / '.'`; `PAPER = SNAP / '.'`; `sys.path.insert(0, '.')` - trailing newline: yes ~~~~~~~~~~~~ python """Setup shared by exact_windows.py: one thread for the numerical libraries, no bytecode files, an address-space limit of 2800 MiB, the import path of sep_p5/ (relative: run from the supplement's top directory), and load_module(path) for the program under test.""" import os import resource import sys from pathlib import Path HERE = Path(__file__).resolve().parent SNAP = Path('.') SUPP = SNAP / '.' PAPER = SNAP / '.' for name in ('OPENBLAS_NUM_THREADS', 'OMP_NUM_THREADS', 'MKL_NUM_THREADS', 'BLIS_NUM_THREADS', 'NUMEXPR_NUM_THREADS', 'VECLIB_MAXIMUM_THREADS'): os.environ[name] = '1' sys.dont_write_bytecode = True resource.setrlimit(resource.RLIMIT_AS, (2800 * 1024**2, 2800 * 1024**2)) sys.path.insert(0, '.') sys.path.insert(0, str(SUPP / 'sep_p5')) import flint flint.ctx.threads = 1 def load_module(path): import importlib.util spec = importlib.util.spec_from_file_location(Path(path).stem, path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module ~~~~~~~~~~~~ #### file: mask_check.py - kind: script - as run: SHA-256 f87c100dc401d35baa49f95f25135ae98030d8efb2ee12eb34c5fde192033710 - as shipped: SHA-256 c9bc9362dec301fb751e4ba400abb30173ec7505a72734aeb92c4baebfd75c3b - identity: identical, after dropping docstrings and comments, to the file as run - trailing newline: yes ~~~~~~~~~~~~ python """mask_check.py: exact check of zcomb's delta masks, READ FROM THE FILE'S BYTES (the 'sure =' and 'maybe =' lines inside zcomb), against the closed-set truth. Product set: x in [c0 h, (c1+1) h] (closed), sigma in [l h, (l+1) h] (closed), W(sigma) = [1/2 - sigma, 1/2). Contract: sure => x in W(sigma) for EVERY (x, sigma); maybe <= x in W(sigma) for SOME (x, sigma), i.e. (some and not maybe) is a violation. Every (l, c0, c1) with 0 <= l < M/2 and 0 <= c0 <= c1 < M. The caller-form count uses only c1 == c0 + 1 and c1 + l >= M/2; it does not impose the callers' l limits or the certificate's admissible meshes. Truth with Fractions. Usage: mask_check.py FILE [FILE ...]""" import re import sys from fractions import Fraction as F def masks(path): src = open(path).read() body = src[src.index("def zcomb"):src.index("def Zk")] s = re.search(r"^\s*sure = (.+)$", body, re.M).group(1); m = re.search(r"^\s*maybe = (.+)$", body, re.M).group(1) py = lambda e: e.replace("&", " and ") return s, m, (lambda c0, c1, ih, l: bool(eval(py(s)))), (lambda c0, c1, ih, l: bool(eval(py(m)))) for path in sys.argv[1:]: s_txt, m_txt, sure, maybe = masks(path) print(f"{path}\n sure = {s_txt}\n maybe = {m_txt}") for M in (40, 120): ih = M // 2; h = F(1, M); half = F(1, 2) fs = mm = fs_r = mm_r = n = 0 fs_a = mm_a = 0 for l in range(0, ih): for c0 in range(0, M): for c1 in range(c0, M): x0, x1 = c0 * h, (c1 + 1) * h; s0, s1 = l * h, (l + 1) * h all_in = x0 >= half - s0 and x1 < half some_in = x1 >= half - s1 and x0 < half reach = (c1 == c0 + 1) and (c1 + l >= ih) actual = reach and (M // 20 <= l < M // 5) and (c1 + l < M) n += 1 if sure(c0, c1, ih, l) and not all_in: fs += 1; fs_r += reach; fs_a += actual if some_in and not maybe(c0, c1, ih, l): mm += 1; mm_r += reach; mm_a += actual print(f" M = {M}: {n} (l, c0, c1) checked; false 'sure' {fs} (caller-form candidates {fs_r}); missed 'maybe' {mm} " f"(caller-form candidates {mm_r}) -> {'CONTRACT HOLDS' if fs == 0 and mm == 0 else 'VIOLATIONS'}") print(f" M = {M}: actual calls (caller form AND M/20 <= l < M/5 AND c1 + l < M, the PhiD loop; the far strata use a subset): " f"false 'sure' {fs_a}, missed 'maybe' {mm_a}; mesh admissible to the certificate (M % 60 == 0): {M % 60 == 0}") ~~~~~~~~~~~~ #### file: run_sec8_F.sh - kind: recipe (a record of what was run; machine-specific helpers are not shipped) - as run: SHA-256 93829dabb9a6ffdeb899bf2beb6e36447c83add2d52a70086234787be3b83842 - as shipped: SHA-256 d2309baa9f147fb4e6eb1ee54a4f49367e536212324a41f1f85102b592d54fe5 - identity: the recipe as run, with a new comment header and these substitutions: a machine-specific path replaced by `.` (1x); a machine-specific path replaced by `python3` (1x); a machine-specific path replaced by `cpulimit.sh` (1x); a name removed (1x) - trailing newline: yes ~~~~~~~~~~~~ bash #!/bin/bash # (F1) wall_check.py at (eta, alpha) = (1/12, 0.35) with N = 360 and 720, and at (1/24, 245/720) with N = 720. (F2)-(F4), runs of # older family programmes, are superseded by the runs of record (run_rec*.sh) and their outputs are not shipped; sec8_F.out still lists them. # Floats; not certificates. export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 FLINT_NUM_THREADS=1 D=. PY=python3 cd $D OUT=$D/sec8_F.out lim() { ulimit -v 3000000; nice -n 10 cpulimit.sh 2400 "$@"; } echo "== $(date -u +%H:%M:%S) start; wall_check.py $(sha256sum wall_check.py | cut -c1-8); cont_hm_copy.py $(sha256sum cont_hm_copy.py | cut -c1-8); cont_hm.py $(sha256sum cont_hm.py | cut -c1-8); cont_hmg.py $(sha256sum cont_hmg.py | cut -c1-8)" >> $OUT echo "-- F1 wall_check" >> $OUT for spec in "360 1 12 126 360" "720 1 12 252 720" "720 1 24 245 720"; do ( lim $PY -B $D/wall_check.py $spec ) > $D/F1_wall_$(echo $spec | tr ' ' '_').raw 2>&1 echo "$(date -u +%H:%M:%S) wall_check $spec: $(command grep -E 'WALL|inner dead|Error|Trace' $D/F1_wall_$(echo $spec | tr ' ' '_').raw | tr '\n' ' ')" >> $OUT done echo "-- F2 drop dead (cont_hm_copy.py)" >> $OUT for env in DUMMY DROP_DEAD; do ( export CONT_NOUPPER=1 $env=1; lim $PY -B $D/cont_hm_copy.py 720 1 24 245 720 1/4,1/3,5/12 4 2 2 ) > $D/F2_a245_$env.raw 2>&1 echo "$(date -u +%H:%M:%S) cont_hm_copy N=720 eta=1/24 alpha=245/720 NOUPPER ASUB=SSUB=2 [$env]: $(command grep -E 'DROP_DEAD|LP status|min f\(1\)|Error|Traceback' $D/F2_a245_$env.raw | tr '\n' ' ' | cut -c1-400)" >> $OUT done echo "-- F3 reach at eta = 1/36 (cont_hm.py, N = 720, ASUB = SSUB = 3)" >> $OUT for a in 209 216 230; do ( export CONT_NOUPPER=1; lim $PY -B $D/cont_hm.py 720 1 36 $a 720 1/4,1/3,5/12 4 3 3 ) > $D/F3_eta36_a$a.raw 2>&1 echo "$(date -u +%H:%M:%S) cont_hm N=720 eta=1/36 alpha=$a/720 edges 1/4,1/3,5/12 NOUPPER ASUB=SSUB=3: $(command grep -E 'LP status|min f\(1\)|RSS|Error|Traceback|Memory' $D/F3_eta36_a$a.raw | tr '\n' ' ' | cut -c1-400)" >> $OUT done echo "-- F4 G family at N = 216 (cont_hmg.py, ASUB = SSUB = 2, midpoint edge)" >> $OUT for spec in "72 90" "74 91"; do set -- $spec for nog in 1 0; do ( export CONT_NOUPPER=1; if [ $nog = 1 ]; then export CONT_NOG=1; fi; lim $PY -B $D/cont_hmg.py 216 1 36 $1 216 $2/216 4 2 2 ) > $D/F4_n216_a$1_nog$nog.raw 2>&1 echo "$(date -u +%H:%M:%S) cont_hmg N=216 eta=1/36 alpha=$1/216 edge $2/216 NOG=$nog ASUB=SSUB=2: $(command grep -E 'LP status|min f\(1\)|G in|Error|Traceback' $D/F4_n216_a$1_nog$nog.raw | tr '\n' ' ' | cut -c1-400)" >> $OUT done done echo "== $(date -u +%H:%M:%S) done" >> $OUT ~~~~~~~~~~~~ #### file: run_diag.sh - kind: recipe (a record of what was run; machine-specific helpers are not shipped) - as run: SHA-256 9caa7f500d8fee6af749239cad41b279065dfc3d8a3e97497fa72eb9fff4c193 - as shipped: SHA-256 1e3b89c09379e044014d7abf4a6e7c56f0a003f51c28405bb6f026fdff2e6132 - identity: the recipe as run, with a new comment header and these substitutions: a machine-specific path replaced by `.` (1x); a machine-specific path replaced by `.` (1x); a machine-specific path replaced by `python3` (1x); a machine-specific path replaced by `cpulimit.sh` (1x) - trailing newline: yes ~~~~~~~~~~~~ bash #!/bin/bash # run_diag.sh M NB TAG: width diagnostic (Steps A and B) of the candidate on cert2_f3_diag.py with the engine settings # KW=2:1,3:-2 GVEC=2:1,3:1 T0ADJ=1 F1ADJ=1 NBQ=32 NBQLO=64 QOUTER=1; stopped by hand once the log showed 'J functions ready'. D=.; M=$1; NB=$2; TAG=$3; L=$D/diag_${TAG}_M${M}.log P8=./c_lp_sol_e36a1760_w2_r050_b010_N1800.pkl ( ulimit -v 2900000; exec env OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 KW=2:1,3:-2 GVEC=2:1,3:1 T0ADJ=1 F1ADJ=1 NBQ=32 NBQLO=64 QOUTER=1 \ nice -n 10 cpulimit.sh 7200 python3 -B -I $D/cert2_f3_diag.py $M $NB $P8 $TAG ) > $L 2>&1 & P=$!; echo "$(date -u +%H:%M:%S) $TAG M=$M NB=$NB launched PID $P" >> $D/diag_runs.txt while kill -0 $P 2>/dev/null; do if command grep -q "J functions ready" $L; then echo "$(date -u +%H:%M:%S) $TAG: J ready; killing PID $P by plan" >> $D/diag_runs.txt; kill $P; break; fi sleep 10 done echo "$(date -u +%H:%M:%S) $TAG ended" >> $D/diag_runs.txt ~~~~~~~~~~~~ #### file: run_diag2.sh - kind: recipe (a record of what was run; machine-specific helpers are not shipped) - as run: SHA-256 e3b6b120a117b5c875ba3a547d3170e80d3db182d231b2b9abf12277d96119ab - as shipped: SHA-256 fd9a072aebb3b8ea83394d77a4daf4b7099bb5e73a9e3d389f566c1b1565544b - identity: the recipe as run, with a new comment header and these substitutions: a machine-specific path replaced by `.` (1x); a machine-specific path replaced by `.` (1x); a machine-specific path replaced by `python3` (1x); a machine-specific path replaced by `cpulimit.sh` (1x) - trailing newline: yes ~~~~~~~~~~~~ bash #!/bin/bash # run_diag2.sh M NB TAG: per-cell dump (Step A) of the candidate on cert2_f3_diag2.py with the same settings; stopped by hand once # the log showed 'T1: B in' (the dump has every level by then). D=.; M=$1; NB=$2; TAG=$3; L=$D/diag2_${TAG}_M${M}.log P8=./c_lp_sol_e36a1760_w2_r050_b010_N1800.pkl ( ulimit -v 2900000; exec env OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 KW=2:1,3:-2 GVEC=2:1,3:1 T0ADJ=1 F1ADJ=1 NBQ=32 NBQLO=64 QOUTER=1 \ nice -n 10 cpulimit.sh 10800 python3 -B -I $D/cert2_f3_diag2.py $M $NB $P8 $TAG ) > $L 2>&1 & P=$!; echo "$(date -u +%H:%M:%S) $TAG (diag2) M=$M NB=$NB launched PID $P" >> $D/diag_runs.txt while kill -0 $P 2>/dev/null; do if command grep -q "T1: B in" $L; then sleep 2; echo "$(date -u +%H:%M:%S) $TAG: T1 done; killing PID $P by plan" >> $D/diag_runs.txt; kill $P; break; fi sleep 10 done echo "$(date -u +%H:%M:%S) $TAG ended" >> $D/diag_runs.txt ~~~~~~~~~~~~ #### file: run_p7ctl.sh - kind: recipe (a record of what was run; machine-specific helpers are not shipped) - as run: SHA-256 ff57c65cc846be1b4c705ce62f8c0b0a664933b6a9ec753d9b2aa4116dbee38c - as shipped: SHA-256 4db7468018d0b327d03b551c7112c91c39751c26fd1870dcc9d92290fc304a34 - identity: the recipe as run, with a new comment header and these substitutions: a machine-specific path replaced by `.` (1x); a machine-specific path replaced by `.` (1x); a machine-specific path replaced by `python3` (1x); a machine-specific path replaced by `cpulimit.sh` (1x) - trailing newline: yes ~~~~~~~~~~~~ bash #!/bin/bash # run_p7ctl.sh: control for the first-order loss: the function of the case (1/30, 4/15) (inputs/kw23m_e30a415_reg1030glo93) on # cert2_f3_diag2.py at M = 3840, NB = 80, KW=2:1,3:-1 GVEC=2:1,3:1; Step A dump; stopped by hand at 'T4: B in'. D=.; L=$D/diag2_p7ctl_M3840.log P7=./kw23m_e30a415_reg1030glo93.pkl ( ulimit -v 2900000; exec env OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 KW=2:1,3:-1 GVEC=2:1,3:1 T0ADJ=1 NBQ=32 NBQLO=64 \ nice -n 10 cpulimit.sh 3600 python3 -B -I $D/cert2_f3_diag2.py 3840 80 $P7 p7ctl ) > $L 2>&1 & P=$!; echo "$(date -u +%H:%M:%S) p7ctl (diag2) M=3840 NB=80 launched PID $P" >> $D/diag_runs.txt while kill -0 $P 2>/dev/null; do if command grep -q "T4: B in" $L; then sleep 2; echo "$(date -u +%H:%M:%S) p7ctl: T4 done; killing PID $P by plan" >> $D/diag_runs.txt; kill $P; break; fi sleep 5 done echo "$(date -u +%H:%M:%S) p7ctl ended" >> $D/diag_runs.txt ~~~~~~~~~~~~ #### file: lane_rerun.sh - kind: recipe (a record of what was run; machine-specific helpers are not shipped) - as run: SHA-256 819b515c3674431d79614562a873191491780f4984a83cadd4e139720ad2d8c2 - as shipped: SHA-256 ef66e257490833a610d323b4482e05a40a0b773e57c41d73ccd84b0509d1841c - identity: the recipe as run, with a new comment header and these substitutions: a machine-specific path replaced by `.` (1x) - trailing newline: yes ~~~~~~~~~~~~ bash #!/bin/bash # The lever families at K = 72, eta = 1/36 (m0 = 2), Type I level 1/2 (thI = 36), alpha = 26/72 (thT = 26), caps 4: famrk.py (run # as famrk_v3_0a34eca9.py, a byte copy of the shipped famrk.py) for ct+s3 and then ct+s2, then verify_fake_v2.py on the saved ct+s3 # solution. lane_rerun.sha256 records the SHA-256 of the two scripts at the start. Each run is under a memory and time guard of the # machine (guard_run.sh, not shipped). Floats; not certificates. D=. cd $D/twist/rk/rerun export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 RSS_KB=5000000 AVAIL_KB=4000000 STOP_AT="2026-11-10 23:58:00" AGG_TL=14000 SAVE=1 sha256sum $D/twist/famrk_v3_0a34eca9.py verify_fake_v2.py > lane_rerun.sha256 for fam in ct+s3 ct+s2; do echo "$(date -u '+%F %T') start $fam" >> lane_rerun.log ETA=1/36 ALPHA=26/72 THETA_RULE=half $D/guard_run.sh rk_K72_thT26_cap4_$fam.out 14400 $D/twist/famrk_v3_0a34eca9.py 72 2 36 26 4 $fam echo "$(date -u '+%F %T') end $fam rc $?: $(command grep -m1 '^FAMILY' rk_K72_thT26_cap4_$fam.out | cut -c1-240)" >> lane_rerun.log done echo "$(date -u '+%F %T') start verify" >> lane_rerun.log RSS_KB=4000000 $D/guard_run.sh verify_K72_26_s3.out 3600 verify_fake_v2.py famrk_K72_m02_thI36_thT26_cap4_ct+s3.npz 72 2 36 26 4 3 echo "$(date -u '+%F %T') end verify rc $?: $(command grep '^VERIFY' verify_K72_26_s3.out | cut -c1-240)" >> lane_rerun.log ~~~~~~~~~~~~ #### file: run_rec6.sh - kind: recipe (a record of what was run; machine-specific helpers are not shipped) - as run: SHA-256 4ed494488100408641c433a7e39c30a4aa4422255ce18349658856763c8f1d7a - as shipped: SHA-256 b70bf72030f6abef8a9221a15d8f7f6655d5a1056394dd8b08c7f560d8349d48 - identity: the recipe as run, with a new comment header and these substitutions: a machine-specific path replaced by `.` (1x); a machine-specific path replaced by `python3` (1x); a machine-specific path replaced by `cpulimit.sh` (1x) - trailing newline: yes ~~~~~~~~~~~~ bash #!/bin/bash # Runs of record of the families (cont_hms_rec6.py): every generated positivity row (ASUB = SSUB = MSUB = 1), the rows of # Lemma 8.1 at |D x| <= EPSV (default 1e-9) with dead-row scale 1024, NOUPPER, CONT_NOX (interior point without crossover), presolve off # and then on if the solve is not accepted. Arguments: OUTFILE, then specs "N ETAD alpha_num edge(s) U LBC|none C" (LBC must be none: # cont_hms_rec6.py refuses a lowered bound). Appends one summary line per solve to OUTFILE. Floats; finite programmes; not certificates. export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 FLINT_NUM_THREADS=1 CONT_NOUPPER=1 CONT_NOX=1 CONT_TLIM=14400 CONT_MSUB=${MSUBV:-1} export CONT_DEADEPS=${EPSV:-1e-9} CONT_DEADSCALE=1024 OUTN=$1; shift D=. PY=python3 cd $D OUT=$D/$OUTN echo "== $(date -u +%H:%M:%S) start (pid $$); cont_hms_rec6.py sha $(sha256sum cont_hms_rec6.py | cut -c1-8); specs: $*" >> $OUT for spec in "$@"; do set -- $spec; NN=$1; ETAD=$2; A=$3; E=$4; U=$5; L=$6; GC=$7 for P in 0 1; do AV=$(free -g | awk '/^Mem:/{print $7}') if [ "$AV" -lt ${MINAV:-6} ]; then echo "$(date -u +%H:%M:%S) SKIP (available ${AV} GB < ${MINAV:-6}): $spec" >> $OUT; break; fi TAG=${PFX:-}n${NN}_e${ETAD}_a${A}_u${U}_l${L}_g${GC}_eps${EPSV:-1e-9}_p${P} ( unset CONT_LBC; [ "$L" != none ] && export CONT_LBC=$L export CONT_PRESOLVE=$P CONT_GCAP=$GC CONT_SAVEX=$D/rec/$TAG ulimit -v 4500000 /usr/bin/time -f "RSS_kB %M user %U s; exit %x" nice -n 10 cpulimit.sh 20000 $PY -B $D/cont_hms_rec6.py $NN 1 $ETAD $A $NN $E $U ${SUB:-1} ${SUB:-1} ) > $D/rec/$TAG.raw 2>&1 echo "$(date -u +%H:%M:%S) REC eps=${EPSV:-1e-9} N=$NN eta=1/$ETAD alpha=$A/$NN edges=$E U=$U LBC=$L GCAP=$GC presolve=$P: $(command grep -oE 'REC3: [^\n]*|FREEMIN: [^\n]*|FREEMIN_CLOSED: [^\n]*|REC4: [^\n]*|CONT_LBC = [^;]*; restored [^\n]*|positivity rows: free [0-9]+, extended [0-9]+|REC: .*|RSS_kB [0-9]+ user [0-9.]+|Error.*|Traceback|MemoryError' $D/rec/$TAG.raw | tr '\n' ' ' | cut -c1-1500)" >> $OUT command grep -q "ACCEPT True" $D/rec/$TAG.raw && break done done echo "== $(date -u +%H:%M:%S) done (pid $$)" >> $OUT ~~~~~~~~~~~~ #### file: run_records_fix6.sh - kind: recipe (a record of what was run; machine-specific helpers are not shipped) - as run: SHA-256 3ff0c4b06246d9712afc95112a6fa3b71321176cad36a19708641a9b537f1241 - as shipped: SHA-256 a949dbee803d9fc37406d81481b9f31589ff2e0eaea873e1fdbc47dc4634432b - identity: the recipe as run, with a new comment header and these substitutions: a machine-specific path replaced by `.` (1x) - trailing newline: yes ~~~~~~~~~~~~ bash #!/bin/bash # The runs of record of section 8 on cont_hms_rec6.py through run_rec6.sh: queue "small" (every N = 72 value: the forced # family with two bins and with bins of one grid step, without the rows of Lemma 8.1, the G family, and the family whose data also see # the largest small component; L = 1 throughout) and queue "big" (that family on the finer and other-eta grids). Floats; finite # programmes; not certificates. D=.; cd $D B1="26/72,27/72,28/72,29/72,30/72,31/72,32/72,33/72,34/72,35/72"; B2="27/72,28/72,29/72,30/72,31/72,32/72,33/72,34/72,35/72" case "$1" in small) PFX=R6M_ bash run_rec6.sh recR6_M72.out "72 36 24 30/72 4 none 50" "72 36 25 31/72 4 none 50" "72 36 25 31/72 50 none 50" "72 36 25 31/72 1000 none 1000" \ "72 36 24 30/72 1000 none 1000" "72 36 26 31/72 2000 none 2000" "72 36 27 31/72 2000 none 2000" CONT_TIEM=1 CONT_NOGREC=1 PFX=R6F_ bash run_rec6.sh recR6_F72.out "72 36 23 30/72 4 none 10000" "72 36 24 30/72 4 none 10000" "72 36 25 31/72 4 none 10000" \ "72 36 26 31/72 4 none 10000" "72 36 23 30/72 1000 none 10000" "72 36 24 30/72 1000 none 10000" "72 36 25 31/72 1000 none 10000" "72 36 26 31/72 1000 none 10000" CONT_TIEM=1 CONT_NOGREC=1 EPSV=1e9 PFX=R6Fdrop_ bash run_rec6.sh recR6_Fdrop72.out "72 36 25 31/72 4 none 10000" "72 36 26 31/72 4 none 10000" CONT_TIEM=1 CONT_NOGREC=1 PFX=R6Fbin1_ bash run_rec6.sh recR6_Fbin72.out "72 36 25 $B1 4 none 10000" "72 36 26 $B2 4 none 10000" "72 36 25 $B1 1000 none 10000" "72 36 26 $B2 1000 none 10000" CONT_TIEM=1 PFX=R6G_ bash run_rec6.sh recR6_G72.out "72 36 25 31/72 4 none 50" "72 36 25 31/72 1000 none 1000" ;; big) PFX=R6M_ bash run_rec6.sh recR6_Mbig.out "90 45 31 38/90 4 none 50" "96 48 34 40/96 4 none 50" "108 36 36 45/108 4 none 50" "90 45 32 39/90 4 none 50" \ "96 48 35 41/96 4 none 50" "108 36 37 45/108 4 none 50" "90 30 30 38/90 4 none 50" "108 27 36 45/108 4 none 50" ;; esac ~~~~~~~~~~~~ #### file: run_gsweep.sh - kind: recipe (a record of what was run; machine-specific helpers are not shipped) - as run: SHA-256 fb003e264801782eae5fcad353134642bdd32c8a038fbd3827cbbb228563ff27 - as shipped: SHA-256 a95e92a7a837a475d574d24e2bd8c8473010115c33e07e8e8a3cbf1f2a8e7d73 - identity: the recipe as run, with a new comment header and these substitutions: a machine-specific path replaced by `.` (1x); a machine-specific path replaced by `python3` (1x); a machine-specific path replaced by `cpulimit.sh` (1x) - trailing newline: yes ~~~~~~~~~~~~ bash #!/bin/bash # The one-rough (G) family at 25/72 on cont_hms_rec6.py for U = C in {50, 200, 1000, 5000, 20000}, the forced family # with bins of one grid step at U = 10^4 (Fbin25), and, at U = C = 1000, two diagnostic solves on a variant of the script that restores one # earlier default at a time (keepE, fixvac; cont_hms_rec6x.py). N = 72, eta = 1/36, edge 31/72, the rows of Lemma 8.1 at 1e-9, # every positivity row. Appends one summary line per solve to gsweep.out. Floats; finite programmes; not certificates. export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 CONT_NOUPPER=1 CONT_NOX=1 CONT_MSUB=1 CONT_DEADSCALE=1024 CONT_TLIM=3600 CONT_DEADEPS=1e-9 CONT_NOGATE= D=.; G=$D/gsweep; PY=python3; OUT=$G/gsweep.out echo "== $(date -u +%H:%M:%S) start (pid $$); rec6 $(sha256sum $D/cont_hms_rec6.py | cut -c1-8), rec6x $(sha256sum $G/cont_hms_rec6x.py | cut -c1-8)" >> $OUT one() { # TAG SCRIPT EDGES U C [ENV...] TAG=$1; SCR=$2; E=$3; U=$4; C=$5; shift 5 for P in 0 1; do ( export CONT_GCAP=$C CONT_PRESOLVE=$P CONT_SAVEX=$G/${TAG}_p$P "$@"; ulimit -v 4500000 /usr/bin/time -f "RSS_kB %M user %U s; exit %x" nice -n 10 cpulimit.sh 3600 $PY -B $SCR 72 1 36 25 72 $E $U 1 1 ) > $G/${TAG}_p$P.raw 2>&1 echo "$(date -u +%H:%M:%S) ${TAG}_p$P: $(command grep -oE 'GATE \(trapezoid|PASS|REC: status [A-Za-z ]+; objective pruned [-0-9.]+ raw [-0-9.]+; float dual lower bound pruned [-0-9.]+ raw [-0-9.]+|raw residuals live [0-9.e+-]+ dead [0-9.e+-]+|ACCEPT [A-Za-z]+|Traceback|Error.*' $G/${TAG}_p$P.raw | tr '\n' ' ' | cut -c1-500)" >> $OUT command grep -q "ACCEPT True" $G/${TAG}_p$P.raw && break done } for C in 50 200 1000 5000 20000; do one G25_UC$C $D/cont_hms_rec6.py 31/72 $C $C CONT_TIEM=1; done one G25_UC1000_keepE $G/cont_hms_rec6x.py 31/72 1000 1000 CONT_TIEM=1 CONT_KEEP_E=1 one G25_UC1000_fixvac $G/cont_hms_rec6x.py 31/72 1000 1000 CONT_TIEM=1 CONT_FIX_VAC=1 one Fbin25_U10000 $D/cont_hms_rec6.py 26/72,27/72,28/72,29/72,30/72,31/72,32/72,33/72,34/72,35/72 10000 10000 CONT_TIEM=1 CONT_NOGREC=1 echo "== $(date -u +%H:%M:%S) done (pid $$)" >> $OUT ~~~~~~~~~~~~ #### file: run_rec6g.sh - kind: recipe (a record of what was run; machine-specific helpers are not shipped) - as run: SHA-256 ce877a32e83a87e930eaa0f32d9c20c28158575a766f408f6424df370759d298 - as shipped: SHA-256 759f94fa408993b536016e9a1442c09a91646c599cfad90eec0748270b363a07 - identity: the recipe as run, with a new comment header and these substitutions: a machine-specific path replaced by `.` (1x); a machine-specific path replaced by `python3` (1x); a machine-specific path replaced by `cpulimit.sh` (1x) - trailing newline: yes ~~~~~~~~~~~~ bash #!/bin/bash # run_rec6.sh with cont_hms_rec6g.py (exact support of the G columns): reruns of the G and richer-family records whose G # columns cont_hms_rec6.py omits, with raw logs in gexact/. Floats; finite programmes; not certificates. export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 FLINT_NUM_THREADS=1 CONT_NOUPPER=1 CONT_NOX=1 CONT_TLIM=14400 CONT_MSUB=1 export CONT_DEADEPS=1e-9 CONT_DEADSCALE=1024 D=./gexact; PY=python3 OUT=$D/$1; shift; cd $D echo "== $(date -u +%H:%M:%S) start (pid $$); cont_hms_rec6g.py sha $(sha256sum cont_hms_rec6g.py | cut -c1-8); specs: $*" >> $OUT for spec in "$@"; do set -- $spec; NN=$1; ETAD=$2; A=$3; E=$4; U=$5; GC=$6 for P in 0 1; do TAG=${PFX:-}n${NN}_e${ETAD}_a${A}_u${U}_g${GC}_p${P} ( export CONT_PRESOLVE=$P CONT_GCAP=$GC CONT_SAVEX=$D/$TAG; ulimit -v 4500000 /usr/bin/time -f "RSS_kB %M user %U s; exit %x" nice -n 10 cpulimit.sh 20000 $PY -B $D/cont_hms_rec6g.py $NN 1 $ETAD $A $NN $E $U 1 1 ) > $D/$TAG.raw 2>&1 echo "$(date -u +%H:%M:%S) $TAG: $(command grep -oE 'GEXACT: [0-9]+ G keys|REC: status [A-Za-z ]+; objective pruned [-0-9.]+ raw [-0-9.]+; float dual lower bound pruned [-0-9.]+ raw [-0-9.]+|raw residuals live [0-9.e+-]+ dead [0-9.e+-]+|ACCEPT [A-Za-z]+|FREEMIN_CLOSED: [^;]*= [-0-9.]+|Traceback|Error.*' $D/$TAG.raw | tr '\n' ' ' | cut -c1-600)" >> $OUT command grep -q "ACCEPT True" $D/$TAG.raw && break done done echo "== $(date -u +%H:%M:%S) done (pid $$)" >> $OUT ~~~~~~~~~~~~ #### file: list_keys.sh - kind: recipe (a record of what was run; machine-specific helpers are not shipped) - as run: SHA-256 f35f68efd24703174c5715087052493f57681d7655389f352e807f9101cd2398 - as shipped: SHA-256 2ee75fdaf42e98301ce068af1b3cbea76cc8be0f1b7302419ab834fad7f19168 - identity: the recipe as run, with a new comment header and these substitutions: a machine-specific path replaced by `.` (1x); a machine-specific path replaced by `python3` (1x); a machine-specific path replaced by `cpulimit.sh` (1x) - trailing newline: yes ~~~~~~~~~~~~ bash #!/bin/bash # Lists, with cont_hms_rec6g.py and CONT_GLISTONLY=1, the G keys whose columns cont_hms_rec6.py omits, for every shipped G and # richer-family setting (enumeration only): output keys.out. export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 CONT_GLISTONLY=1 D=./gexact; PY=python3; OUT=$D/keys.out; cd $D echo "== $(date -u +%H:%M:%S) start (pid $$); cont_hms_rec6g.py sha $(sha256sum cont_hms_rec6g.py | cut -c1-8)" >> $OUT for spec in "72 36 24 30/72" "72 36 25 31/72" "72 36 26 31/72" "72 36 27 31/72" "90 45 31 38/90" "90 45 32 39/90" "96 48 34 40/96" "96 48 35 41/96" \ "108 36 36 45/108" "108 36 37 45/108" "90 30 30 38/90" "108 27 36 45/108"; do set -- $spec R=$( ( ulimit -v 4500000; nice -n 10 cpulimit.sh 3000 $PY -B $D/cont_hms_rec6g.py $1 1 $2 $3 $1 $4 4 1 1 ) 2>&1 | command grep -E "GEXACT|Traceback|Error" ) echo "N=$1 eta=1/$2 alpha'=$3/$1 edges=$4: $R" >> $OUT done echo "== $(date -u +%H:%M:%S) done (pid $$)" >> $OUT ~~~~~~~~~~~~ #### file: run_ediag.sh - kind: recipe (a record of what was run; machine-specific helpers are not shipped) - as run: SHA-256 2ac2305a124479c709927fe856188f83628b73592a0b2e7e60224ef1da52446f - as shipped: SHA-256 d88fff9e0f5b3992dd16e1bd207d0bd904046fe71765ea5ee7742f9ed5854d4e - identity: the recipe as run, with a new comment header and these substitutions: a machine-specific path replaced by `.` (1x); a machine-specific path replaced by `python3` (1x); a machine-specific path replaced by `cpulimit.sh` (1x) - trailing newline: yes ~~~~~~~~~~~~ bash #!/bin/bash # Diagnostics of the treatment of E (cont_hms_rec6e.py, cont_hms_rec6e2.py, and one solve with cont_hms_rec6x.py): each line # gives a tag, the script and its settings; presolve off and then on if the solve is not accepted; raw logs in ediag/, summary in # ediag.out. Floats; finite programmes; not certificates. export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 CONT_NOUPPER=1 CONT_NOX=1 CONT_MSUB=1 CONT_DEADSCALE=1024 CONT_TLIM=3600 CONT_DEADEPS=1e-9 D=./ediag; PY=python3; OUT=$D/ediag.out TAG=$1; SCR=$2; set -- $3 "${@:4}"; NN=$1; ETAD=$2; A=$3; E=$4; U=$5; C=$6; shift 6 echo "== $(date -u +%H:%M:%S) start $TAG (pid $$); $(basename $SCR) $(sha256sum $SCR | cut -c1-8); env: $*" >> $OUT for P in 0 1; do ( export CONT_GCAP=$C CONT_PRESOLVE=$P CONT_SAVEX=$D/${TAG}_p$P "$@"; ulimit -v 4500000 /usr/bin/time -f "RSS_kB %M user %U s; exit %x" nice -n 10 cpulimit.sh 3600 $PY -B $SCR $NN 1 $ETAD $A $NN $E $U 1 1 ) > $D/${TAG}_p$P.raw 2>&1 echo "$(date -u +%H:%M:%S) ${TAG}_p$P: $(command grep -oE 'CONT_ECAP: [^\n]*|EOPT: [^\n]*|REC: status [A-Za-z ]+; objective pruned [-0-9.]+ raw [-0-9.]+; float dual lower bound pruned [-0-9.]+ raw [-0-9.]+|raw residuals live [0-9.e+-]+ dead [0-9.e+-]+|ACCEPT [A-Za-z]+|Traceback|Error.*' $D/${TAG}_p$P.raw | tr '\n' ' ' | cut -c1-700)" >> $OUT command grep -q "ACCEPT True" $D/${TAG}_p$P.raw && break done ~~~~~~~~~~~~ #### file: run_einfo.sh - kind: recipe (a record of what was run; machine-specific helpers are not shipped) - as run: SHA-256 8ee646638d0ec86d6ff6f841f4d0140aae0deec10006122a6a44d306d11ae57c - as shipped: SHA-256 35cf223367ade80cb85b1ea90a86f45dbeb660ead10ca7558ac4ac7d7331676f - identity: the recipe as run, with a new comment header and these substitutions: a machine-specific path replaced by `.` (1x); a machine-specific path replaced by `python3` (1x); a machine-specific path replaced by `cpulimit.sh` (1x) - trailing newline: yes ~~~~~~~~~~~~ bash #!/bin/bash # The EINFO line of cont_hms_rec6e.py (CONT_EINFO_ONLY=1: the column and T0 coefficient of the constant c(0, 2, top bin)) for # every grid setting of the records, listed in einfo_settings.txt: output einfo.out. export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 CONT_EINFO_ONLY=1 D=./ediag; PY=python3; OUT=$D/einfo.out echo "== $(date -u +%H:%M:%S) start (pid $$); cont_hms_rec6e.py $(sha256sum $D/cont_hms_rec6e.py | cut -c1-8)" >> $OUT while read -r ARGS; do [ -z "$ARGS" ] && continue R=$( ( ulimit -v 4500000; nice -n 10 cpulimit.sh 3000 $PY -B $D/cont_hms_rec6e.py $ARGS ) 2>&1 | command grep -E "N=[0-9]+ eta|EINFO|Traceback|Error" | tr '\n' ' ') echo "$ARGS | $R" >> $OUT done < $D/einfo_settings.txt echo "== $(date -u +%H:%M:%S) done (pid $$)" >> $OUT ~~~~~~~~~~~~ #### file: run_p5_window.sh - kind: recipe (a record of what was run; machine-specific helpers are not shipped) - as run: SHA-256 78d89324b29315a8122cbb9d17437c7f9f71a30b3848879c1aac6ce565c731b9 - as shipped: SHA-256 78d89324b29315a8122cbb9d17437c7f9f71a30b3848879c1aac6ce565c731b9 - identity: the recipe as run, with a new comment header - trailing newline: yes ~~~~~~~~~~~~ bash #!/bin/sh # run_p5_window.sh: the exact checks of the window masks of sep_p5/p5rig.py (experiments/README.md, "The window masks of # sep_p5/p5rig.py"). Run it from the supplement's top directory after unpacking experiments/ into OUTDIR: # PY= sh OUTDIR/run_p5_window.sh OUTDIR > p5_window_runs.txt # It writes p5rig_oldmasks.py (the shipped program with its two masks as first run) in the current directory, and the reports # exact_shipped.json and exact_oldmasks.json in OUTDIR. O=$1 PY=${PY:-python3} sed -e 's/(c1 < ih - 1)/(c1 <= ih - 1)/' -e 's/(c1 >= ih - l - 2)/(c1 >= ih - l - 1)/' sep_p5/p5rig.py > p5rig_oldmasks.py sha256sum sep_p5/p5rig.py p5rig_oldmasks.py diff sep_p5/p5rig.py p5rig_oldmasks.py "$PY" -B "$O/mask_check.py" sep_p5/p5rig.py p5rig_oldmasks.py "$PY" -B "$O/exact_windows.py" sep_p5/p5rig.py exact_shipped.json; echo "exact_windows.py on sep_p5/p5rig.py: exit status $?" "$PY" -B "$O/exact_windows.py" p5rig_oldmasks.py exact_oldmasks.json; echo "exact_windows.py on p5rig_oldmasks.py: exit status $?" ~~~~~~~~~~~~