#!/usr/bin/env python3 """sidereal_verify — re-check a public Sidereal Research report. Standalone, stdlib-only, no account required. Point it at a public report URL and it verifies, on YOUR machine: 1. FAITHFULNESS — every sentence in the report's proof sidecar appears verbatim in the published page (nothing was edited after the proofs were built), and the sidecar's class tallies match its own entries. 2. LIVE SOURCES — for proofs that carry a supporting quote, fetch the cited source from the open web and confirm the quoted text actually appears there. This check does not depend on Sidereal's servers being honest: the source is fetched from its own host. What this does NOT prove: whether paraphrased (attributed-class) sentences fairly summarize their sources — that is the judged audit's domain, described at https://siderealintelligence.com/methodology — nor the truth of the sources themselves. Sources behind paywalls or taken offline are reported as UNREACHABLE, never as failures. Usage: python3 sidereal_verify.py https://siderealintelligence.com/p/ python3 sidereal_verify.py --sources 25 # more live fetches """ from __future__ import annotations import html import json import re import sys import unicodedata import urllib.request UA = {"User-Agent": "sidereal-verify/1.0 (independent proof check)"} def fetch(url: str, timeout: int = 30) -> str: req = urllib.request.Request(url, headers=UA) with urllib.request.urlopen(req, timeout=timeout) as r: raw = r.read() for enc in ("utf-8", "latin-1"): try: return raw.decode(enc) except UnicodeDecodeError: continue return raw.decode("utf-8", "replace") def to_text(page_html: str) -> str: t = re.sub(r"(?is)<(script|style)[^>]*>.*?", " ", page_html) t = re.sub(r"(?s)<[^>]+>", " ", t) return normalize(html.unescape(t)) def normalize(s: str) -> str: """Whitespace- and typography-insensitive form for containment checks: curly quotes/dashes flattened, accents kept, spacing collapsed. Rendering must not create false mismatches.""" s = unicodedata.normalize("NFKC", s) s = (s.replace("‘", "'").replace("’", "'") .replace("“", '"').replace("”", '"') .replace("–", "-").replace("—", "-") .replace(" ", " ")) # markdown emphasis renders as tags; literal markers never # reach the page text s = s.replace("**", "").replace("`", "").replace("*", "") s = re.sub(r"\s+", " ", s) # inline tags (, ) strip to spaces, leaving "word ," / # "( word" seams the source text never had s = re.sub(r"\s+([.,;:!?)\]])", r"\1", s) s = re.sub(r"([(\[])\s+", r"\1", s) return s.strip().lower() def _quote_in(quote: str, body: str) -> bool: """Full-quote containment, else sentence fragments — live pages inject footnote markers mid-text; require at least half the substantial fragments (all, if only one).""" q = normalize(quote) if q[:300] in body: return True frags = [normalize(f) for f in re.split(r"(?<=[.!?])\s+", quote)] frags = [f for f in frags if len(f) >= 30] if not frags: return False hits = sum(1 for f in frags if f in body) return hits == 1 if len(frags) == 1 else hits >= (len(frags) + 1) // 2 def main() -> int: args = [a for a in sys.argv[1:] if not a.startswith("--")] if not args: print(__doc__) return 2 url = args[0].rstrip("/") max_sources = 10 if "--sources" in sys.argv: max_sources = int(sys.argv[sys.argv.index("--sources") + 1]) print(f"report : {url}") page = to_text(fetch(url)) doc = json.loads(fetch(url + "/proofs.json")) proofs = doc.get("proofs") or [] print(f"sidecar: {len(proofs)} sentence proofs, " f"version {doc.get('proofs_version')}") # ── 1. faithfulness: sidecar ↔ published page ──────────────────── # Sidecar sentences carry [^src-…] citation markers; the published # page renders those as superscript numerals. Split at the markers # and require every substantial fragment to appear. def in_page(sentence: str) -> bool: frags = [normalize(f) for f in re.split(r"\[\^[^\]]+\]", sentence)] frags = [f for f in frags if len(f) >= 20] if not frags: frags = [normalize(re.sub(r"\[\^[^\]]+\]", " ", sentence))] return all(f in page for f in frags) missing = [p for p in proofs if not in_page(p.get("sentence", ""))] tally: dict[str, int] = {} for p in proofs: c = p.get("entailment_class", "?") tally[c] = tally.get(c, 0) + 1 declared = doc.get("classes") or {} tally_ok = declared == tally count_ok = doc.get("sentence_count") == len(proofs) print("\n[1] faithfulness") print(f" sentences present in page : {len(proofs) - len(missing)}" f"/{len(proofs)}") for p in missing[:5]: print(f" MISSING: {p.get('sentence', '')[:90]!r}") print(f" class tally matches : " f"{'yes' if tally_ok else f'NO — declared {declared}, counted {tally}'}") print(f" sentence count matches : {'yes' if count_ok else 'NO'}") # ── 2. live sources: quoted evidence exists where cited ───────── srcmap = doc.get("sources") or {} quoted = [p for p in proofs if p.get("quote") and srcmap.get(p.get("quote_source") or (p.get("source_ids") or [""])[0], {}).get("url")] print(f"\n[2] live cited sources (checking up to {max_sources} of " f"{len(quoted)} quote-bearing proofs)") confirmed = unreachable = absent = 0 cache: dict[str, str | None] = {} for p in quoted[:max_sources]: sid = p.get("quote_source") or (p.get("source_ids") or [""])[0] surl = srcmap[sid]["url"] if surl not in cache: try: cache[surl] = to_text(fetch(surl)) except Exception: cache[surl] = None body = cache[surl] if body is None: unreachable += 1 print(f" UNREACHABLE {surl[:70]}") elif _quote_in(p["quote"], body): confirmed += 1 else: absent += 1 print(f" QUOTE NOT FOUND at {surl[:60]}") print(f" {p['quote'][:100]!r}") print(f" confirmed in live source : {confirmed}") print(f" unreachable (paywall/gone): {unreachable}") print(f" not found (investigate) : {absent}") faithful = not missing and tally_ok and count_ok if not faithful: verdict = "DISCREPANCIES FOUND — the published page does not " \ "match its proof record" elif absent: verdict = ("PASS WITH NOTES — page matches its proof record; " f"{absent} quote(s) not found in today's live " "source (pages drift, full text may sit behind a " "landing page — investigate)") else: verdict = "PASS" print(f"\nverdict: {verdict}") print("(unreachable sources are never failures)") return 0 if faithful else 1 if __name__ == "__main__": sys.exit(main())