#!/usr/bin/env python3 """Verify the frozen Markdown Layer repository panel (Python 3.10+). Offline metadata checks: python3 reproduce-panel.py --manifest repository-panel.json Recompute from privately saved raw tree responses, without network: python3 reproduce-panel.py --manifest repository-panel.json --raw-directory raw Read the original 21 complete pinned trees from GitHub, skipping the 3 unknowns: python3 reproduce-panel.py --manifest repository-panel.json --fetch-pinned --output NEW_DIRECTORY The network mode makes at most 22 read-only API requests (rate check + 21 trees), never follows moving branch names, retries failures, or reads file contents. It requires an empty output directory. Optional auth: GH_TOKEN/GITHUB_TOKEN or an existing `gh` CLI login. Credentials are never printed or written. No new license grant is made by this file; see the project methodology. """ import argparse,collections,datetime,hashlib,json,os,pathlib,shutil,subprocess,urllib.request,urllib.error API='https://api.github.com';LIMIT=12*1024*1024 NAMES=('AGENTS.md','CLAUDE.md','GEMINI.md','copilot-instructions.md') def digest(data):return hashlib.sha256(data).hexdigest() def derive(tree): assert tree.get('truncated') is False,'Provider tree incomplete' rows=[r for r in tree['tree'] if r['type']=='blob' and r['mode'] in ('100644','100755')] paths={r['path'] for r in rows};md=[r for r in rows if pathlib.PurePosixPath(r['path']).suffix.lower() in ('.md','.markdown')] categories={n:[] for n in NAMES} for r in rows: p=r['path'];name=p.rsplit('/',1)[-1] if name in NAMES[:3]:categories[name].append(p) elif p=='.github/copilot-instructions.md':categories['copilot-instructions.md'].append(p) named=sum(categories.values(),[]) return {'regularTrackedFiles':len(rows),'regularTrackedBlobBytes':sum(r['size'] for r in rows),'markdownFiles':len(md),'markdownBytes':sum(r['size'] for r in md),'mdxFiles':sum(pathlib.PurePosixPath(r['path']).suffix.lower()=='.mdx' for r in rows),'rootMarkdownReadme':any(p.lower() in ('readme.md','readme.markdown') for p in paths),'docsDirectory':any(r['type']=='tree' and r['path']=='docs' for r in tree['tree']),'namedContextCandidateFiles':len(named),'namedContextCandidatePresent':bool(named),'namedContextNestedFiles':sum('/' in p for p in named),'namedContextCategories':{n:len(v) for n,v in categories.items()},'symlinkEntries':sum(r['mode']=='120000' for r in tree['tree']),'submoduleEntries':sum(r['mode']=='160000' for r in tree['tree'])} def token(): for name in ('GH_TOKEN','GITHUB_TOKEN'): if os.environ.get(name):return os.environ[name] if shutil.which('gh'): p=subprocess.run(['gh','auth','token'],capture_output=True,text=True,timeout=15) if p.returncode==0:return p.stdout.strip() or None return None def main(): parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--manifest',type=pathlib.Path,required=True) mode=parser.add_mutually_exclusive_group();mode.add_argument('--raw-directory',type=pathlib.Path);mode.add_argument('--fetch-pinned',action='store_true');parser.add_argument('--output',type=pathlib.Path) args=parser.parse_args();d=json.loads(args.manifest.read_text());repos=d['repositories'];complete=[r for r in repos if r['treeComplete']] assert len(repos)==d['metrics']['repositoriesSelected'] and len(complete)==d['metrics']['completeTrees'] assert sum(r['markdownFiles'] for r in complete)==d['metrics']['markdownFileCount'] assert sum(r['markdownBytes'] for r in complete)==d['metrics']['markdownBlobBytes'] assert sum(r['namedContextCandidatePresent'] for r in complete)==d['metrics']['repositoriesWithNamedContextCandidates'] assert all(r['markdownFiles'] is None for r in repos if not r['treeComplete']) report={'manifestSha256':digest(args.manifest.read_bytes()),'verifiedAt':datetime.datetime.now(datetime.timezone.utc).isoformat(),'mode':'metadata-only','selectedRepositories':len(repos),'completeTrees':len(complete),'originalUnknowns':[r['full_name'] for r in repos if not r['treeComplete']],'requests':[],'trees':[]} cached={} if args.raw_directory: for p in args.raw_directory.glob('*.json'): raw=p.read_bytes();h=digest(raw) if h in {r['rawTreeSha256'] for r in complete}:cached[h]=raw report['mode']='offline-original-tree-recompute' auth=token() if args.fetch_pinned else None def get(url,filename): headers={'Accept':'application/vnd.github+json','User-Agent':'MD2FILE-Markdown-Layer-Reproduction-1.0','X-GitHub-Api-Version':'2022-11-28'} if auth:headers['Authorization']='Bearer '+auth try:response=urllib.request.urlopen(urllib.request.Request(url,headers=headers),timeout=40) except urllib.error.HTTPError as e:response=e with response:raw=response.read(LIMIT+1);status=response.status record={'url':url,'httpStatus':status,'receivedBytes':len(raw),'completeResponse':len(raw)<=LIMIT,'sha256':digest(raw),'savedFile':filename};report['requests'].append(record) (args.output/filename).write_bytes(raw);(args.output/filename).chmod(0o600) if status!=200 or len(raw)>LIMIT:raise RuntimeError('Request failed or exceeded12MiB; no retry performed') return raw if args.fetch_pinned: if not args.output:parser.error('--fetch-pinned requires --output') if args.output.exists() and any(args.output.iterdir()):parser.error('Output directory must be empty; refusing duplicate collection') args.output.mkdir(parents=True,exist_ok=True);args.output.chmod(0o700);report['mode']='pinned-tree-refetch' try: if args.fetch_pinned: rate=json.loads(get(API+'/rate_limit','rate.json')) if rate['resources']['core']['remaining']