#!/usr/bin/env python3 """Generate or check the notices embedded in the Slides release.""" import hashlib import json import os from pathlib import Path import subprocess import sys import tempfile from html.parser import HTMLParser ROOT = Path(__file__).resolve().parent.parent OUT = ROOT / 'internal/slides/credits' ENV = os.environ | {'GOPRIVATE': 'amadan.net/oficina/estilo.git'} EXCLUDE = ','.join(['amadan.net/oficina/estilo.git', 'amadan.net/oficina/home/handoff', 'amadan.net/oficina/home/connect']) def run(args, cwd=ROOT, env=ENV): return subprocess.check_output(args, cwd=cwd, env=env) def digest(path): return hashlib.sha256(path.read_bytes()).hexdigest() def inputs(): files = [ROOT / p for p in ['core/Cargo.toml', 'core/Cargo.lock', 'core/pdf/Cargo.toml', 'core/pdf/Cargo.lock', 'rust-toolchain.toml', 'fixtures/package.json', 'fixtures/package-lock.json', 'internal/slides/core/testdata/slides_core.wasm', 'internal/slides/pdfcore/testdata/slides_pdf.wasm']] for pattern in ['internal/slides/static/*.js', 'internal/slides/fonts/files/*', 'internal/slides/static/fonts/*', 'internal/slides/credits/upstream/*']: files.extend(ROOT.glob(pattern)) return {str(p.relative_to(ROOT)): digest(p) for p in sorted(files) if p.is_file()} def notices(directory): result = [] for path in sorted(directory.rglob("*")): lower = path.name.lower() if path.suffix.lower() in [".go", ".rs", ".js", ".ts", ".c", ".h", ".py"]: continue if path.is_file() and any(lower == word or lower.startswith(word + '.') or lower.startswith(word + '-') or lower.startswith(word + '_') for word in ['license', 'licence', 'copying', 'notice', 'copyright', 'patents']): if path.is_symlink() or path.stat().st_size > 2 * 1024 * 1024: raise ValueError('notice must be a bounded regular file') text = path.read_bytes().decode('utf-8') if not text.strip(): raise ValueError('empty notice') result.append({'Name': str(path.relative_to(directory)), 'Text': text}) return result def go_inventory(): with tempfile.TemporaryDirectory() as temp: tool = str(Path(temp) / 'notices') host = json.loads(run(['go', 'env', '-json', 'GOHOSTOS', 'GOHOSTARCH'])) run(['go', 'build', '-o', tool, 'amadan.net/oficina/estilo.git/cmd/oficina-notices'], env=ENV | {'GOOS': host['GOHOSTOS'], 'GOARCH': host['GOHOSTARCH'], 'CGO_ENABLED': '0'}) return json.loads(run([tool, '-package', './cmd/slides', '-lucide', '-onest', '-exclude', EXCLUDE], env=ENV | {'GOOS': 'linux', 'GOARCH': 'arm64', 'CGO_ENABLED': '0', 'GOWORK': 'off'})) class NoticeText(HTMLParser): def __init__(self): super().__init__(convert_charrefs=True) self.parts = [] def handle_data(self, data): self.parts.append(data) def handle_starttag(self, tag, attrs): if tag in ['p', 'h1', 'h2', 'h3', 'li', 'pre', 'br']: self.parts.append('\n') def asset_inventory(): fixtures = ROOT / 'fixtures' run(['npm', 'ci', '--ignore-scripts', '--no-audit', '--no-fund'], cwd=fixtures) js = r''' import { build } from 'esbuild'; import { readFileSync, readdirSync } from 'node:fs'; const inputs = new Set(); for (const [name, entry] of [['editor', 'editor-entry.js']]) { const result = await build({entryPoints:[entry],bundle:true,format:'esm',minify:true,target:'es2020',outfile:`../internal/slides/static/${name}.js`,write:false,metafile:true}); const original = readdirSync('../internal/slides/static').filter(p => p.startsWith(name+'.') && p.endsWith('.js')); if (original.length !== 1 || !Buffer.from(result.outputFiles.find(p => p.path.endsWith('.js')).contents).equals(readFileSync('../internal/slides/static/'+original[0]))) throw new Error('committed bundle differs from locked source'); for (const path of Object.keys(result.metafile.inputs)) if (path.includes('node_modules/')) inputs.add(path); } console.log(JSON.stringify([...inputs])); ''' bundled = json.loads(run(['node', '--input-type=module', '-e', js], cwd=fixtures)) directories = set() for name in bundled: directory = (fixtures / name).resolve().parent node_root = (fixtures / 'node_modules').resolve() while not (directory / 'package.json').is_file(): if directory == node_root or not directory.is_relative_to(node_root): raise ValueError('bundled package provenance is missing') directory = directory.parent if not directory.is_relative_to(node_root): raise ValueError('bundled package escapes node_modules') directories.add(directory) components = [] for directory in sorted(directories): package = json.loads((directory / 'package.json').read_text()) components.append({'Name': package['name'], 'Version': package['version'], 'SourceURL': 'https://www.npmjs.com/package/' + package['name'] + '/v/' + package['version'], 'Notices': notices(directory)}) selected = {} for crate, script, built, embedded in [ ('core', 'scripts/build-wasm.sh', 'core/target/wasm32-unknown-unknown/release/slides_core.wasm', 'internal/slides/core/testdata/slides_core.wasm'), ('core/pdf', 'scripts/build-pdf-wasm.sh', 'core/pdf/target/wasm32-unknown-unknown/release/slides_pdf.wasm', 'internal/slides/pdfcore/testdata/slides_pdf.wasm')]: run(['bash', script]) if (ROOT / built).read_bytes() != (ROOT / embedded).read_bytes(): raise ValueError('committed Wasm differs from locked source') metadata = json.loads(run(['cargo', 'metadata', '--locked', '--filter-platform', 'wasm32-unknown-unknown', '--format-version', '1'], cwd=ROOT / crate)) packages = {p['id']: p for p in metadata['packages']} nodes = {p['id']: p for p in metadata['resolve']['nodes']} pending, seen = [metadata['resolve']['root']], set() while pending: current = pending.pop() if current in seen: continue seen.add(current) pending.extend(d['pkg'] for d in nodes[current]['deps'] if any(k['kind'] != 'dev' for k in d['dep_kinds'])) for identity in seen: package = packages[identity] if identity != metadata['resolve']['root']: if package['source'] != 'registry+https://github.com/rust-lang/crates.io-index': raise ValueError('unverified Cargo source replacement') selected[(package['name'], package['version'])] = package for key in sorted(selected): package = selected[key] directory = Path(package['manifest_path']).parent texts = notices(directory) if package['name'] in ['yrs', 'krilla', 'krilla-svg']: expected = '3996e91df00acdb67334695944bc43f60627103f' if package['name'] == 'yrs' else '3ffdf0588cf98050aad6edba51ca70162e1fb5b5' vcs = json.loads((directory / '.cargo_vcs_info.json').read_text()) if vcs['git']['sha1'] != expected: raise ValueError('upstream source revision changed') for record in json.loads((OUT / 'upstream/provenance.json').read_text()): prefix = 'yrs-' if package['name'] == 'yrs' else 'krilla-' if not record['file'].startswith(prefix): continue path = OUT / 'upstream' / record['file'] if digest(path) != record['sha256']: raise ValueError('original notice checksum changed') texts.append({'Name': record['file'], 'Text': path.read_bytes().decode('utf-8')}) components.append({'Name': package['name'], 'Version': package['version'], 'SourceURL': 'https://crates.io/crates/' + package['name'] + '/' + package['version'], 'Notices': texts}) sysroot = Path(run(['rustc', '--print', 'sysroot']).decode().strip()) rust_version = run(['rustc', '--version']).decode().split()[1] text = NoticeText() text.feed((sysroot / 'share/doc/rust/COPYRIGHT-library.html').read_text()) components.append({'Name': 'Rust standard library', 'Version': rust_version, 'SourceURL': 'https://doc.rust-lang.org/' + rust_version + '/COPYRIGHT-library.html', 'Notices': [{'Name': 'Copyright notices', 'Text': ''.join(text.parts)}]}) components.append({'Name': 'Liberation Fonts', 'SourceURL': 'https://github.com/liberationfonts/liberation-fonts', 'Notices': notices(ROOT / 'internal/slides/fonts/files')}) if any(not item['Notices'] for item in components): raise ValueError('an asset dependency lacks its original notice') return {'inputs': inputs(), 'components': components} def main(): if sys.argv[1:] not in [[], ['--check']]: raise SystemExit('usage: scripts/credits.py [--check]') checking = sys.argv[1:] == ['--check'] assets_file = OUT / 'assets.json' assets = json.loads(assets_file.read_text()) if checking else asset_inventory() if assets['inputs'] != inputs(): raise SystemExit('credit asset inventory is stale; run scripts/credits.py') inventory = go_inventory() inventory['components'].extend(assets['components']) inventory['components'].sort(key=lambda item: item['Name']) target = OUT / 'notices.json' if checking: if json.loads(target.read_text()) != inventory: raise SystemExit('embedded credits are stale; run scripts/credits.py') else: assets_file.write_text(json.dumps(assets, indent=2) + '\n') target.write_text(json.dumps(inventory, indent=2) + '\n') print('credits: ' + ('checked' if checking else 'generated')) if __name__ == '__main__': main()