#!/usr/bin/env python3 """CI release artifacts and the SHA-specific Gitea deployment gate (stdlib only).""" import argparse import hashlib import io import itertools import json import os import re import shutil import subprocess import sys import tempfile import urllib.error import urllib.parse import urllib.request import zipfile from pathlib import Path SHA = re.compile(r'[0-9a-f]{40}') DIGEST = re.compile(r'sha256:[0-9a-f]{64}') IMAGES = { 'error-pages': ('errorpages', 'errorpages/Dockerfile'), 'forust-homepage': ('homepages', 'homepages/Dockerfile.forust'), 'xdfnx-homepage': ('homepages', 'homepages/Dockerfile.xdfnx'), } def command(*args, **kwargs): """Arguments are passed directly to the executable, never to a shell.""" return subprocess.check_output(args, text=True, **kwargs).strip() # noqa: S603, S607 def validate_release(data, sha=None): if data.get('version') != 1 or not SHA.fullmatch(data.get('sha', '')): raise ValueError('Invalid release version or SHA') if sha is not None and data['sha'] != sha: raise ValueError('Release SHA does not match the checked CI commit') expected = {f'gcr.forust.xyz/forust/{name}' for name in IMAGES} if set(data.get('images', {})) != expected: raise ValueError('Release must contain all owned images') if not all(DIGEST.fullmatch(value) for value in data['images'].values()): raise ValueError('Release has an invalid image digest') if set(data.get('inputs', {})) != expected or not all( re.fullmatch(r'[0-9a-f]{64}', value) for value in data['inputs'].values() ): raise ValueError('Release has invalid build input fingerprints') return data class NoRedirect(urllib.request.HTTPRedirectHandler): def redirect_request(self, _req, _fp, _code, _msg, _headers, _newurl): return None class Gitea: def __init__(self): self.origin = os.environ['GITHUB_SERVER_URL'].rstrip('/') if urllib.parse.urlsplit(self.origin).scheme != 'https': raise ValueError('Gitea API must use HTTPS') self.repository = os.environ['GITHUB_REPOSITORY'] if not re.fullmatch(r'[\w.-]+/[\w.-]+', self.repository): raise ValueError('Invalid Gitea repository') self.token = os.environ['GITEA_TOKEN'] self.base = f'{self.origin}/api/v1/repos/{self.repository}' def request(self, url, *, archive=False): if not url.startswith(self.base + '/'): raise ValueError('Refusing to send the Actions token to another origin') req = urllib.request.Request(url, headers={'Authorization': f'token {self.token}'}) # noqa: S310 -- HTTPS origin validated above opener = urllib.request.build_opener(NoRedirect()) try: response = opener.open(req, timeout=30) # noqa: S310 except urllib.error.HTTPError as error: if not archive or error.code not in (301, 302, 303, 307, 308): raise RuntimeError(f'Gitea API returned HTTP {error.code}') from None target = urllib.parse.urljoin(url, error.headers['Location']) if urllib.parse.urlsplit(target).scheme != 'https': raise ValueError('Artifact redirect must use HTTPS') from None # Signed storage redirects must never receive the Gitea token. response = urllib.request.urlopen(target, timeout=30) # noqa: S310 with response: payload = response.read(8 * 1024 * 1024 + 1) if len(payload) > 8 * 1024 * 1024: raise ValueError('Gitea response exceeds 8 MiB') return payload if archive else json.loads(payload) def pages(self, path, key, **params): for page in range(1, 101): query = urllib.parse.urlencode({**params, 'page': page, 'limit': 50}) data = self.request(f'{self.base}/{path}?{query}') entries = data[key] yield from entries if len(entries) < 50: return raise RuntimeError('Gitea pagination limit exceeded') def successful_runs(self, sha=None): params = {'branch': 'main', 'status': 'success', 'exclude_pull_requests': 'true'} if sha: params['head_sha'] = sha for run in self.pages('actions/workflows/ci.yaml/runs', 'workflow_runs', **params): if ( run.get('status') == 'completed' and run.get('conclusion') == 'success' and run.get('head_branch') == 'main' and run.get('event') in ('push', 'workflow_dispatch') and (run.get('repository') or {}).get('full_name') == self.repository and (run.get('head_repository') or run.get('repository') or {}).get('full_name') == self.repository and (sha is None or run.get('head_sha') == sha) ): yield run def release(self, run): sha = run['head_sha'] jobs = list(self.pages(f'actions/runs/{run["id"]}/jobs', 'jobs')) # A green workflow with a skipped build must not authorize a deploy. if not any(job.get('name') == 'build' and job.get('conclusion') == 'success' for job in jobs): raise ValueError('CI build job did not succeed') artifacts = self.request(f'{self.base}/actions/runs/{run["id"]}/artifacts')['artifacts'] matching = [a for a in artifacts if a['name'] == f'release-{sha}' and not a.get('expired')] if len(matching) != 1: raise ValueError('CI release artifact is missing, expired or ambiguous; rerun CI') blob = self.request(f'{self.base}/actions/artifacts/{matching[0]["id"]}/zip', archive=True) with zipfile.ZipFile(io.BytesIO(blob)) as archive: files = [entry for entry in archive.infolist() if not entry.is_dir()] if len(files) != 1 or files[0].filename != 'release.json' or files[0].file_size > 256 * 1024: raise ValueError('Unexpected release archive contents') return validate_release(json.loads(archive.read(files[0])), sha) def fingerprint(context, dockerfile): tree = command('git', 'ls-tree', '-r', 'HEAD', '--', context, dockerfile, '.gitea/workflows/release.py') return hashlib.sha256(tree.encode()).hexdigest() def gate(output, requested_ref, event_sha): command('git', 'fetch', '--quiet', 'origin', 'main') if event_sha: if not SHA.fullmatch(event_sha): raise ValueError('Invalid workflow_run SHA') sha = event_sha else: if requested_ref == 'main': requested_ref = 'origin/main' sha = command('git', 'rev-parse', '--verify', '--end-of-options', f'{requested_ref}^{{commit}}') if not SHA.fullmatch(sha): raise ValueError('Invalid deploy SHA') command('git', 'merge-base', '--is-ancestor', sha, 'origin/main') api = Gitea() runs = list(api.successful_runs(sha)) if not runs: raise ValueError(f'No successful main CI for {sha}; run CI before deploying') release = api.release(max(runs, key=lambda run: run['id'])) output.write_text(json.dumps(release, indent=2) + '\n') if os.environ.get('GITHUB_OUTPUT'): with Path(os.environ['GITHUB_OUTPUT']).open('a') as stream: stream.write(f'sha={sha}\n') print(f'CI gate accepted {sha}') def prepare_images(output): sha = command('git', 'rev-parse', 'HEAD') if sha != os.environ['GITHUB_SHA'] or not SHA.fullmatch(sha): raise ValueError('Build checkout does not match GITHUB_SHA') api = Gitea() previous = None for run in sorted(itertools.islice(api.successful_runs(), 50), key=lambda item: item['id'], reverse=True): if str(run['id']) == os.environ.get('GITHUB_RUN_ID'): continue try: previous = api.release(run) break except ValueError: # Expired artifacts only cost a rebuild; mutable tags are never a fallback. continue targets = [] for name, (context, dockerfile) in IMAGES.items(): image = f'gcr.forust.xyz/forust/{name}' inputs = fingerprint(context, dockerfile) old_digest = (previous or {}).get('images', {}).get(image) targets.append( { 'name': name, 'image': image, 'context': context, 'dockerfile': dockerfile, 'inputs': inputs, 'reuse_digest': old_digest if (previous or {}).get('inputs', {}).get(image) == inputs else None, } ) output.write_text(json.dumps({'sha': sha, 'targets': targets}, indent=2) + '\n') if os.environ.get('GITHUB_OUTPUT'): with Path(os.environ['GITHUB_OUTPUT']).open('a') as stream: stream.write('matrix=' + json.dumps({'include': targets}, separators=(',', ':')) + '\n') print(f'Prepared {len(targets)} image jobs; {sum(t["reuse_digest"] is None for t in targets)} require builds') def checked_plan(path): data = json.loads(path.read_text()) sha = command('git', 'rev-parse', 'HEAD') if data.get('sha') != sha or sha != os.environ['GITHUB_SHA'] or not SHA.fullmatch(sha): raise ValueError('Image plan does not match the checked source commit') targets = data.get('targets', []) if sorted(t['name'] for t in targets) != sorted(IMAGES): raise ValueError('Image plan must contain each owned image once') for target in targets: name = target['name'] context, dockerfile = IMAGES[name] if (target['context'], target['dockerfile'], target['image']) != ( context, dockerfile, f'gcr.forust.xyz/forust/{name}', ) or target['inputs'] != fingerprint(context, dockerfile): raise ValueError('Image plan has invalid build inputs') if target['reuse_digest'] is not None and not DIGEST.fullmatch(target['reuse_digest']): raise ValueError('Image plan has an invalid reuse digest') return data def build_images(output, report, name, plan): data = checked_plan(plan) sha = data['sha'] target = next(t for t in data['targets'] if t['name'] == name) context, dockerfile = IMAGES[name] docker_config = tempfile.mkdtemp(prefix='homelab-registry-') builder_config = Path.home() / '.cache/homelab-ci/buildx' builder_config.mkdir(parents=True, exist_ok=True) env = {**os.environ, 'DOCKER_CONFIG': docker_config, 'BUILDX_CONFIG': str(builder_config)} try: report['phase'] = 'Registry login' subprocess.run( # noqa: S603, S607 [ shutil.which('docker') or '/usr/bin/docker', 'login', 'gcr.forust.xyz', '-u', os.environ['REGISTRY_USERNAME'], '--password-stdin', ], input=os.environ['REGISTRY_PASSWORD'], text=True, check=True, env=env, ) report['phase'] = 'Prepare the builder' builder = 'homelab-ci' versions = dict( re.findall(r'^([A-Z_]+)="([^"\n]+)"$', Path('.gitea/workflows/tool-versions.env').read_text(), re.MULTILINE) ) image = versions['BUILDKIT_IMAGE'] signature = builder_config / 'homelab-ci-image' exists = ( subprocess.run( # noqa: S603 [shutil.which('docker') or '/usr/bin/docker', 'buildx', 'inspect', builder], capture_output=True, env=env, ).returncode == 0 ) if exists and (not signature.exists() or signature.read_text().strip() != image): command('docker', 'buildx', 'rm', '--keep-state', builder, env=env) exists = False if not exists: command( 'docker', 'buildx', 'create', '--name', builder, '--driver', 'docker-container', '--driver-opt', f'image={image}', '--buildkitd-config', '.gitea/runner/buildkitd.toml', env=env, ) signature.write_text(image + '\n') release = {'version': 1, 'sha': sha, 'images': {}, 'inputs': {}} report['images'] = release['images'] report['phase'] = f'Build or reuse {name}' report['current'] = name image = f'gcr.forust.xyz/forust/{name}' inputs = target['inputs'] old_digest = target['reuse_digest'] exists = False if old_digest: exists = ( subprocess.run( # noqa: S603, S607 [ shutil.which('docker') or '/usr/bin/docker', 'buildx', 'imagetools', 'inspect', f'{image}@{old_digest}', ], capture_output=True, env=env, timeout=60, ).returncode == 0 ) if exists: print(f'Reuse {name}: inputs unchanged') digest = old_digest report['reused'].append(name) else: print(f'Build {name}', flush=True) metadata = Path(docker_config) / 'metadata.json' command( 'docker', 'buildx', 'build', '--builder', builder, '--platform', 'linux/amd64', '--provenance=false', '--cache-from', f'type=registry,ref={image}:buildcache', '--cache-to', f'type=registry,ref={image}:buildcache,mode=max', '--output', f'type=image,name={image},push-by-digest=true,name-canonical=true,push=true', '--metadata-file', str(metadata), '--file', dockerfile, context, env=env, ) digest = json.loads(metadata.read_text())['containerimage.digest'] report['built'].append(name) release['images'][image] = digest release['inputs'][image] = inputs if not DIGEST.fullmatch(digest): raise ValueError('Image job returned an invalid digest') output.write_text(json.dumps(release, indent=2) + '\n') report['current'] = None report['phase'] = 'Release file saved' finally: # Cleanup errors must neither leak credentials nor mask the original build error. try: subprocess.run( # noqa: S603 [ shutil.which('docker') or '/usr/bin/docker', 'buildx', 'prune', '--builder', 'homelab-ci', '--force', '--max-used-space', '1gb', ], env=env, timeout=60, ) except (OSError, subprocess.TimeoutExpired): print('CI builder cache cleanup deferred', flush=True) finally: shutil.rmtree(docker_config) def write_summary(lines): path = os.environ.get('GITHUB_STEP_SUMMARY') if path: try: with Path(path).open('a') as stream: stream.write('\n'.join(lines) + '\n\n') except OSError: print('WARNING: cannot write the job summary') def check_summary(): lines = [ f'## {os.environ["SUMMARY_CHECK"]}', '', f'- Commit: `{os.environ.get("GITHUB_SHA", "unknown")}`', f'- Result: **{os.environ["SUMMARY_RESULT"]}**', ] if os.environ.get('SUMMARY_FAILED_STEP'): lines.append(f'- Failed step: {os.environ["SUMMARY_FAILED_STEP"]}') if os.environ['SUMMARY_RESULT'] != 'success': lines.append('- Open the failed step log for the error details.') write_summary(lines) def build(output, name, plan): report = {'phase': 'Check the source commit', 'current': None, 'built': [], 'reused': [], 'images': {}} result = 'failure' try: build_images(output, report, name, plan) result = 'success' finally: lines = [ f'## Image release `{os.environ.get("GITHUB_SHA", "unknown")}`', '', f'- Result: **{result}**', f'- Last stage: {report["phase"]}', ] if result == 'failure': lines.append('- No release from this build can be deployed. Open the failed step log.') if report['current']: lines.append(f'- Image at the failure: `{report["current"]}`') for title, key in (('Built', 'built'), ('Reused from successful CI', 'reused')): lines.extend(['', f'### {title}']) lines.extend(f'- `{name}`' for name in report[key]) if not report[key]: lines.append('- None') lines.extend(['', '### Completed image digests']) lines.extend(f'- `{image}@{digest}`' for image, digest in report['images'].items()) if not report['images']: lines.append('- None') write_summary(lines) def render(stream, destination): release = validate_release(json.loads(Path(os.environ['RELEASE_FILE']).read_text()), os.environ['DEPLOY_SHA']) image_line = re.compile( r"^(\s*(?:-\s*)?image:\s*)(['\"]?)(gcr\.forust\.xyz/forust/[\w.-]+)(?::[\w.-]+|@sha256:[0-9a-f]{64})\2(\s*(?:#.*)?)$" ) rendered = [] for line in stream: match = image_line.fullmatch(line.rstrip('\n')) if match: prefix, quote, image, tail = match.groups() if image not in release['images']: raise ValueError(f'Owned image missing from checked release: {image}') line = f'{prefix}{quote}{image}@{release["images"][image]}{quote}{tail}\n' elif re.match(r'\s*(?:-\s*)?image:', line) and 'gcr.forust.xyz/forust/' in line: raise ValueError('Unsupported owned image syntax; refusing to apply a mutable tag') rendered.append(line) destination.writelines(rendered) def finalize_images(output, fragments, plan): data = checked_plan(plan) sha = data['sha'] release = {'version': 1, 'sha': sha, 'images': {}, 'inputs': {}} for name in IMAGES: fragment = json.loads((fragments / f'image-{name}' / 'image.json').read_text()) image = f'gcr.forust.xyz/forust/{name}' if fragment.get('sha') != sha or fragment.get('version') != 1 or set(fragment.get('images', {})) != {image}: raise ValueError('Image job artifact is missing or belongs to another commit') target = next(t for t in data['targets'] if t['name'] == name) if fragment.get('inputs') != {image: target['inputs']}: raise ValueError('Image artifact does not match the build plan') release['images'].update(fragment['images']) release['inputs'].update(fragment['inputs']) validate_release(release, sha) # Only a complete set of successful image jobs can publish the release tags. docker_config = tempfile.mkdtemp(prefix='homelab-registry-') env = {**os.environ, 'DOCKER_CONFIG': docker_config} try: subprocess.run( # noqa: S603, S607 [ shutil.which('docker') or '/usr/bin/docker', 'login', 'gcr.forust.xyz', '-u', os.environ['REGISTRY_USERNAME'], '--password-stdin', ], input=os.environ['REGISTRY_PASSWORD'], text=True, check=True, env=env, ) for image, digest in release['images'].items(): command( 'docker', 'buildx', 'imagetools', 'create', '--prefer-index=false', '--tag', f'{image}:sha-{sha}', f'{image}@{digest}', env=env, timeout=90, ) output.write_text(json.dumps(release, indent=2) + '\n') finally: shutil.rmtree(docker_config) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('action', choices=('prepare', 'image', 'finalize', 'gate', 'render', 'check-summary')) parser.add_argument('--output', type=Path, default=Path('release.json')) parser.add_argument('--ref', default='main') parser.add_argument('--event-sha', default='') parser.add_argument('--image', choices=IMAGES) parser.add_argument('--plan', type=Path, default=Path('build-plan.json')) parser.add_argument('--fragments', type=Path, default=Path('artifacts')) args = parser.parse_args() if args.action == 'check-summary': check_summary() elif args.action == 'render': render(sys.stdin, sys.stdout) elif args.action == 'gate': gate(args.output, args.ref, args.event_sha) elif args.action == 'prepare': prepare_images(args.output) elif args.action == 'image': if not args.image: parser.error('--image is required') build(args.output, args.image, args.plan) else: finalize_images(args.output, args.fragments, args.plan) if __name__ == '__main__': main()