#!/usr/bin/env python3 """CI release artifacts and the SHA-specific Gitea deployment gate (stdlib only).""" import argparse import hashlib import io import itertools import json import os import re import shutil import subprocess import sys import tempfile import urllib.error import urllib.parse import urllib.request import zipfile from pathlib import Path SHA = re.compile(r'[0-9a-f]{40}') DIGEST = re.compile(r'sha256:[0-9a-f]{64}') IMAGES = { 'error-pages': ('errorpages', 'errorpages/Dockerfile'), 'forust-homepage': ('homepages', 'homepages/Dockerfile.forust'), 'xdfnx-homepage': ('homepages', 'homepages/Dockerfile.xdfnx'), } # These images are released by the EDU application repository. EXTERNAL_IMAGES = {'gcr.forust.xyz/forust/session-keeper', 'gcr.forust.xyz/forust/webinar-checker'} def command(*args, **kwargs): """Arguments are passed directly to the executable, never to a shell.""" return subprocess.check_output(args, text=True, **kwargs).strip() # noqa: S603, S607 def validate_release(data, sha=None): if data.get('version') != 1 or not SHA.fullmatch(data.get('sha', '')): raise ValueError('Invalid release version or SHA') if sha is not None and data['sha'] != sha: raise ValueError('Release SHA does not match the checked CI commit') expected = {f'gcr.forust.xyz/forust/{name}' for name in IMAGES} if set(data.get('images', {})) != expected: raise ValueError('Release must contain all owned images') if not all(DIGEST.fullmatch(value) for value in data['images'].values()): raise ValueError('Release has an invalid image digest') if set(data.get('inputs', {})) != expected or not all( re.fullmatch(r'[0-9a-f]{64}', value) for value in data['inputs'].values() ): raise ValueError('Release has invalid build input fingerprints') return data class NoRedirect(urllib.request.HTTPRedirectHandler): def redirect_request(self, _req, _fp, _code, _msg, _headers, _newurl): return None class Gitea: def __init__(self): self.origin = os.environ['GITHUB_SERVER_URL'].rstrip('/') if urllib.parse.urlsplit(self.origin).scheme != 'https': raise ValueError('Gitea API must use HTTPS') self.repository = os.environ['GITHUB_REPOSITORY'] if not re.fullmatch(r'[\w.-]+/[\w.-]+', self.repository): raise ValueError('Invalid Gitea repository') self.token = os.environ['GITEA_TOKEN'] self.base = f'{self.origin}/api/v1/repos/{self.repository}' def request(self, url, *, archive=False): if not url.startswith(self.base + '/'): raise ValueError('Refusing to send the Actions token to another origin') req = urllib.request.Request(url, headers={'Authorization': f'token {self.token}'}) # noqa: S310 -- HTTPS origin validated above opener = urllib.request.build_opener(NoRedirect()) try: response = opener.open(req, timeout=30) # noqa: S310 except urllib.error.HTTPError as error: if not archive or error.code not in (301, 302, 303, 307, 308): raise RuntimeError(f'Gitea API returned HTTP {error.code}') from None target = urllib.parse.urljoin(url, error.headers['Location']) if urllib.parse.urlsplit(target).scheme != 'https': raise ValueError('Artifact redirect must use HTTPS') from None # Signed storage redirects must never receive the Gitea token. response = urllib.request.urlopen(target, timeout=30) # noqa: S310 with response: payload = response.read(8 * 1024 * 1024 + 1) if len(payload) > 8 * 1024 * 1024: raise ValueError('Gitea response exceeds 8 MiB') return payload if archive else json.loads(payload) def pages(self, path, key, **params): for page in range(1, 101): query = urllib.parse.urlencode({**params, 'page': page, 'limit': 50}) data = self.request(f'{self.base}/{path}?{query}') entries = data[key] yield from entries if len(entries) < 50: return raise RuntimeError('Gitea pagination limit exceeded') def successful_runs(self, sha=None): params = {'branch': 'main', 'status': 'success', 'exclude_pull_requests': 'true'} if sha: params['head_sha'] = sha for run in self.pages('actions/workflows/ci.yaml/runs', 'workflow_runs', **params): if ( run.get('status') == 'completed' and run.get('conclusion') == 'success' and run.get('head_branch') == 'main' and run.get('event') in ('push', 'workflow_dispatch') and (run.get('repository') or {}).get('full_name') == self.repository and (run.get('head_repository') or run.get('repository') or {}).get('full_name') == self.repository and (sha is None or run.get('head_sha') == sha) ): yield run def release(self, run): sha = run['head_sha'] jobs = list(self.pages(f'actions/runs/{run["id"]}/jobs', 'jobs')) # A green workflow with a skipped build must not authorize a deploy. if not any(job.get('name') == 'build' and job.get('conclusion') == 'success' for job in jobs): raise ValueError('CI build job did not succeed') artifacts = self.request(f'{self.base}/actions/runs/{run["id"]}/artifacts')['artifacts'] matching = [a for a in artifacts if a['name'] == f'release-{sha}' and not a.get('expired')] if len(matching) != 1: raise ValueError('CI release artifact is missing, expired or ambiguous; rerun CI') blob = self.request(f'{self.base}/actions/artifacts/{matching[0]["id"]}/zip', archive=True) with zipfile.ZipFile(io.BytesIO(blob)) as archive: files = [entry for entry in archive.infolist() if not entry.is_dir()] if len(files) != 1 or files[0].filename != 'release.json' or files[0].file_size > 256 * 1024: raise ValueError('Unexpected release archive contents') return validate_release(json.loads(archive.read(files[0])), sha) def fingerprint(context, dockerfile): tree = command('git', 'ls-tree', '-r', 'HEAD', '--', context, dockerfile, '.gitea/workflows/release.py') return hashlib.sha256(tree.encode()).hexdigest() def gate(output, requested_ref, event_sha): command('git', 'fetch', '--quiet', 'origin', 'main') if event_sha: if not SHA.fullmatch(event_sha): raise ValueError('Invalid workflow_run SHA') sha = event_sha else: if requested_ref == 'main': requested_ref = 'origin/main' sha = command('git', 'rev-parse', '--verify', '--end-of-options', f'{requested_ref}^{{commit}}') if not SHA.fullmatch(sha): raise ValueError('Invalid deploy SHA') command('git', 'merge-base', '--is-ancestor', sha, 'origin/main') api = Gitea() runs = list(api.successful_runs(sha)) if not runs: raise ValueError(f'No successful main CI for {sha}; run CI before deploying') release = api.release(max(runs, key=lambda run: run['id'])) output.write_text(json.dumps(release, indent=2) + '\n') if os.environ.get('GITHUB_OUTPUT'): with Path(os.environ['GITHUB_OUTPUT']).open('a') as stream: stream.write(f'sha={sha}\n') print(f'CI gate accepted {sha}') def build(output): sha = command('git', 'rev-parse', 'HEAD') if sha != os.environ['GITHUB_SHA'] or not SHA.fullmatch(sha): raise ValueError('Build checkout does not match GITHUB_SHA') api = Gitea() previous = None for run in sorted(itertools.islice(api.successful_runs(), 50), key=lambda item: item['id'], reverse=True): if str(run['id']) == os.environ.get('GITHUB_RUN_ID'): continue try: previous = api.release(run) break except ValueError: # Expired artifacts only cost a rebuild; mutable tags are never a fallback. continue docker_config = tempfile.mkdtemp(prefix='homelab-registry-') builder_config = Path.home() / '.cache/homelab-ci/buildx' builder_config.mkdir(parents=True, exist_ok=True) env = {**os.environ, 'DOCKER_CONFIG': docker_config, 'BUILDX_CONFIG': str(builder_config)} try: subprocess.run( # noqa: S603, S607 [ shutil.which('docker') or '/usr/bin/docker', 'login', 'gcr.forust.xyz', '-u', os.environ['REGISTRY_USERNAME'], '--password-stdin', ], input=os.environ['REGISTRY_PASSWORD'], text=True, check=True, env=env, ) builder = 'homelab-ci' versions = dict( re.findall(r'^([A-Z_]+)="([^"\n]+)"$', Path('.gitea/workflows/tool-versions.env').read_text(), re.MULTILINE) ) image = versions['BUILDKIT_IMAGE'] signature = builder_config / 'homelab-ci-image' exists = ( subprocess.run( # noqa: S603 [shutil.which('docker') or '/usr/bin/docker', 'buildx', 'inspect', builder], capture_output=True, env=env, ).returncode == 0 ) if exists and (not signature.exists() or signature.read_text().strip() != image): command('docker', 'buildx', 'rm', '--keep-state', builder, env=env) exists = False if not exists: command( 'docker', 'buildx', 'create', '--name', builder, '--driver', 'docker-container', '--driver-opt', f'image={image}', '--buildkitd-config', '.gitea/runner/buildkitd.toml', env=env, ) signature.write_text(image + '\n') release = {'version': 1, 'sha': sha, 'images': {}, 'inputs': {}} built = [] reused = [] for name, (context, dockerfile) in IMAGES.items(): image = f'gcr.forust.xyz/forust/{name}' inputs = fingerprint(context, dockerfile) old_digest = (previous or {}).get('images', {}).get(image) exists = False if old_digest and previous['inputs'].get(image) == inputs: exists = ( subprocess.run( # noqa: S603, S607 [ shutil.which('docker') or '/usr/bin/docker', 'buildx', 'imagetools', 'inspect', f'{image}@{old_digest}', ], capture_output=True, env=env, timeout=60, ).returncode == 0 ) if exists: print(f'Reuse {name}: inputs unchanged') digest = old_digest reused.append((name, image, digest)) else: print(f'Build {name}', flush=True) built.append((name, image)) metadata = Path(docker_config) / 'metadata.json' command( 'docker', 'buildx', 'build', '--builder', builder, '--push', '--platform', 'linux/amd64', '--provenance=false', '--cache-from', f'type=registry,ref={image}:buildcache', '--cache-to', f'type=registry,ref={image}:buildcache,mode=max', '--tag', f'{image}:sha-{sha}', '--metadata-file', str(metadata), '--file', dockerfile, context, env=env, ) digest = json.loads(metadata.read_text())['containerimage.digest'] release['images'][image] = digest release['inputs'][image] = inputs validate_release(release, sha) output.write_text(json.dumps(release, indent=2) + '\n') summary = os.environ.get('GITHUB_STEP_SUMMARY') if summary: lines = [f'## Image release for `{sha}`', '', '### Built'] lines.extend(f'- `{name}` — `{image}`' for name, image in built) if not built: lines.append('- None') lines.extend(['', '### Reused from successful CI']) lines.extend(f'- `{name}` — `{image}@{digest}`' for name, image, digest in reused) if not reused: lines.append('- None') lines.extend(['', '### Release digests']) lines.extend( f'- `{name}` — `{image}@{release["images"][image]}`' for name in IMAGES for image in [f'gcr.forust.xyz/forust/{name}'] ) Path(summary).write_text('\n'.join(lines) + '\n') finally: # Cleanup errors must neither leak credentials nor mask the original build error. try: subprocess.run( # noqa: S603 [ shutil.which('docker') or '/usr/bin/docker', 'buildx', 'prune', '--builder', 'homelab-ci', '--force', '--max-used-space', '1gb', ], env=env, timeout=60, ) except (OSError, subprocess.TimeoutExpired): print('CI builder cache cleanup deferred', flush=True) finally: shutil.rmtree(docker_config) def render(stream, destination): release = validate_release(json.loads(Path(os.environ['RELEASE_FILE']).read_text()), os.environ['DEPLOY_SHA']) image_line = re.compile( r"^(\s*(?:-\s*)?image:\s*)(['\"]?)(gcr\.forust\.xyz/forust/[\w.-]+)(?::[\w.-]+|@sha256:[0-9a-f]{64})\2(\s*(?:#.*)?)$" ) rendered = [] for line in stream: match = image_line.fullmatch(line.rstrip('\n')) if match: prefix, quote, image, tail = match.groups() if image in EXTERNAL_IMAGES and f'{image}@sha256:' in line: rendered.append(line) continue if image not in release['images']: raise ValueError(f'Owned image missing from checked release: {image}') line = f'{prefix}{quote}{image}@{release["images"][image]}{quote}{tail}\n' elif re.match(r'\s*(?:-\s*)?image:', line) and 'gcr.forust.xyz/forust/' in line: raise ValueError('Unsupported owned image syntax; refusing to apply a mutable tag') rendered.append(line) destination.writelines(rendered) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('action', choices=('build', 'gate', 'render')) parser.add_argument('--output', type=Path, default=Path('release.json')) parser.add_argument('--ref', default='main') parser.add_argument('--event-sha', default='') args = parser.parse_args() if args.action == 'render': render(sys.stdin, sys.stdout) elif args.action == 'gate': gate(args.output, args.ref, args.event_sha) else: build(args.output) if __name__ == '__main__': main()