ci: add per-image jobs to homelab CI

.gitea/workflows/ci.yaml

@@ -375,48 +375,142 @@ jobs:
           elif [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
             printf '## %s\n\n- Result: **%s**\n- Failed step: %s\n' "$SUMMARY_CHECK" "$SUMMARY_RESULT" "$SUMMARY_FAILED_STEP" >>"$GITHUB_STEP_SUMMARY" || true
           fi
-  build:
-    needs:
-      - compose
-      - workflows
-      - shell
-      - formatting
-      - python
-      - yaml
-      - dockerfiles
-      - kubernetes
+  image-plan:
+    needs: [compose, workflows, shell, formatting, python, yaml, dockerfiles, kubernetes]
     if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
     runs-on: homelab
-    timeout-minutes: 60
+    timeout-minutes: 10
+    outputs:
+      matrix: ${{ steps.plan.outputs.matrix }}
     steps:
       - name: Checkout repository
+        id: source
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
         with:
           fetch-depth: 0
-        id: source
-      - name: Build changed images and write release
+      - name: Detect build inputs against successful CI
+        id: plan
         env:
           GITEA_TOKEN: ${{ github.token }}
-          REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
-          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
-        run: python3 .gitea/workflows/release.py build
-        id: check
-      - name: Store commit release
+        run: python3 .gitea/workflows/release.py prepare --output build-plan.json
+      - name: Store the image plan
+        id: artifact
         uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf
         with:
-          name: release-${{ github.sha }}
-          path: release.json
+          name: build-plan
+          path: build-plan.json
           if-no-files-found: error
           retention-days: 30
+
+      - name: Write the plan result
+        if: always()
+        env:
+          SUMMARY_CHECK: Image plan
+          SUMMARY_RESULT: ${{ job.status }}
+          SUMMARY_FAILED_STEP: >-
+            ${{ steps.plan.conclusion == 'failure' && 'Build input detection' ||
+                steps.artifact.conclusion == 'failure' && 'Plan upload' ||
+                steps.source.conclusion == 'failure' && 'Source checkout' || '' }}
+        shell: bash
+        run: |
+          if [ -f .gitea/workflows/release.py ]; then
+            python3 .gitea/workflows/release.py check-summary
+          elif [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
+            printf '## Image plan\n\nResult: %s\n' "$SUMMARY_RESULT" >>"$GITHUB_STEP_SUMMARY" || true
+          fi
+
+  images:
+    name: Image (${{ matrix.name }})
+    needs: [image-plan]
+    if: needs.image-plan.result == 'success'
+    runs-on: homelab
+    timeout-minutes: 60
+    strategy:
+      max-parallel: 1
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.image-plan.outputs.matrix || '{"include":[{"name":"inactive"}]}') }}
+    steps:
+      - name: Checkout repository
+        id: source
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+      - name: Download the checked image plan
+        id: inputs
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
+        with:
+          name: build-plan
+      - name: Build or reuse this image
+        id: check
+        env:
+          IMAGE_NAME: ${{ matrix.name }}
+          REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
+          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
+        run: python3 .gitea/workflows/release.py image --image "$IMAGE_NAME" --output image.json
+      - name: Store the image result
         id: artifact
+        uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf
+        with:
+          name: image-${{ matrix.name }}
+          path: image.json
+          if-no-files-found: error
+          retention-days: 30
       - name: Write the job result
         if: always()
         env:
-          SUMMARY_CHECK: Image build and release artifact
+          SUMMARY_CHECK: Image (${{ matrix.name }})
           SUMMARY_RESULT: ${{ job.status }}
-          SUMMARY_FAILED_STEP:
-            ${{ steps.check.conclusion == 'failure' && 'Check or image build' || steps.artifact.conclusion == 'failure'
-            && 'Release artifact upload' || steps.source.conclusion == 'failure' && 'Source checkout' || '' }}
+          SUMMARY_FAILED_STEP: >-
+            ${{ steps.check.conclusion == 'failure' && 'Build or tag images' ||
+                steps.artifact.conclusion == 'failure' && 'Artifact upload' ||
+                steps.inputs.conclusion == 'failure' && 'Artifact download' ||
+                steps.source.conclusion == 'failure' && 'Source checkout' || '' }}
+        shell: bash
+        run: |
+          if [ -f .gitea/workflows/release.py ]; then
+            python3 .gitea/workflows/release.py check-summary
+          elif [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
+            printf '## %s\n\n- Result: **%s**\n- Failed step: %s\n' "$SUMMARY_CHECK" "$SUMMARY_RESULT" "$SUMMARY_FAILED_STEP" >>"$GITHUB_STEP_SUMMARY" || true
+          fi
+
+  # Retain the build job name required by the immutable release deployment gate.
+  build:
+    needs: [image-plan, images]
+    runs-on: homelab
+    timeout-minutes: 15
+    steps:
+      - name: Checkout repository
+        id: source
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+      - name: Download all image results
+        id: inputs
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
+        with:
+          path: artifacts
+      - name: Pin SHA tags and write the complete release
+        id: check
+        env:
+          REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
+          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
+        run: >-
+          python3 .gitea/workflows/release.py finalize
+          --plan artifacts/build-plan/build-plan.json
+      - name: Store commit release
+        id: artifact
+        uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf
+        with:
+          name: release-${{ github.sha }}
+          path: release.json
+          if-no-files-found: error
+          retention-days: 30
+      - name: Write the job result
+        if: always()
+        env:
+          SUMMARY_CHECK: Image release and SHA tags
+          SUMMARY_RESULT: ${{ job.status }}
+          SUMMARY_FAILED_STEP: >-
+            ${{ steps.check.conclusion == 'failure' && 'Build or tag images' ||
+                steps.artifact.conclusion == 'failure' && 'Artifact upload' ||
+                steps.inputs.conclusion == 'failure' && 'Artifact download' ||
+                steps.source.conclusion == 'failure' && 'Source checkout' || '' }}
         shell: bash
         run: |
           if [ -f .gitea/workflows/release.py ]; then

.gitea/workflows/release.py

@@ -26,9 +26,6 @@ IMAGES = {
     'xdfnx-homepage': ('homepages', 'homepages/Dockerfile.xdfnx'),
 }
 
-# These images are released by the EDU application repository.
-EXTERNAL_IMAGES = {'gcr.forust.xyz/forust/session-keeper', 'gcr.forust.xyz/forust/webinar-checker'}
-
 
 def command(*args, **kwargs):
     """Arguments are passed directly to the executable, never to a shell."""
@@ -163,11 +160,10 @@ def gate(output, requested_ref, event_sha):
     print(f'CI gate accepted {sha}')
 
 
-def build_images(output, report):
+def prepare_images(output):
     sha = command('git', 'rev-parse', 'HEAD')
     if sha != os.environ['GITHUB_SHA'] or not SHA.fullmatch(sha):
         raise ValueError('Build checkout does not match GITHUB_SHA')
-    report['phase'] = 'Find a successful CI release'
     api = Gitea()
     previous = None
     for run in sorted(itertools.islice(api.successful_runs(), 50), key=lambda item: item['id'], reverse=True):
@@ -179,6 +175,55 @@ def build_images(output, report):
         except ValueError:
             # Expired artifacts only cost a rebuild; mutable tags are never a fallback.
             continue
+    targets = []
+    for name, (context, dockerfile) in IMAGES.items():
+        image = f'gcr.forust.xyz/forust/{name}'
+        inputs = fingerprint(context, dockerfile)
+        old_digest = (previous or {}).get('images', {}).get(image)
+        targets.append(
+            {
+                'name': name,
+                'image': image,
+                'context': context,
+                'dockerfile': dockerfile,
+                'inputs': inputs,
+                'reuse_digest': old_digest if (previous or {}).get('inputs', {}).get(image) == inputs else None,
+            }
+        )
+    output.write_text(json.dumps({'sha': sha, 'targets': targets}, indent=2) + '\n')
+    if os.environ.get('GITHUB_OUTPUT'):
+        with Path(os.environ['GITHUB_OUTPUT']).open('a') as stream:
+            stream.write('matrix=' + json.dumps({'include': targets}, separators=(',', ':')) + '\n')
+    print(f'Prepared {len(targets)} image jobs; {sum(t["reuse_digest"] is None for t in targets)} require builds')
+
+
+def checked_plan(path):
+    data = json.loads(path.read_text())
+    sha = command('git', 'rev-parse', 'HEAD')
+    if data.get('sha') != sha or sha != os.environ['GITHUB_SHA'] or not SHA.fullmatch(sha):
+        raise ValueError('Image plan does not match the checked source commit')
+    targets = data.get('targets', [])
+    if sorted(t['name'] for t in targets) != sorted(IMAGES):
+        raise ValueError('Image plan must contain each owned image once')
+    for target in targets:
+        name = target['name']
+        context, dockerfile = IMAGES[name]
+        if (target['context'], target['dockerfile'], target['image']) != (
+            context,
+            dockerfile,
+            f'gcr.forust.xyz/forust/{name}',
+        ) or target['inputs'] != fingerprint(context, dockerfile):
+            raise ValueError('Image plan has invalid build inputs')
+        if target['reuse_digest'] is not None and not DIGEST.fullmatch(target['reuse_digest']):
+            raise ValueError('Image plan has an invalid reuse digest')
+    return data
+
+
+def build_images(output, report, name, plan):
+    data = checked_plan(plan)
+    sha = data['sha']
+    target = next(t for t in data['targets'] if t['name'] == name)
+    context, dockerfile = IMAGES[name]
     docker_config = tempfile.mkdtemp(prefix='homelab-registry-')
     builder_config = Path.home() / '.cache/homelab-ci/buildx'
     builder_config.mkdir(parents=True, exist_ok=True)
@@ -235,64 +280,63 @@ def build_images(output, report):
             signature.write_text(image + '\n')
         release = {'version': 1, 'sha': sha, 'images': {}, 'inputs': {}}
         report['images'] = release['images']
-        for name, (context, dockerfile) in IMAGES.items():
-            report['phase'] = f'Build or reuse {name}'
-            report['current'] = name
-            image = f'gcr.forust.xyz/forust/{name}'
-            inputs = fingerprint(context, dockerfile)
-            old_digest = (previous or {}).get('images', {}).get(image)
-            exists = False
-            if old_digest and previous['inputs'].get(image) == inputs:
-                exists = (
-                    subprocess.run(  # noqa: S603, S607
-                        [
-                            shutil.which('docker') or '/usr/bin/docker',
-                            'buildx',
-                            'imagetools',
-                            'inspect',
-                            f'{image}@{old_digest}',
-                        ],
-                        capture_output=True,
-                        env=env,
-                        timeout=60,
-                    ).returncode
-                    == 0
-                )
-            if exists:
-                print(f'Reuse {name}: inputs unchanged')
-                digest = old_digest
-                report['reused'].append(name)
-            else:
-                print(f'Build {name}', flush=True)
-                metadata = Path(docker_config) / 'metadata.json'
-                command(
-                    'docker',
-                    'buildx',
-                    'build',
-                    '--builder',
-                    builder,
-                    '--push',
-                    '--platform',
-                    'linux/amd64',
-                    '--provenance=false',
-                    '--cache-from',
-                    f'type=registry,ref={image}:buildcache',
-                    '--cache-to',
-                    f'type=registry,ref={image}:buildcache,mode=max',
-                    '--tag',
-                    f'{image}:sha-{sha}',
-                    '--metadata-file',
-                    str(metadata),
-                    '--file',
-                    dockerfile,
-                    context,
+        report['phase'] = f'Build or reuse {name}'
+        report['current'] = name
+        image = f'gcr.forust.xyz/forust/{name}'
+        inputs = target['inputs']
+        old_digest = target['reuse_digest']
+        exists = False
+        if old_digest:
+            exists = (
+                subprocess.run(  # noqa: S603, S607
+                    [
+                        shutil.which('docker') or '/usr/bin/docker',
+                        'buildx',
+                        'imagetools',
+                        'inspect',
+                        f'{image}@{old_digest}',
+                    ],
+                    capture_output=True,
                     env=env,
-                )
-                digest = json.loads(metadata.read_text())['containerimage.digest']
-                report['built'].append(name)
-            release['images'][image] = digest
-            release['inputs'][image] = inputs
-        validate_release(release, sha)
+                    timeout=60,
+                ).returncode
+                == 0
+            )
+        if exists:
+            print(f'Reuse {name}: inputs unchanged')
+            digest = old_digest
+            report['reused'].append(name)
+        else:
+            print(f'Build {name}', flush=True)
+            metadata = Path(docker_config) / 'metadata.json'
+            command(
+                'docker',
+                'buildx',
+                'build',
+                '--builder',
+                builder,
+                '--platform',
+                'linux/amd64',
+                '--provenance=false',
+                '--cache-from',
+                f'type=registry,ref={image}:buildcache',
+                '--cache-to',
+                f'type=registry,ref={image}:buildcache,mode=max',
+                '--output',
+                f'type=image,name={image},push-by-digest=true,name-canonical=true,push=true',
+                '--metadata-file',
+                str(metadata),
+                '--file',
+                dockerfile,
+                context,
+                env=env,
+            )
+            digest = json.loads(metadata.read_text())['containerimage.digest']
+            report['built'].append(name)
+        release['images'][image] = digest
+        release['inputs'][image] = inputs
+        if not DIGEST.fullmatch(digest):
+            raise ValueError('Image job returned an invalid digest')
         output.write_text(json.dumps(release, indent=2) + '\n')
         report['current'] = None
         report['phase'] = 'Release file saved'
@@ -343,11 +387,11 @@ def check_summary():
     write_summary(lines)
 
 
-def build(output):
+def build(output, name, plan):
     report = {'phase': 'Check the source commit', 'current': None, 'built': [], 'reused': [], 'images': {}}
     result = 'failure'
     try:
-        build_images(output, report)
+        build_images(output, report, name, plan)
         result = 'success'
     finally:
         lines = [
@@ -382,9 +426,6 @@ def render(stream, destination):
         match = image_line.fullmatch(line.rstrip('\n'))
         if match:
             prefix, quote, image, tail = match.groups()
-            if image in EXTERNAL_IMAGES and f'{image}@sha256:' in line:
-                rendered.append(line)
-                continue
             if image not in release['images']:
                 raise ValueError(f'Owned image missing from checked release: {image}')
             line = f'{prefix}{quote}{image}@{release["images"][image]}{quote}{tail}\n'
@@ -394,12 +435,66 @@ def render(stream, destination):
     destination.writelines(rendered)
 
 
+def finalize_images(output, fragments, plan):
+    data = checked_plan(plan)
+    sha = data['sha']
+    release = {'version': 1, 'sha': sha, 'images': {}, 'inputs': {}}
+    for name in IMAGES:
+        fragment = json.loads((fragments / f'image-{name}' / 'image.json').read_text())
+        image = f'gcr.forust.xyz/forust/{name}'
+        if fragment.get('sha') != sha or fragment.get('version') != 1 or set(fragment.get('images', {})) != {image}:
+            raise ValueError('Image job artifact is missing or belongs to another commit')
+        target = next(t for t in data['targets'] if t['name'] == name)
+        if fragment.get('inputs') != {image: target['inputs']}:
+            raise ValueError('Image artifact does not match the build plan')
+        release['images'].update(fragment['images'])
+        release['inputs'].update(fragment['inputs'])
+    validate_release(release, sha)
+    # Only a complete set of successful image jobs can publish the release tags.
+    docker_config = tempfile.mkdtemp(prefix='homelab-registry-')
+    env = {**os.environ, 'DOCKER_CONFIG': docker_config}
+    try:
+        subprocess.run(  # noqa: S603, S607
+            [
+                shutil.which('docker') or '/usr/bin/docker',
+                'login',
+                'gcr.forust.xyz',
+                '-u',
+                os.environ['REGISTRY_USERNAME'],
+                '--password-stdin',
+            ],
+            input=os.environ['REGISTRY_PASSWORD'],
+            text=True,
+            check=True,
+            env=env,
+        )
+        for image, digest in release['images'].items():
+            command(
+                'docker',
+                'buildx',
+                'imagetools',
+                'create',
+                '--prefer-index=false',
+                '--tag',
+                f'{image}:sha-{sha}',
+                f'{image}@{digest}',
+                env=env,
+                timeout=90,
+            )
+        output.write_text(json.dumps(release, indent=2) + '\n')
+    finally:
+        shutil.rmtree(docker_config)
+
+
 def main():
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument('action', choices=('build', 'gate', 'render', 'check-summary'))
+    parser.add_argument('action', choices=('prepare', 'image', 'finalize', 'gate', 'render', 'check-summary'))
     parser.add_argument('--output', type=Path, default=Path('release.json'))
     parser.add_argument('--ref', default='main')
     parser.add_argument('--event-sha', default='')
+    parser.add_argument('--image', choices=IMAGES)
+    parser.add_argument('--plan', type=Path, default=Path('build-plan.json'))
+    parser.add_argument('--fragments', type=Path, default=Path('artifacts'))
     args = parser.parse_args()
     if args.action == 'check-summary':
         check_summary()
@@ -407,8 +502,14 @@ def main():
         render(sys.stdin, sys.stdout)
     elif args.action == 'gate':
         gate(args.output, args.ref, args.event_sha)
+    elif args.action == 'prepare':
+        prepare_images(args.output)
+    elif args.action == 'image':
+        if not args.image:
+            parser.error('--image is required')
+        build(args.output, args.image, args.plan)
     else:
-        build(args.output)
+        finalize_images(args.output, args.fragments, args.plan)
 
 
 if __name__ == '__main__':