Show failure details in CI and deploy summaries

.gitea/workflows/deploy-controller.py

@@ -212,14 +212,17 @@ def execute(run_id):
                 raise ValueError(f'Interrupted deploy {other.name}; run recover first')
         status['state'] = 'running'
         atomic_json(directory / 'status.json', status)
+        phase = 'plan'
         try:
             plan = make_plan(directory)
             print(
                 json.dumps({'selected': plan['selected'], 'helm': plan['helm'], 'manual_removals': plan['removed']}),
                 flush=True,
             )
+            phase = 'doctor'
             if not stage(directory, 'doctor', 600):
                 raise RuntimeError('Preflight failed')
+            phase = 'validate'
             if not stage(directory, 'validate', 1200):
                 raise RuntimeError('Validation failed')
             if json.loads((directory / 'request.json').read_text())['mode'] == 'plan':
@@ -228,6 +231,7 @@ def execute(run_id):
                 atomic_json(directory / 'status.json', status)
                 return
             # Budget includes both rollout checks and rollback waves, plus API overhead.
+            phase = 'Recovery budget'
             count = int(
                 command(
                     'bash',
@@ -239,14 +243,24 @@ def execute(run_id):
             verify_budget = max(600, 2 * math.ceil(count / 4) * 300 + 120)
             if verify_budget > 7200:
                 raise ValueError('More than two hours of recovery required; split this deploy')
+            phase = 'apply-k8s'
             k8s_ok = stage(directory, 'apply-k8s', 2700)
+            phase = 'apply-compose'
             compose_ok = stage(directory, 'apply-compose', 1800) if k8s_ok else False
+            phase = 'verify-k8s'
             verify_ok = stage(directory, 'verify-k8s', verify_budget)
+            phase = 'smoke'
             smoke_ok = stage(directory, 'smoke', 600)
             if not all((k8s_ok, compose_ok, verify_ok, smoke_ok)):
                 raise RuntimeError('Deploy failed; inspect stage logs and recovery report')
+            phase = 'Save the successful baseline'
             finish_success(directory, plan)
         except Exception as error:
+            status = json.loads((directory / 'status.json').read_text())
+            status['failure_stage'] = next(
+                (name for name, result in status['stages'].items() if result.get('result') == 'failure'), phase
+            )
+            atomic_json(directory / 'status.json', status)
             with (directory / 'controller.log').open('a') as stream:
                 stream.write(f'{error}\n')
             recover(directory)
@@ -305,6 +319,52 @@ def summary(run_id):
         f'- Mode: `{request["mode"]}`',
         f'- Refresh third-party images: `{request["refresh_images"]}`',
     ]
+    status = json.loads((directory / 'status.json').read_text())
+    if status.get('failure_stage'):
+        lines.append(f'- Failed stage: **{status["failure_stage"]}**')
+    lines.extend(
+        [
+            '',
+            f'- Observed run state: **{status["state"]}**',
+            '',
+            '### Stage results',
+            '| Stage | Result | Exit code |',
+            '| --- | --- | --- |',
+        ]
+    )
+    for name in ('doctor', 'validate', 'apply-k8s', 'apply-compose', 'verify-k8s', 'smoke'):
+        stage_result = status['stages'].get(name, {})
+        lines.append(f'| {name} | {stage_result.get("result", "not started")} | {stage_result.get("exit_code", "—")} |')
+    lines.extend(['', '### Apply and Helm recovery results'])
+    events_file = directory / 'apply-events.jsonl'
+    events = []
+    if events_file.exists():
+        for line in events_file.read_text().splitlines():
+            try:
+                events.append(json.loads(line))
+            except json.JSONDecodeError:
+                lines.append('- An operation record is incomplete. Check the stage log.')
+    latest = {(event['action'], event['target']): event['result'] for event in events}
+    lines.extend(f'- `{action}` `{target}`: **{result}**' for (action, target), result in latest.items())
+    if not latest:
+        lines.append('- No apply results were recorded.')
+    lines.append('- A completed apply does not confirm health. See verification and smoke results.')
+    lines.extend(['', '### Kubernetes recovery'])
+    pointer = directory / 'snapshot/current'
+    failed = Path(pointer.read_text().strip()) / 'failed-workloads' if pointer.exists() else None
+    if failed and failed.exists():
+        contents = failed.read_text()
+        counts = dict(re.findall(r'^(ROLLED_BACK|UNRECOVERED)=([0-9]+)$', contents, re.MULTILINE))
+        if not contents.strip():
+            lines.append('- No failed workloads were recorded. See the verification result above.')
+        elif counts:
+            lines.append(f'- Workloads restored: **{counts.get("ROLLED_BACK", "unknown")}**')
+            lines.append(f'- Workloads that need manual recovery: **{counts.get("UNRECOVERED", "unknown")}**')
+        else:
+            lines.append('- Rollback has no recorded result yet. Check the verification log.')
+    else:
+        lines.append('- No workload rollback was recorded. This does not confirm health.')
+    lines.append('- Compose requires manual recovery. Use the saved command in the apply log.')
     if not plan_file.exists():
         lines.extend(['', 'Plan was not created. Check the controller log.'])
         print('\n'.join(lines))