test: isolate CI summary and output files

tests/ci_test_case.py

@@ -0,0 +1,22 @@
+"""Keep unit-test workflow commands out of the real CI job files."""
+
+import os
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+
+class IsolatedCITestCase(unittest.TestCase):
+    def setUp(self):
+        super().setUp()
+        directory = tempfile.TemporaryDirectory(prefix='homelab-test-ci-')
+        self.addCleanup(directory.cleanup)
+        paths = {}
+        for variable in ('GITHUB_STEP_SUMMARY', 'GITHUB_OUTPUT'):
+            path = Path(directory.name) / variable
+            path.touch()
+            paths[variable] = str(path)
+        environment = patch.dict(os.environ, paths)
+        environment.start()
+        self.addCleanup(environment.stop)

tests/test_ci_output_isolation.py

@@ -0,0 +1,37 @@
+"""Run the real unit tests with external CI files and detect leaked writes."""
+
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+from ci_test_case import IsolatedCITestCase
+
+
+class CIOutputIsolationTests(IsolatedCITestCase):
+    def test_unit_suite_preserves_external_ci_files(self):
+        tests = Path(__file__).resolve().parent
+        modules = sorted(p.stem for p in tests.glob('test_*.py') if p.name != Path(__file__).name)
+        with tempfile.TemporaryDirectory() as scratch:
+            environment = os.environ.copy()
+            environment['PYTHONPATH'] = str(tests) + os.pathsep + environment.get('PYTHONPATH', '')
+            expected = {}
+            for variable in ('GITHUB_STEP_SUMMARY', 'GITHUB_OUTPUT'):
+                path = Path(scratch) / variable
+                content = f'external {variable}\n'
+                path.write_text(content)
+                environment[variable] = str(path)
+                expected[path] = content
+            result = subprocess.run(  # noqa: S603 -- Run local test modules with the current Python interpreter.
+                [sys.executable, '-m', 'unittest', *modules, '-q'],
+                cwd=tests.parent,
+                env=environment,
+                capture_output=True,
+                text=True,
+                check=False,
+                timeout=60,
+            )
+            self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+            for path, content in expected.items():
+                self.assertEqual(path.read_text(), content, f'Unit tests wrote to external {path.name}')

tests/test_cicd.py

@@ -9,6 +9,8 @@ import unittest
 from pathlib import Path
 from unittest.mock import patch
 
+from ci_test_case import IsolatedCITestCase
+
 ROOT = Path(__file__).resolve().parents[1]
 
 
@@ -34,7 +36,7 @@ def release(sha='a' * 40):
     }
 
 
-class ReleaseGateTests(unittest.TestCase):
+class ReleaseGateTests(IsolatedCITestCase):
     def test_release_rejects_wrong_sha_missing_images_and_mutable_tags(self):
         for mutation in ('sha', 'missing', 'tag'):
             data = release()
@@ -91,8 +93,9 @@ class ReleaseGateTests(unittest.TestCase):
                 api.release({'id': 1, 'head_sha': 'a' * 40})
 
 
-class SelectionTests(unittest.TestCase):
+class SelectionTests(IsolatedCITestCase):
     def setUp(self):
+        super().setUp()
         self.scratch = tempfile.TemporaryDirectory()
         self.addCleanup(self.scratch.cleanup)
         self.repo = Path(self.scratch.name)
@@ -171,7 +174,7 @@ class SelectionTests(unittest.TestCase):
         self.assertEqual(result['selected']['k8s'], ['one', 'postgres', 'two'])
 
 
-class ComposeConfigurationTests(unittest.TestCase):
+class ComposeConfigurationTests(IsolatedCITestCase):
     def test_pin_preserves_project_volumes_paths_and_previous_image(self):
         with tempfile.TemporaryDirectory() as scratch:
             root = Path(scratch)
@@ -305,7 +308,7 @@ class ComposeConfigurationTests(unittest.TestCase):
                 )
 
 
-class ControllerTests(unittest.TestCase):
+class ControllerTests(IsolatedCITestCase):
     def test_completed_stage_cannot_apply_again(self):
         with tempfile.TemporaryDirectory() as scratch:
             directory = Path(scratch)

tests/test_cicd_lifecycle.py

@@ -10,10 +10,11 @@ import zipfile
 from pathlib import Path
 from unittest.mock import Mock, patch
 
+from ci_test_case import IsolatedCITestCase
 from test_cicd import ROOT, controller, release, release_module
 
 
-class ArtifactTests(unittest.TestCase):
+class ArtifactTests(IsolatedCITestCase):
     def test_archive_rejects_nested_or_extra_files(self):
         api = object.__new__(release_module.Gitea)
         api.base = 'https://example.test/api/v1/repos/a/b'
@@ -103,7 +104,7 @@ class ArtifactTests(unittest.TestCase):
             self.assertEqual(json.loads((root / 'error-pages.json').read_text())['sha'], 'e' * 40)
 
 
-class DurableRunTests(unittest.TestCase):
+class DurableRunTests(IsolatedCITestCase):
     def test_duplicate_start_only_reattaches(self):
         with tempfile.TemporaryDirectory() as scratch:
             state = Path(scratch)
@@ -203,7 +204,7 @@ class DurableRunTests(unittest.TestCase):
             self.assertEqual(json.loads((directory / 'status.json').read_text())['state'], 'failure')
 
 
-class FailureSummaryTests(unittest.TestCase):
+class FailureSummaryTests(IsolatedCITestCase):
     def test_build_failure_keeps_progress_and_does_not_expose_exception_text(self):
         with tempfile.TemporaryDirectory() as scratch:
             summary = Path(scratch) / 'summary.md'
@@ -261,7 +262,7 @@ class FailureSummaryTests(unittest.TestCase):
             self.assertIn('Compose requires manual recovery', content)
 
 
-class InstallerTests(unittest.TestCase):
+class InstallerTests(IsolatedCITestCase):
     def test_version_comparison_is_exact_without_network_or_host_packages(self):
         with tempfile.TemporaryDirectory() as scratch:
             root = Path(scratch)

tests/test_image_matrix.py

@@ -3,10 +3,10 @@
 import json
 import os
 import tempfile
-import unittest
 from pathlib import Path
 from unittest.mock import Mock, call, patch
 
+from ci_test_case import IsolatedCITestCase
 from test_cicd import release, release_module
 
 
@@ -26,7 +26,7 @@ def plan_data(changed):
     return {'sha': 'a' * 40, 'targets': targets}
 
 
-class MatrixTests(unittest.TestCase):
+class MatrixTests(IsolatedCITestCase):
     def test_no_change_one_image_all_images_and_missing_baseline(self):
         for changed in (set(), {'error-pages'}, set(release_module.IMAGES)):
             with self.subTest(changed=changed), tempfile.TemporaryDirectory() as scratch:

tests/test_netbird_runtime.py

@@ -7,11 +7,14 @@ import tempfile
 import unittest
 from pathlib import Path
 
+from ci_test_case import IsolatedCITestCase
+
 ROOT = Path(__file__).resolve().parents[1]
 
 
-class NetbirdRuntimeTests(unittest.TestCase):
+class NetbirdRuntimeTests(IsolatedCITestCase):
     def setUp(self):
+        super().setUp()
         self.temp = tempfile.TemporaryDirectory()
         self.addCleanup(self.temp.cleanup)
         self.root = Path(self.temp.name)