PR feedback

Author: frostebite

.github/workflows/cloud-runner-integrity.yml

@@ -76,6 +76,15 @@ jobs:
         with:
           node-version: 20
           cache: 'yarn'
+      - name: Clean up disk space before tests
+        run: |
+          # Clean up any leftover cache files from previous runs
+          rm -rf ./cloud-runner-cache/* || true
+          # Clean up system caches and temporary files
+          sudo apt-get clean || true
+          docker system prune -f || true
+          # Show available disk space
+          df -h
       - run: yarn install --frozen-lockfile
       - run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
         timeout-minutes: 60
@@ -131,6 +140,15 @@ jobs:
         with:
           node-version: 20
           cache: 'yarn'
+      - name: Clean up disk space before tests
+        run: |
+          # Clean up any leftover cache files from previous runs
+          rm -rf ./cloud-runner-cache/* || true
+          # Clean up system caches and temporary files
+          sudo apt-get clean || true
+          docker system prune -f || true
+          # Show available disk space
+          df -h
       - run: yarn install --frozen-lockfile
       - run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
         timeout-minutes: 60

dist/index.js

@@ -7,7 +7,43 @@ class KubernetesPods {
     const phase = pods[0]?.status?.phase || 'undefined status';
     CloudRunnerLogger.log(`Getting pod status: ${phase}`);
     if (phase === `Failed`) {
-      throw new Error(`K8s pod failed`);
+      const pod = pods[0];
+      const containerStatuses = pod.status?.containerStatuses || [];
+      const conditions = pod.status?.conditions || [];
+      const events = (await kubeClient.listNamespacedEvent(namespace)).body.items
+        .filter((x) => x.involvedObject?.name === podName)
+        .map((x) => ({
+          message: x.message || '',
+          reason: x.reason || '',
+          type: x.type || '',
+        }));
+
+      const errorDetails: string[] = [];
+      errorDetails.push(`Pod: ${podName}`);
+      errorDetails.push(`Phase: ${phase}`);
+      
+      if (conditions.length > 0) {
+        errorDetails.push(`Conditions: ${JSON.stringify(conditions.map(c => ({ type: c.type, status: c.status, reason: c.reason, message: c.message })), undefined, 2)}`);
+      }
+
+      if (containerStatuses.length > 0) {
+        containerStatuses.forEach((cs, idx) => {
+          if (cs.state?.waiting) {
+            errorDetails.push(`Container ${idx} (${cs.name}) waiting: ${cs.state.waiting.reason} - ${cs.state.waiting.message || ''}`);
+          }
+          if (cs.state?.terminated) {
+            errorDetails.push(`Container ${idx} (${cs.name}) terminated: ${cs.state.terminated.reason} - ${cs.state.terminated.message || ''} (exit code: ${cs.state.terminated.exitCode})`);
+          }
+        });
+      }
+
+      if (events.length > 0) {
+        errorDetails.push(`Recent events: ${JSON.stringify(events.slice(-5), undefined, 2)}`);
+      }
+
+      const errorMessage = `K8s pod failed\n${errorDetails.join('\n')}`;
+      CloudRunnerLogger.log(errorMessage);
+      throw new Error(errorMessage);
     }
 
     return running;

dist/index.js.map

@@ -87,5 +87,19 @@ describe('Cloud Runner Caching', () => {
       expect(build2NotContainsZeroLibraryCacheFilesMessage).toBeTruthy();
       expect(build2NotContainsZeroLFSCacheFilesMessage).toBeTruthy();
     }, 1_000_000_000);
+    afterAll(async () => {
+      // Clean up cache files to prevent disk space issues
+      if (CloudRunnerOptions.providerStrategy === `local-docker` || CloudRunnerOptions.providerStrategy === `aws`) {
+        const cachePath = `./cloud-runner-cache`;
+        if (fs.existsSync(cachePath)) {
+          try {
+            CloudRunnerLogger.log(`Cleaning up cache directory: ${cachePath}`);
+            await CloudRunnerSystem.Run(`rm -rf ${cachePath}/* || true`);
+          } catch (error: any) {
+            CloudRunnerLogger.log(`Failed to cleanup cache: ${error.message}`);
+          }
+        }
+      }
+    });
   }
 });

src/model/cloud-runner/providers/k8s/kubernetes-pods.ts

@@ -86,6 +86,23 @@ describe('Cloud Runner Retain Workspace', () => {
         CloudRunnerLogger.log(
           `Cleaning up ./cloud-runner-cache/${path.basename(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
         );
+        try {
+          await CloudRunnerSystem.Run(
+            `rm -rf ./cloud-runner-cache/${path.basename(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)} || true`,
+          );
+        } catch (error: any) {
+          CloudRunnerLogger.log(`Failed to cleanup workspace: ${error.message}`);
+        }
+      }
+      // Clean up cache files to prevent disk space issues
+      const cachePath = `./cloud-runner-cache`;
+      if (fs.existsSync(cachePath)) {
+        try {
+          CloudRunnerLogger.log(`Cleaning up cache directory: ${cachePath}`);
+          await CloudRunnerSystem.Run(`rm -rf ${cachePath}/* || true`);
+        } catch (error: any) {
+          CloudRunnerLogger.log(`Failed to cleanup cache: ${error.message}`);
+        }
       }
     });
   }