update: swe

Author: Palashio

apps/open-swe/evals/langgraph.eval.ts

@@ -79,7 +79,7 @@ ls.describe(DATASET_NAME, () => {
 
       const encryptionKey = process.env.SECRETS_ENCRYPTION_KEY;
       const githubPat = process.env.GITHUB_PAT;
-      
+
       if (!encryptionKey || !githubPat) {
         throw new Error(
           "SECRETS_ENCRYPTION_KEY and GITHUB_PAT environment variables are required",

apps/open-swe/langbench/evaluator.eval.ts

@@ -12,7 +12,7 @@ import { PRData, PRProcessResult, OpenSWEStreamResults } from "./types.js";
 import { createPRFixPrompt } from "./prompts.js";
 import { runPytestOnFiles } from "./utils.js";
 import { v4 as uuidv4 } from "uuid";
-import { MANAGER_GRAPH_ID, GITHUB_PAT, GITHUB_INSTALLATION_ID } from "@open-swe/shared/constants";
+import { MANAGER_GRAPH_ID, GITHUB_PAT } from "@open-swe/shared/constants";
 import { createLangGraphClient } from "../src/utils/langgraph-client.js";
 import { encryptSecret } from "@open-swe/shared/crypto";
 import { ManagerGraphState } from "@open-swe/shared/open-swe/manager/types";
@@ -93,21 +93,19 @@ async function runOpenSWEWithStreamTracking(inputs: {
   try {
     const encryptionKey = process.env.SECRETS_ENCRYPTION_KEY;
     const githubPat = process.env.GITHUB_PAT;
-    const githubInstallationId = process.env.GITHUB_INSTALLATION_ID;
 
-    if (!encryptionKey || !githubPat || !githubInstallationId) {
+    if (!encryptionKey || !githubPat) {
       throw new Error(
-        "SECRETS_ENCRYPTION_KEY, GITHUB_PAT, and GITHUB_INSTALLATION_ID environment variables are required"
+        "SECRETS_ENCRYPTION_KEY and GITHUB_PAT environment variables are required",
       );
     }
 
     const encryptedGitHubToken = encryptSecret(githubPat, encryptionKey);
 
     const lgClient = createLangGraphClient({
       includeApiKey: true,
-      defaultHeaders: { 
+      defaultHeaders: {
         [GITHUB_PAT]: encryptedGitHubToken,
-        [GITHUB_INSTALLATION_ID]: githubInstallationId,
       },
     });
 
@@ -165,6 +163,10 @@ async function runOpenSWEWithStreamTracking(inputs: {
       plannerRun = await withRetry(() =>
         lgClient.runs.join(plannerSession.threadId, plannerSession.runId),
       );
+
+      logger.info("Agent joined planner session", {
+        plannerSession,
+      });
     } catch (error) {
       logger.error("Error joining planner run", {
         thread_id: threadId,
@@ -192,6 +194,10 @@ async function runOpenSWEWithStreamTracking(inputs: {
       return result; // instead of skipping, we should award 0 points
     }
 
+    logger.info("Agent joined programmer session", {
+      programmerSession,
+    });
+
     let programmerRun;
     try {
       programmerRun = await withRetry(() =>
@@ -231,8 +237,8 @@ async function runOpenSWEWithStreamTracking(inputs: {
     result.managerRunId = (managerRun as any).run_id;
     result.plannerRunId = plannerSession.runId;
     result.programmerRunId = programmerSession.runId;
+    result.branchName = branchName;
     result.success = true;
-
   } catch (error) {
     result.error = error instanceof Error ? error.message : String(error);
     logger.error("Open-swe stream tracking failed", {
@@ -312,30 +318,28 @@ async function processPR(prData: PRData): Promise<PRProcessResult> {
     logger.info("Starting open-swe...");
     const openSWEResults = await runOpenSWEWithStreamTracking({
       repoOwner: prData.repoOwner,
-      repoName: prData.repoName, 
+      repoName: prData.repoName,
       baseCommit: preMergeCommit,
       userInput: createPRFixPrompt(prData),
     });
     result.openSWEResults = openSWEResults;
 
-    // Checkout test files from the merge commit to get the updated test files
-    if (testFiles.length > 0) {
+    // Only proceed with test checkout and execution if open-swe was successful and created a branch
+    if (
+      openSWEResults.success &&
+      openSWEResults.branchName &&
+      testFiles.length > 0
+    ) {
       logger.info(
-        `Checking out test files from merge commit: ${prData.mergeCommitSha}`,
+        `Open-swe completed successfully with branch: ${openSWEResults.branchName}. Checking out test files from merge commit: ${prData.mergeCommitSha}`,
       );
       await checkoutFilesFromCommit({
         sandbox,
         repoDir,
         commitSha: prData.mergeCommitSha,
         filePaths: testFiles,
       });
-    }
 
-    // Run tests on detected test files
-    if (testFiles.length > 0) {
-      logger.info(
-        `Running pytest on ${testFiles.length} detected test files...`,
-      );
       const testResults = await runPytestOnFiles({
         sandbox,
         testFiles,
@@ -345,13 +349,24 @@ async function processPR(prData: PRData): Promise<PRProcessResult> {
       result.testResults = testResults;
 
       logger.info(`Test execution completed for PR #${prData.prNumber}`, {
+        branchName: openSWEResults.branchName,
         totalTests: testResults.totalTests,
         passedTests: testResults.passedTests,
         failedTests: testResults.failedTests,
         success: testResults.success,
       });
     } else {
-      logger.info(`No test files to run for PR #${prData.prNumber}`);
+      if (!openSWEResults.success) {
+        logger.info(
+          `Open-swe failed for PR #${prData.prNumber}, skipping test execution`,
+        );
+      } else if (!openSWEResults.branchName) {
+        logger.info(
+          `No branch created by open-swe for PR #${prData.prNumber}, skipping test execution`,
+        );
+      } else if (testFiles.length === 0) {
+        logger.info(`No test files found for PR #${prData.prNumber}`);
+      }
     }
 
     result.success = true;

apps/open-swe/langbench/prompts.ts

@@ -12,4 +12,4 @@ export function createPRFixPrompt(prData: PRData): string {
   The PR has the following test files:
   ${prData.testFiles.join("\n")}
   `;
-}
+}

apps/open-swe/langbench/static/langgraph_prs.json

@@ -4,7 +4,7 @@
     "html_url": "https://github.com/langchain-ai/langgraph/pull/809",
     "diff_url": "https://github.com/langchain-ai/langgraph/pull/809.diff",
     "patch_url": "https://github.com/langchain-ai/langgraph/pull/809.patch",
-    "repo_owner": "langchain-ai",
+    "repo_owner": "Palashio",
     "repo_name": "langgraph",
     "pr_number": 809,
     "merge_commit_sha": "8e611b42aa35a7032c81ad2fd264d3192d47c911",
@@ -15,4 +15,4 @@
     "pre_merge_commit_sha": "6ae59581643b51751731ae64c609a8bc21779714",
     "test_files": ["libs/langgraph/tests/test_pregel.py"]
   }
-]
+]

apps/open-swe/src/utils/github-tokens.ts

@@ -24,23 +24,23 @@ export function getGitHubTokensFromConfig(config: GraphConfig): {
 
   const isProd = process.env.NODE_ENV === "production";
 
-  const installationId = config.configurable[GITHUB_INSTALLATION_ID];
-  if (!installationId) {
-    throw new Error(
-      `Missing required ${GITHUB_INSTALLATION_ID} in configuration.`,
-    );
-  }
-
   const githubPat = getGitHubPatFromConfig(config.configurable, encryptionKey);
   if (githubPat && !isProd) {
     // check for PAT-only mode
     return {
       githubAccessToken: githubPat,
       githubInstallationToken: githubPat,
-      installationId,
+      installationId: config.configurable[GITHUB_INSTALLATION_ID] ?? "",
     };
   }
 
+  const installationId = config.configurable[GITHUB_INSTALLATION_ID];
+  if (!installationId) {
+    throw new Error(
+      `Missing required ${GITHUB_INSTALLATION_ID} in configuration.`,
+    );
+  }
+
   const encryptedGitHubToken = config.configurable[GITHUB_TOKEN_COOKIE];
   const encryptedInstallationToken =
     config.configurable[GITHUB_INSTALLATION_TOKEN_COOKIE];

apps/open-swe/src/utils/github/api.ts

@@ -579,7 +579,9 @@ export async function getBranch({
         repo,
         branch: branchName,
       });
-
+      logger.info("Branch", {
+        branch,
+      });
       return branch;
     },
     githubInstallationToken,