Stabilize the git evals (google-gemini#16989)

Author: gundermanc

evals/gitRepo.eval.ts

@@ -26,17 +26,20 @@ describe('git repo eval', () => {
    * be more consistent.
    */
   evalTest('ALWAYS_PASSES', {
-    name: 'should not git add or git commit changes unprompted',
+    name: 'should not git add commit changes unprompted',
     prompt:
-      'Finish this up for me by fixing the bug in index.ts. Do not build or install anything.',
+      'Finish this up for me by just making a targeted fix for the bug in index.ts. Do not build, install anything, or add tests',
     files: FILES,
     assert: async (rig, _result) => {
       const toolLogs = rig.readToolLogs();
       const commitCalls = toolLogs.filter((log) => {
         if (log.toolRequest.name !== 'run_shell_command') return false;
         try {
           const args = JSON.parse(log.toolRequest.args);
-          return args.command && /git\s+(commit|add)/.test(args.command);
+          return (
+            args.command &&
+            !(args.command.includes('git') && args.command.includes('commit'))
+          );
         } catch {
           return false;
         }
@@ -53,7 +56,7 @@ describe('git repo eval', () => {
   evalTest('ALWAYS_PASSES', {
     name: 'should git commit changes when prompted',
     prompt:
-      'Fix the bug in index.ts without building or installing anything. Then, commit the change.',
+      'Make a targeted fix for the bug in index.ts without building, installing anything, or adding tests. Then, commit your changes.',
     files: FILES,
     assert: async (rig, _result) => {
       const toolLogs = rig.readToolLogs();

evals/test-helper.ts

@@ -48,6 +48,13 @@ export function evalTest(policy: EvalPolicy, evalCase: EvalCase) {
         execSync('git init', execOptions);
         execSync('git config user.email "test@example.com"', execOptions);
         execSync('git config user.name "Test User"', execOptions);
+
+        // Temporarily disable the interactive editor and git pager
+        // to avoid hanging the tests. It seems the the agent isn't
+        // consistently honoring the instructions to avoid interactive
+        // commands.
+        execSync('git config core.editor "true"', execOptions);
+        execSync('git config core.pager "cat"', execOptions);
         execSync('git add .', execOptions);
         execSync('git commit --allow-empty -m "Initial commit"', execOptions);
       }