gen2e: include all aria attributes, set default spolicy to model

rhighs · Jun 11, 2024 · 2e97768 · 2e97768
1 parent 319419f
commit 2e97768
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 6 deletions.
diff --git a/packages/gen2e/src/gen.ts b/packages/gen2e/src/gen.ts
@@ -107,7 +107,7 @@ const evalLoop = async (
   const errors: Error[] = [];
   const attempts: string[] = [];
 
-  const _spolicy = policies.screenshot ?? "onfail";
+  const _spolicy = policies.screenshot ?? "model";
   const _model = (model ?? env.OPENAI_MODEL) as Gen2ELLMAgentModel;
 
   const shouldScreenshot = (
@@ -297,7 +297,7 @@ const _gen: GenType = (
         evalCode,
         policies: {
           maxRetries: options?.policies?.maxRetries ?? 3,
-          screenshot: options?.policies?.screenshot ?? "onfail",
+          screenshot: options?.policies?.screenshot ?? "model",
         },
       },
       {
@@ -420,7 +420,7 @@ _gen.test = function (
             evalCode,
             policies: {
               maxRetries: options?.policies?.maxRetries ?? 3,
-              screenshot: options?.policies?.screenshot ?? "onfail",
+              screenshot: options?.policies?.screenshot ?? "model",
             },
           },
           {

diff --git a/packages/gen2e/src/snapshot.ts b/packages/gen2e/src/snapshot.ts
@@ -82,8 +82,7 @@ const attributes: {
       "href",
       "role",
       "title",
-      "aria-label",
-      "aria-labelledby",
+      "aria*",
       "data-testid",
       "data-*",
       "for",
@@ -94,7 +93,7 @@ const attributes: {
     frame: ["src"],
   },
   high: {
-    "*": ["id", "placeholder", "data-testid", "textContent"],
+    "*": ["id", "placeholder", "data-testid", "textContent", "aria*"],
     iframe: ["src"],
     frame: ["src"],
   },