error reporting

benchmarks-evalite/benchmark.eval.ts

@@ -82,11 +82,26 @@ evalite("Electronics Engineer", {
           expected: question.answer,
         })
       }
+      return output
     }
-
-    return output
+    return `${evaluation.error}\nCode: ${code}`
   },
   experimental_customColumns: async (result) => {
+    if (typeof result.output === "string")
+      return [
+        {
+          label: "Prompt",
+          value: result.input.prompt,
+        },
+        {
+          label: "Code",
+          value: result.output,
+        },
+        {
+          label: "Result",
+          value: "Circuit failed",
+        },
+      ]
     return [
       {
         label: "Prompt",

benchmarks-evalite/prompt.ts

@@ -4,8 +4,6 @@ import {
   fp,
 } from "@tscircuit/footprinter"
 
-import { writeFile } from "node:fs"
-
 async function fetchFileContent(url: string): Promise<string> {
   try {
     const response = await fetch(url)
@@ -45,7 +43,6 @@ export const createPrompt = async () => {
     .filter((line) => !line.startsWith("#"))
     .join("\n")
     .replace(/\n\n+/g, "\n\n")
-  writeFile("propsDoc.md", cleanedPropsDoc, { encoding: "utf-8" }, (err) => {})
 
   return `
 You are an expert in electronic circuit design and tscircuit, and your job is to create a circuit board in tscircuit with the user-provided description.

benchmarks-evalite/scorers/circuit-scorer.ts

@@ -8,20 +8,22 @@ export const CircuitScorer = createScorer<
       answer: boolean
     }[]
   },
-  {
-    results: {
-      result: boolean
-      expected: boolean
-    }[]
-    code: string
-  }
+  | {
+      results: {
+        result: boolean
+        expected: boolean
+      }[]
+      code: string
+    }
+  | string
 >({
   name: "circuit_scorer",
   description: "Evaluates circuit code for presence of key components",
   scorer: ({ input, output }) => {
     if (!output) {
       return { score: 0 }
     }
+    if (typeof output === "string") return { score: 0 }
 
     const score = output.results.reduce((acc, { result, expected }) => {
       return acc + (result === expected ? 0.25 : 0)