Merge pull request #2 from tscircuit/run-support

run support checkpoint

.gitignore

@@ -174,3 +174,4 @@ dist
 # Finder (MacOS) folder config
 .DS_Store
 .vscode
+.aider*

package.json

@@ -14,10 +14,19 @@
   "type": "module",
   "devDependencies": {
     "@anthropic-ai/sdk": "^0.27.3",
+    "@biomejs/biome": "^1.9.3",
+    "@tscircuit/core": "^0.0.100",
+    "@types/babel__standalone": "^7.1.7",
     "@types/bun": "latest",
+    "@types/react": "^18.3.11",
+    "react": "^18.3.1",
     "tsup": "^8.3.0"
   },
   "peerDependencies": {
     "typescript": "^5.0.0"
+  },
+  "dependencies": {
+    "@babel/standalone": "^7.25.7",
+    "extract-codefence": "^0.0.3"
   }
 }

scripts/threshold-test.ts

@@ -0,0 +1,78 @@
+import { spawn } from "node:child_process"
+import fs from "node:fs"
+import path from "node:path"
+
+function runTest(directory: string): Promise<boolean> {
+  return new Promise((resolve) => {
+    const child = spawn("bun", ["test", directory], { stdio: "ignore" })
+    child.on("close", (code) => {
+      resolve(code === 0)
+    })
+  })
+}
+
+async function runTests(directory: string, times: number): Promise<number> {
+  const testPromises = Array(times)
+    .fill(null)
+    .map(() => runTest(directory))
+  const results = await Promise.all(testPromises)
+  return results.filter(Boolean).length
+}
+
+function getAllTestFiles(directory: string): string[] {
+  const testFiles: string[] = []
+
+  function traverseDirectory(dir: string) {
+    const files = fs.readdirSync(dir)
+
+    for (const file of files) {
+      const fullPath = path.join(dir, file)
+      if (fs.statSync(fullPath).isDirectory()) {
+        traverseDirectory(fullPath)
+      } else if (file.endsWith(".test.ts") || file.endsWith(".test.tsx")) {
+        testFiles.push(fullPath)
+      }
+    }
+  }
+
+  traverseDirectory(directory)
+  return testFiles
+}
+
+async function main() {
+  const args = process.argv.slice(2)
+  const directory = args[0]
+  const times = parseInt(args[2], 10)
+  const threshold = parseFloat(args[4])
+
+  if (!directory || Number.isNaN(times) || Number.isNaN(threshold)) {
+    console.error(
+      "Usage: bun run scripts/threshold-test.ts <directory> --times <number> --threshold <number>",
+    )
+    process.exit(1)
+  }
+
+  const testFiles = getAllTestFiles(directory)
+  console.log(`Found ${testFiles.length} test files in ${directory}`)
+
+  console.time("Tests duration")
+  const successCount = await runTests(directory, times)
+  console.timeEnd("Tests duration")
+
+  const successRate = successCount / times
+
+  console.log(`Tests ran ${times} times`)
+  console.log(`Success rate: ${(successRate * 100).toFixed(2)}%`)
+
+  if (successRate >= threshold) {
+    console.log("Threshold met!")
+    process.exit(0)
+  } else {
+    console.error(
+      `Threshold not met. Expected ${threshold * 100}%, got ${(successRate * 100).toFixed(2)}%`,
+    )
+    process.exit(1)
+  }
+}
+
+main()

tests/fixtures/run-prompt.tsx

@@ -0,0 +1,77 @@
+import Anthropic from "@anthropic-ai/sdk"
+import * as React from "react"
+import { Circuit } from "@tscircuit/core"
+import { safeEvaluateCode } from "./safe-evaluate-code"
+import { extractCodefence } from "extract-codefence"
+
+const anthropic = new Anthropic()
+
+export const runInitialPrompt = async (
+  systemPrompt: string,
+  userPrompt: string,
+  opts: {
+    model?: "claude-3-5-sonnet-20240620" | "claude-3-haiku-20240307"
+    type?: "board" | "footprint" | "package" | "model"
+  } = {},
+) => {
+  const type = opts.type ?? "board"
+  const completion = await anthropic.messages.create({
+    model: opts.model ?? "claude-3-5-sonnet-20240620",
+    system: systemPrompt,
+    messages: [
+      {
+        role: "user",
+        content: userPrompt,
+      },
+    ],
+    max_tokens: 4096,
+  })
+
+  const responseText: string = (completion as any).content[0]?.text
+
+  const codefence = extractCodefence(responseText)
+
+  if (!codefence) {
+    throw new Error("No codefence found in response")
+  }
+
+  // Run the codefence, detect syntax errors, and evaluate circuit
+  const {
+    success,
+    error,
+    errorStage,
+    circuit,
+    circuitJson,
+    hasSyntaxError,
+    syntaxError,
+    circuitErrors,
+    typescriptErrors,
+  } = safeEvaluateCode(codefence, type)
+
+  if (success) {
+    return {
+      success: true as const,
+      codefence,
+      error,
+      hasSyntaxError: false,
+      syntaxError: undefined,
+      circuitErrors: [],
+      typescriptErrors: [],
+      circuit,
+      circuitJson,
+    }
+  }
+
+  return {
+    success: false as const,
+    codefence,
+    error,
+    errorStage,
+    hasSyntaxError,
+    syntaxError,
+    circuitErrors,
+    typescriptErrors,
+    circuit,
+    circuitJson,
+  }
+}

tests/fixtures/safe-evaluate-code.tsx

@@ -0,0 +1,127 @@
+import Anthropic from "@anthropic-ai/sdk"
+import * as React from "react"
+import { Circuit } from "@tscircuit/core"
+import { safeTranspileCode } from "./transpile-code"
+
+export const safeEvaluateCode = (
+  code: string,
+  type: "board" | "footprint" | "package" | "model" = "board",
+):
+  | {
+      success: true
+      circuit: Circuit
+      circuitJson: any
+      errorStage?: undefined
+      error?: undefined
+      hasSyntaxError?: undefined
+      syntaxError?: undefined
+      circuitErrors?: undefined
+      typescriptErrors?: undefined
+    }
+  | {
+      success: false
+      error: string
+      errorStage: string
+      hasSyntaxError: boolean
+      syntaxError?: string
+      circuitErrors?: any[]
+      typescriptErrors?: string[]
+      circuit?: undefined
+      circuitJson?: undefined
+    } => {
+  globalThis.React = React
+
+  const { success, error, transpiledCode } = safeTranspileCode(code)
+
+  if (error) {
+    return {
+      success: false,
+      error: error,
+      errorStage: "transpilation",
+      hasSyntaxError: true,
+      syntaxError: error,
+      circuitErrors: [],
+      typescriptErrors: [],
+    }
+  }
+
+  const functionBody = `var exports = {}; var module = { exports }; ${transpiledCode}; return module;`
+  let module: any
+  try {
+    module = Function(functionBody).call(globalThis)
+  } catch (error: any) {
+    return {
+      success: false,
+      error: error.message,
+      errorStage: "transpilation",
+      hasSyntaxError: false,
+    }
+  }
+
+  if (Object.keys(module.exports).length > 1) {
+    return {
+      success: false,
+      error: `Too many exports, only export one thing. You exported: ${JSON.stringify(Object.keys(module.exports))}`,
+      errorStage: "generation-result",
+      hasSyntaxError: false,
+    }
+  }
+
+  const primaryKey = Object.keys(module.exports)[0]
+  const UserElm = (props: any) =>
+    React.createElement(module.exports[primaryKey], props)
+
+  // Construct React tree
+  try {
+    const tree = <UserElm />
+  } catch (error: any) {
+    return {
+      success: false,
+      errorStage: "react-tree-construction",
+      error: error.toString(),
+      hasSyntaxError: true,
+    }
+  }
+
+  const circuit = new Circuit()
+  try {
+    if (type === "board") {
+      circuit.add(<UserElm />)
+    } else if (type === "package") {
+      circuit.add(
+        <board width="10mm" height="10mm">
+          <UserElm name="U1" />
+        </board>,
+      )
+    } else if (type === "footprint") {
+      circuit.add(
+        <board width="10mm" height="10mm">
+          <chip name="U1" footprint={<UserElm />} />
+        </board>,
+      )
+    } else if (type === "model") {
+      // Note: This part might need adjustment as we don't have access to jscad-related imports
+      // const jscadGeoms: any[] = []
+      // const { createJSCADRoot } = createJSCADRenderer(jscadPlanner as any)
+      // const jscadRoot = createJSCADRoot(jscadGeoms)
+      // jscadRoot.render(<UserElm />)
+      circuit.add(
+        <board width="10mm" height="10mm">
+          {/* <chip name="U1" cadModel={{}} /> */}
+        </board>,
+      )
+    }
+
+    circuit.render()
+    const circuitJson = circuit.getCircuitJson()
+
+    return { success: true, circuitJson, circuit }
+  } catch (error: any) {
+    return {
+      success: false,
+      error: error.toString(),
+      errorStage: "circuit-rendering",
+      hasSyntaxError: false,
+    }
+  }
+}

tests/fixtures/transpile-code.tsx

@@ -0,0 +1,31 @@
+// if (!code) return ""
+// if (isStreaming) return ""
+// try {
+//   const result = Babel.transform(code, {
+//     presets: ["react", "typescript"],
+//     plugins: ["transform-modules-commonjs"],
+//     filename: "virtual.tsx",
+//   })
+//   return result.code || ""
+// } catch (error: any) {
+//   console.error("Babel compilation error:", error)
+//   return `Error: ${error.message}`
+// }
+
+
+import * as Babel from "@babel/standalone"
+
+export const safeTranspileCode = (code: string): { success: true, transpiledCode: string, error: undefined } | { success: false, error: string, transpiledCode: undefined } => {
+  if (!code) return { success: true, transpiledCode: "", error: undefined }
+  try {
+    const result = Babel.transform(code, {
+      presets: ["react", "typescript"],
+      plugins: ["transform-modules-commonjs"],
+      filename: "index.tsx",
+    })
+    return { success: true, transpiledCode: result.code || "", error: undefined }
+  } catch (error: any) {
+    console.error("Babel compilation error:", error)
+    return { success: false, error: error.message, transpiledCode: undefined }
+  }
+}

tests/prompts/create-circuit-board1/create-circuit-board1-prompt1.test.ts → tests/prompts/create-circuit-board1/create-circuit-board1-sample1.test.ts

@@ -6,8 +6,14 @@ import { sample1 } from "tests/samples/sample1"
 test("create-circuit-board1-prompt1", async () => {
   const systemPrompt = createCircuitBoard1Template({ currentCode: "" })
 
-  const code = await runInitialPrompt(systemPrompt, sample1)
+  const { success, circuit } = await runInitialPrompt(systemPrompt, sample1, {
+    model: "claude-3-haiku-20240307",
+    type: "board",
+  })
 
-  console.log(code)
-  expect(code).toBeDefined()
+  expect(success).toBe(true)
+
+  const led = circuit?.selectOne("led")
+
+  expect(led).toBeDefined()
 })

tests/samples/sample1.ts

@@ -1 +1 @@
-export const sample1 = "blinking led using a 555 timer"
+export const sample1 = "just an led with an 0402 footprint"