From d071fa9fb2d1fff98e3feb2dfdeefa1bb46ebcbe Mon Sep 17 00:00:00 2001
From: seveibar <seveibar@gmail.com>
Date: Mon, 30 Sep 2024 21:06:26 -0700
Subject: [PATCH] set structure for running benchmarks

---
 README.md                                 | 14 ++++++++++
 package.json                              |  2 +-
 prompt-templates/create-circuit-board1.ts | 31 ++++++++++++++++++-----
 3 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 7668151..614375b 100644
--- a/README.md
+++ b/README.md
@@ -4,3 +4,17 @@
 
 This repo contains benchmarks for tscircuit system prompts used for
 automatically generating tscircuit code.
+
+## Running Benchmarks
+
+You can use `bun run benchmark` to select and run a benchmark. A single prompt takes about 10s-15s to
+run when run with `sonnet`. We have a set of samples (see the [tests/samples](./tests/samples) directory)
+that the benchmarks run against. When you change a prompt, you must run the benchmark
+for that prompt to update the benchmark snapshot. This is how we record degradation
+or improvement in the response quality. Each sample is run 5 times and two tests
+are run:
+
+1. Does the output from the prompt compile?
+2. Does the output produce the expected circuit?
+
+The benchmark shows the percentage of samples that pass (1) and (2)
diff --git a/package.json b/package.json
index 858c17e..79228a8 100644
--- a/package.json
+++ b/package.json
@@ -9,7 +9,7 @@
   "author": "Severin Ibarluzea",
   "scripts": {
     "build": "tsup lib/index.ts --format esm --dts",
-    "test": "bun test"
+    "test": "bun test --timeout 60000"
   },
   "type": "module",
   "devDependencies": {
diff --git a/prompt-templates/create-circuit-board1.ts b/prompt-templates/create-circuit-board1.ts
index f1280ff..3e52966 100644
--- a/prompt-templates/create-circuit-board1.ts
+++ b/prompt-templates/create-circuit-board1.ts
@@ -22,15 +22,15 @@ Here's an overview of the tscircuit API:
   7: "EN",
   8: "GND",
 }} />
-<diode name="D1" />
-<resistor name="R1" resistance="1k" />
-<capacitor name="C1" capacitance="100nF" />
+<diode name="D1" footprint="0805" />
+<resistor name="R1" resistance="1k" footprint="0402" />
+<capacitor name="C1" capacitance="100nF" footprint="0603" />
 <trace from=".R1 .pin1" to=".C1 .pin1" />
 <trace from=".U1 .pin5" to=".D1 .pin2" />
 <trace from=".U1 .D3" to=".U1 .GND" />
 <trace from=".U1 .D2" to="net.VCC" />
-<resistor pullupFor=".U1 .D1" pullupTo="net.VCC" />
-<resistor decouplingFor=".U1 .VCC" decouplingTo="net.GND" />
+<resistor pullupFor=".U1 .D1" pullupTo="net.VCC" footprint="axial_p0.2in" />
+<resistor decouplingFor=".U1 .VCC" decouplingTo="net.GND" footprint="axial_p5.08mm" />
 
 ### footprint strings
 
@@ -49,6 +49,7 @@ soic8_p1.27mm
 dip16
 pinrow10
 tssop20_p0.5mm
+sot23
 
 ### Notes
 
@@ -58,11 +59,27 @@ tssop20_p0.5mm
   to connect components.
 - Any component can have a \`name\` prop
 - \`pcbX\` and \`pcbY\` are optional and default to 0.
-- A board is centered on the origin (pcbX=0, pcbY=0)
+- A board is centered on the origin (pcbX=0, pcbY=0), so to place a component
+  at the center it must be placed at pcbX=0,pcbY=0. Similarly, if you're trying
+  to layout components around the center, you would make ones to the left of
+  the center have negative pcbX values, below the center have negative pcbY,
+  and to the right of the center have positive pcbX values, and above the
+  center have positive pcbY values.
+- Generally every component that is going to be placed should be given a
+  footprint
 
+### Trace Reference Syntax
 
-### Quirks
+Traces are created using the \`<trace />\` component. The \`from\` and \`to\`
+fields are CSS selectors that reference the components to connect.
+
+Examples:
+
+<trace from=".U1 .pin1" to=".R1 .pin1" />
+<trace from=".U1 .D3" to=".U1 .GND" />
+<trace from=".U1 .D2" to="net.VCC" />
 
+### Quirks
 
 ### Output