feat: fs support with drizzle and duckdb-wasm (#30)

* feat: fs support with drizzle and duckdb-wasm * docs: fix lint errors * fix: apply suggestions * fix: lockfile
moeru-ai · Feb 22, 2025 · d985ade · d985ade
1 parent dae594e
commit d985ade
Show file tree

Hide file tree

Showing 18 changed files with 1,425 additions and 148 deletions.
diff --git a/README.md b/README.md
@@ -85,7 +85,7 @@ flowchart TD
   SVRT["@proj-airi/server-runtime"]
   MC_AGENT("Minecraft Agent")
   XSAI["xsai"]
-  
+
   subgraph airi-vtuber
     DB0 --> DB1 --> DB2 --> CORE
     ICONS --> UI --> Stage --> CORE

diff --git a/cspell.config.yaml b/cspell.config.yaml
@@ -97,6 +97,7 @@ words:
   - onnxruntime
   - openai
   - openrouter
+  - OPFS
   - opusscript
   - pgvector
   - picklist

diff --git a/packages/drizzle-duckdb-wasm/package.json b/packages/drizzle-duckdb-wasm/package.json
@@ -55,7 +55,9 @@
     "play:build": "vite build",
     "play:preview": "vite preview",
     "typecheck": "tsc --noEmit",
-    "db:generate": "drizzle-kit generate"
+    "db:generate": "drizzle-kit generate",
+    "test": "vitest",
+    "test:run": "vitest run"
   },
   "peerDependencies": {
     "web-worker": "^1.5.0"
@@ -67,7 +69,7 @@
   },
   "dependencies": {
     "@date-fns/tz": "^1.2.0",
-    "@duckdb/duckdb-wasm": "^1.29.0",
+    "@duckdb/duckdb-wasm": "1.29.1-dev68.0",
     "@proj-airi/duckdb-wasm": "workspace:^",
     "apache-arrow": "^19.0.1",
     "date-fns": "^4.1.0",
@@ -78,8 +80,10 @@
   "devDependencies": {
     "@unocss/reset": "^66.0.0",
     "@vitejs/plugin-vue": "^5.2.1",
+    "@vitest/browser": "^3.0.6",
     "@vueuse/core": "^12.7.0",
     "drizzle-kit": "^0.30.4",
+    "playwright": "^1.50.1",
     "superjson": "^2.2.2",
     "vite": "^6.1.1",
     "vue": "^3.5.13",

diff --git a/packages/drizzle-duckdb-wasm/playground/src/App.vue b/packages/drizzle-duckdb-wasm/playground/src/App.vue
@@ -1,89 +1,258 @@
 <script setup lang="ts">
 import type { DuckDBWasmDrizzleDatabase } from '../../src'
 
-import { useDebounceFn } from '@vueuse/core'
+import { DuckDBAccessMode } from '@duckdb/duckdb-wasm'
+import { DBStorageType } from '@proj-airi/duckdb-wasm'
 import { serialize } from 'superjson'
-import { onMounted, onUnmounted, ref, watch } from 'vue'
+import { computed, onMounted, onUnmounted, ref } from 'vue'
 
 import { drizzle } from '../../src'
+import { buildDSN } from '../../src/dsn'
 import * as schema from '../db/schema'
 import { users } from '../db/schema'
 import migration1 from '../drizzle/0000_cute_kulan_gath.sql?raw'
 
 const db = ref<DuckDBWasmDrizzleDatabase<typeof schema>>()
 const results = ref<Record<string, unknown>[]>()
 const schemaResults = ref<Record<string, unknown>[]>()
-const query = ref(`SELECT 1 + 1 AS result`)
+const isMigrated = ref(false)
 
-onMounted(async () => {
-  db.value = drizzle('duckdb-wasm://?bundles=import-url', { schema })
+const storage = ref<DBStorageType>()
+const path = ref('test.db')
+const logger = ref(true)
+const readOnly = ref(false)
+
+const dsn = computed(() => {
+  return buildDSN({
+    scheme: 'duckdb-wasm:',
+    bundles: 'import-url',
+    logger: logger.value,
+    ...storage.value === DBStorageType.ORIGIN_PRIVATE_FS && {
+      storage: {
+        type: storage.value,
+        path: path.value,
+        accessMode: readOnly.value ? DuckDBAccessMode.READ_ONLY : DuckDBAccessMode.READ_WRITE,
+      },
+    },
+  })
+})
+
+const query = ref(`SELECT * FROM 'users'`)
+
+async function connect() {
+  isMigrated.value = false
+  db.value = drizzle(dsn.value, { schema })
   await db.value?.execute('INSTALL vss;')
   await db.value?.execute('LOAD vss;')
+}
 
+async function migrate() {
   await db.value?.execute(migration1)
-
-  results.value = await db.value?.execute(query.value)
-
-  await db.value.insert(users).values({
+  await db.value?.insert(users).values({
     id: '9449af72-faad-4c97-8a45-69f9f1ca1b05',
     decimal: '1.23456',
     numeric: '1.23456',
     real: 1.23456,
     double: 1.23456,
     interval: '365 day',
   })
+  isMigrated.value = true
+}
+
+async function insert() {
+  await db.value?.insert(users).values({
+    id: crypto.randomUUID().replace(/-/g, ''),
+    decimal: '1.23456',
+    numeric: '1.23456',
+    real: 1.23456,
+    double: 1.23456,
+    interval: '365 day',
+  })
+}
+
+async function reconnect() {
+  const client = await db.value?.$client
+  await client?.close()
+  await connect()
+}
+
+async function execute() {
+  results.value = await db.value?.execute(query.value)
+}
+
+async function executeORM() {
+  schemaResults.value = await db.value?.select().from(users)
+}
 
-  const usersResults = await db.value.select().from(users)
+async function shallowListOPFS() {
+  const opfsRoot = await navigator.storage.getDirectory()
+  const files: string[] = []
+  for await (const name of opfsRoot.keys()) {
+    files.push(name)
+  }
+  // eslint-disable-next-line no-console
+  console.log(['Files in OPFS:', ...files].join('\n'))
+}
+
+async function wipeOPFS() {
+  await db.value?.$client.then(client => client.close())
+  const opfsRoot = await navigator.storage.getDirectory()
+  const promises: Promise<void>[] = []
+  for await (const name of opfsRoot.keys()) {
+    promises.push(opfsRoot.removeEntry(name, { recursive: true }).then(() => {
+      // eslint-disable-next-line no-console
+      console.info(`File removed from OPFS: "${name}"`)
+    }))
+  }
+  await Promise.all(promises)
+}
+
+onMounted(async () => {
+  await connect()
+  await migrate()
+
+  results.value = await db.value?.execute(query.value)
+  const usersResults = await db.value?.select().from(users)
   schemaResults.value = usersResults
 })
 
 onUnmounted(() => {
   db.value?.$client.then(client => client.close())
 })
-
-watch(query, useDebounceFn(async () => {
-  results.value = await db.value?.execute(query.value)
-}, 1000))
 </script>
 
 <template>
-  <div flex flex-col gap-4 p-4>
+  <div flex flex-col gap-2 p-4>
     <h1 text-2xl>
       <code>@duckdb/duckdb-wasm</code> + <code>drizzle-orm</code> Playground
     </h1>
     <div flex flex-col gap-2>
       <h2 text-xl>
-        Executing
+        Storage
       </h2>
-      <div>
-        <textarea v-model="query" h-full w-full rounded-lg bg="neutral-100 dark:neutral-800" p-4 font-mono />
+      <div flex flex-row gap-2>
+        <div flex flex-row gap-2>
+          <input id="in-memory" v-model="storage" type="radio" :value="undefined">
+          <label for="in-memory">In-Memory</label>
+        </div>
+        <div flex flex-row gap-2>
+          <input id="opfs" v-model="storage" type="radio" :value="DBStorageType.ORIGIN_PRIVATE_FS">
+          <label for="opfs">Origin Private FS</label>
+        </div>
       </div>
     </div>
-    <div flex flex-col gap-2>
+    <div grid grid-cols-3 gap-2>
+      <div flex flex-col gap-2>
+        <h2 text-xl>
+          Logger
+        </h2>
+        <div flex flex-row gap-2>
+          <input id="logger" v-model="logger" type="checkbox">
+          <label for="logger">Enable</label>
+        </div>
+      </div>
+      <div flex flex-col gap-2>
+        <h2 text-xl>
+          Read-only
+        </h2>
+        <div flex flex-row gap-2>
+          <input id="readOnly" v-model="readOnly" type="checkbox">
+          <label for="readOnly">Read-only (DB file creation will fail)</label>
+        </div>
+      </div>
+    </div>
+    <div v-if="storage === DBStorageType.ORIGIN_PRIVATE_FS" flex flex-col gap-2>
       <h2 text-xl>
-        Results
+        Path
       </h2>
-      <div whitespace-pre-wrap p-4 font-mono>
-        {{ JSON.stringify(serialize(results).json, null, 2) }}
+      <div flex flex-col gap-1>
+        <input v-model="path" type="text" w-full rounded-lg p-4 font-mono bg="neutral-100 dark:neutral-800">
+        <div text-sm>
+          <ul list-disc-inside>
+            <li>
+              Leading slash is optional ("/path/to/database.db" is equivalent to "path/to/database.db")
+            </li>
+            <li>Empty path is INVALID</li>
+          </ul>
+        </div>
       </div>
     </div>
     <div flex flex-col gap-2>
       <h2 text-xl>
-        Executing
+        DSN (read-only)
       </h2>
       <div>
-        <pre whitespace-pre-wrap rounded-lg p-4 font-mono bg="neutral-100 dark:neutral-800">
-await db.insert(users).values({ id: '9449af72-faad-4c97-8a45-69f9f1ca1b05' })
-await db.select().from(users)
-        </pre>
+        <input v-model="dsn" readonly type="text" w-full rounded-lg p-4 font-mono bg="neutral-100 dark:neutral-800">
       </div>
     </div>
-    <div flex flex-col gap-2>
-      <h2 text-xl>
-        Schema Results
-      </h2>
-      <div whitespace-pre-wrap p-4 font-mono>
-        {{ JSON.stringify(serialize(schemaResults).json, null, 2) }}
+    <div flex flex-row justify-between gap-2>
+      <div flex flex-row gap-2>
+        <button rounded-lg bg="pink-100 dark:pink-700" px-4 py-2 @click="reconnect">
+          Reconnect
+        </button>
+        <button rounded-lg bg="orange-100 dark:orange-700" px-4 py-2 :class="{ 'cursor-not-allowed': isMigrated }" :disabled="isMigrated" @click="migrate">
+          {{ isMigrated ? 'Already migrated 🥳' : 'Migrate' }}
+        </button>
+        <button rounded-lg bg="purple-100 dark:purple-700" px-4 py-2 @click="insert">
+          Insert
+        </button>
+      </div>
+      <div flex flex-row gap-2>
+        <button rounded-lg bg="green-100 dark:green-700" px-4 py-2 @click="shallowListOPFS">
+          List OPFS (See console)
+        </button>
+        <button rounded-lg bg="red-100 dark:red-700" px-4 py-2 @click="wipeOPFS">
+          Wipe OPFS
+        </button>
+      </div>
+    </div>
+    <div grid grid-cols-2 gap-2>
+      <div flex flex-col gap-2>
+        <h2 text-xl>
+          Executing
+        </h2>
+        <div>
+          <textarea v-model="query" h-full w-full rounded-lg bg="neutral-100 dark:neutral-800" p-4 font-mono />
+        </div>
+        <div flex flex-row gap-2>
+          <button rounded-lg bg="blue-100 dark:blue-700" px-4 py-2 @click="execute">
+            Execute
+          </button>
+        </div>
+        <div flex flex-col gap-2>
+          <h2 text-xl>
+            Results
+          </h2>
+          <div whitespace-pre-wrap p-4 font-mono>
+            {{ JSON.stringify(serialize(results).json, null, 2) }}
+          </div>
+        </div>
+      </div>
+      <div>
+        <div flex flex-col gap-2>
+          <h2 text-xl>
+            Executing (ORM, read-only)
+          </h2>
+          <div>
+            <pre whitespace-pre-wrap rounded-lg p-4 font-mono bg="neutral-100 dark:neutral-800">
+await db.insert(users).values({ id: '9449af72-faad-4c97-8a45-69f9f1ca1b05' })
+await db.select().from(users)
+            </pre>
+          </div>
+          <div flex flex-row gap-2>
+            <button rounded-lg bg="blue-100 dark:blue-700" px-4 py-2 @click="executeORM">
+              Execute
+            </button>
+          </div>
+        </div>
+        <div flex flex-col gap-2>
+          <h2 text-xl>
+            Schema Results
+          </h2>
+          <div whitespace-pre-wrap p-4 font-mono>
+            {{ JSON.stringify(serialize(schemaResults).json, null, 2) }}
+          </div>
+        </div>
       </div>
     </div>
   </div>