feat: support downloading car files from @helia/verified-fetch

packages/verified-fetch/README.md

@@ -347,6 +347,39 @@ if (res.headers.get('Content-Type') === 'application/json') {
 console.info(obj) // ...
 ```
 
+## The `Accept` header
+
+The `Accept` header can be passed to override certain response processing, or to ensure that the final `Content-Type` of the response is the one that is expected.
+
+If the final `Content-Type` does not match the `Accept` header, or if the content cannot be represented in the format dictated by the `Accept` header, or you have configured a custom content type parser, and that parser returns a value that isn't in the accept header, a [406: Not Acceptible](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/406) response will be returned:
+
+```typescript
+import { verifiedFetch } from '@helia/verified-fetch'
+
+const res = await verifiedFetch('ipfs://bafyJPEGImageCID', {
+  headers: {
+    accept: 'image/png'
+  }
+})
+
+console.info(res.status) // 406 - the image was a JPEG but we specified PNG as the accept header
+```
+
+It can also be used to skip processing the data from some formats such as `DAG-CBOR` if you wish to handle decoding it yourself:
+
+```typescript
+import { verifiedFetch } from '@helia/verified-fetch'
+
+const res = await verifiedFetch('ipfs://bafyDAGCBORCID', {
+  headers: {
+    accept: 'application/octet-stream'
+  }
+})
+
+console.info(res.headers.get('accept')) // application/octet-stream
+const buf = await res.arrayBuffer() // raw bytes, not processed as JSON
+```
+
 ## Comparison to fetch
 
 This module attempts to act as similarly to the `fetch()` API as possible.

packages/verified-fetch/package.json

@@ -141,6 +141,7 @@
     "release": "aegir release"
   },
   "dependencies": {
+    "@helia/car": "^3.0.0",
     "@helia/block-brokers": "^2.0.1",
     "@helia/http": "^1.0.1",
     "@helia/interface": "^4.0.0",
@@ -155,6 +156,7 @@
     "cborg": "^4.0.9",
     "hashlru": "^2.3.0",
     "ipfs-unixfs-exporter": "^13.5.0",
+    "it-to-browser-readablestream": "^2.0.6",
     "multiformats": "^13.1.0",
     "progress-events": "^1.0.0"
   },
@@ -163,6 +165,7 @@
     "@helia/dag-json": "^3.0.0",
     "@helia/json": "^3.0.0",
     "@helia/utils": "^0.0.1",
+    "@ipld/car": "^5.2.6",
     "@libp2p/logger": "^4.0.5",
     "@libp2p/peer-id-factory": "^4.0.5",
     "@sgtpooki/file-type": "^1.0.1",
@@ -171,9 +174,11 @@
     "blockstore-core": "^4.4.0",
     "datastore-core": "^9.2.8",
     "helia": "^4.0.1",
+    "ipns": "^9.0.0",
     "it-last": "^3.0.4",
     "it-to-buffer": "^4.0.5",
     "magic-bytes.js": "^1.8.0",
+    "p-defer": "^4.0.0",
     "sinon": "^17.0.1",
     "sinon-ts": "^2.0.0",
     "uint8arrays": "^5.0.1"

packages/verified-fetch/src/index.ts

@@ -320,6 +320,39 @@
  * console.info(obj) // ...
  * ```
  *
+ * ## The `Accept` header
+ *
+ * The `Accept` header can be passed to override certain response processing, or to ensure that the final `Content-Type` of the response is the one that is expected.
+ *
+ * If the final `Content-Type` does not match the `Accept` header, or if the content cannot be represented in the format dictated by the `Accept` header, or you have configured a custom content type parser, and that parser returns a value that isn't in the accept header, a [406: Not Acceptible](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/406) response will be returned:
+ *
+ * ```typescript
+ * import { verifiedFetch } from '@helia/verified-fetch'
+ *
+ * const res = await verifiedFetch('ipfs://bafyJPEGImageCID', {
+ *   headers: {
+ *     accept: 'image/png'
+ *   }
+ * })
+ *
+ * console.info(res.status) // 406 - the image was a JPEG but we specified PNG as the accept header
+ * ```
+ *
+ * It can also be used to skip processing the data from some formats such as `DAG-CBOR` if you wish to handle decoding it yourself:
+ *
+ * ```typescript
+ * import { verifiedFetch } from '@helia/verified-fetch'
+ *
+ * const res = await verifiedFetch('ipfs://bafyDAGCBORCID', {
+ *   headers: {
+ *     accept: 'application/octet-stream'
+ *   }
+ * })
+ *
+ * console.info(res.headers.get('accept')) // application/octet-stream
+ * const buf = await res.arrayBuffer() // raw bytes, not processed as JSON
+ * ```
+ *
  * ## Comparison to fetch
  *
  * This module attempts to act as similarly to the `fetch()` API as possible.
@@ -449,6 +482,10 @@ import type { ProgressEvent, ProgressOptions } from 'progress-events'
  */
 export type Resource = string | CID
 
+export interface ResourceDetail {
+  resource: Resource
+}
+
 export interface CIDDetail {
   cid: CID
   path: string

packages/verified-fetch/src/utils/get-content-disposition-filename.ts

@@ -0,0 +1,18 @@
+/**
+ * Takes a filename URL param and returns a string for use in a
+ * `Content-Disposition` header
+ */
+export function getContentDispositionFilename (filename: string): string {
+  const asciiOnly = replaceNonAsciiCharacters(filename)
+
+  if (asciiOnly === filename) {
+    return `filename="${filename}"`
+  }
+
+  return `filename="${asciiOnly}"; filename*=UTF-8''${encodeURIComponent(filename)}`
+}
+
+function replaceNonAsciiCharacters (filename: string): string {
+  // eslint-disable-next-line no-control-regex
+  return filename.replace(/[^\x00-\x7F]/g, '_')
+}

packages/verified-fetch/src/utils/parse-url-string.ts

@@ -19,6 +19,8 @@ export interface ParseUrlStringOptions extends ProgressOptions<ResolveProgressEv
 
 export interface ParsedUrlQuery extends Record<string, string | unknown> {
   format?: RequestFormatShorthand
+  download?: boolean
+  filename?: string
 }
 
 export interface ParsedUrlStringResults {
@@ -109,14 +111,22 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin
   }
 
   // parse query string
-  const query: Record<string, string> = {}
+  const query: Record<string, any> = {}
 
   if (queryString != null && queryString.length > 0) {
     const queryParts = queryString.split('&')
     for (const part of queryParts) {
       const [key, value] = part.split('=')
       query[key] = decodeURIComponent(value)
     }
+
+    if (query.download != null) {
+      query.download = query.download === 'true'
+    }
+
+    if (query.filename != null) {
+      query.filename = query.filename.toString()
+    }
   }
 
   /**

packages/verified-fetch/src/utils/select-output-type.ts

@@ -0,0 +1,163 @@
+import { code as dagCborCode } from '@ipld/dag-cbor'
+import { code as dagJsonCode } from '@ipld/dag-json'
+import { code as dagPbCode } from '@ipld/dag-pb'
+import { code as jsonCode } from 'multiformats/codecs/json'
+import { code as rawCode } from 'multiformats/codecs/raw'
+import type { RequestFormatShorthand } from '../types.js'
+import type { CID } from 'multiformats/cid'
+
+const CID_TYPE_MAP: Record<number, string[]> = {
+  [dagCborCode]: [
+    'application/json',
+    'application/vnd.ipld.dag-cbor',
+    'application/cbor',
+    'application/vnd.ipld.dag-json',
+    'application/octet-stream',
+    'application/vnd.ipld.raw',
+    'application/vnd.ipfs.ipns-record',
+    'application/vnd.ipld.car'
+  ],
+  [dagJsonCode]: [
+    'application/json',
+    'application/vnd.ipld.dag-cbor',
+    'application/cbor',
+    'application/vnd.ipld.dag-json',
+    'application/octet-stream',
+    'application/vnd.ipld.raw',
+    'application/vnd.ipfs.ipns-record',
+    'application/vnd.ipld.car'
+  ],
+  [jsonCode]: [
+    'application/json',
+    'application/vnd.ipld.dag-cbor',
+    'application/cbor',
+    'application/vnd.ipld.dag-json',
+    'application/octet-stream',
+    'application/vnd.ipld.raw',
+    'application/vnd.ipfs.ipns-record',
+    'application/vnd.ipld.car'
+  ],
+  [dagPbCode]: [
+    'application/octet-stream',
+    'application/json',
+    'application/vnd.ipld.dag-cbor',
+    'application/cbor',
+    'application/vnd.ipld.dag-json',
+    'application/vnd.ipld.raw',
+    'application/vnd.ipfs.ipns-record',
+    'application/vnd.ipld.car',
+    'application/x-tar'
+  ],
+  [rawCode]: [
+    'application/octet-stream',
+    'application/vnd.ipld.raw',
+    'application/vnd.ipfs.ipns-record',
+    'application/vnd.ipld.car'
+  ]
+}
+
+/**
+ * Selects an output mime-type based on the CID and a passed `Accept` header
+ */
+export function selectOutputType (cid: CID, accept?: string): string | undefined {
+  const cidMimeTypes = CID_TYPE_MAP[cid.code]
+
+  if (accept != null) {
+    return chooseMimeType(accept, cidMimeTypes)
+  }
+}
+
+function chooseMimeType (accept: string, validMimeTypes: string[]): string | undefined {
+  const requestedMimeTypes = accept
+    .split(',')
+    .map(s => {
+      const parts = s.trim().split(';')
+
+      return {
+        mimeType: `${parts[0]}`.trim(),
+        weight: parseQFactor(parts[1])
+      }
+    })
+    .sort((a, b) => {
+      if (a.weight === b.weight) {
+        return 0
+      }
+
+      if (a.weight > b.weight) {
+        return -1
+      }
+
+      return 1
+    })
+    .map(s => s.mimeType)
+
+  for (const headerFormat of requestedMimeTypes) {
+    for (const mimeType of validMimeTypes) {
+      if (headerFormat.includes(mimeType)) {
+        return mimeType
+      }
+
+      if (headerFormat === '*/*') {
+        return mimeType
+      }
+
+      if (headerFormat.startsWith('*/') && mimeType.split('/')[1] === headerFormat.split('/')[1]) {
+        return mimeType
+      }
+
+      if (headerFormat.endsWith('/*') && mimeType.split('/')[0] === headerFormat.split('/')[0]) {
+        return mimeType
+      }
+    }
+  }
+}
+
+/**
+ * Parses q-factor weighting from the accept header to allow letting some mime
+ * types take precedence over others.
+ *
+ * If the q-factor for an acceptable mime representation is omitted it defaults
+ * to `1`.
+ *
+ * All specified values should be in the range 0-1.
+ *
+ * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept#q
+ */
+function parseQFactor (str?: string): number {
+  if (str != null) {
+    str = str.trim()
+  }
+
+  if (str == null || !str.startsWith('q=')) {
+    return 1
+  }
+
+  const factor = parseFloat(str.replace('q=', ''))
+
+  if (isNaN(factor)) {
+    return 0
+  }
+
+  return factor
+}
+
+const FORMAT_TO_MIME_TYPE: Record<RequestFormatShorthand, string> = {
+  raw: 'application/vnd.ipld.raw',
+  car: 'application/vnd.ipld.car',
+  'dag-json': 'application/vnd.ipld.dag-json',
+  'dag-cbor': 'application/vnd.ipld.dag-cbor',
+  json: 'application/json',
+  cbor: 'application/cbor',
+  'ipns-record': 'application/vnd.ipfs.ipns-record',
+  tar: 'application/x-tar'
+}
+
+/**
+ * Converts a `format=...` query param to a mime type as would be found in the
+ * `Accept` header, if a valid mapping is available
+ */
+export function queryFormatToAcceptHeader (format?: RequestFormatShorthand): string | undefined {
+  if (format != null) {
+    return FORMAT_TO_MIME_TYPE[format]
+  }
+}