From 66c7617e22bf18baafbc2bed06728dce89d3b1a3 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 15 Feb 2024 19:15:27 +0000 Subject: [PATCH 01/10] feat: handle Accept header for raw types in @helia/verified-fetch Let users get raw data back from CIDs that would otherwise trigger decoding as JSON or CBOR etc by specifying an `Accept` header. ```typescript const res = await verifiedFetch(cid, { headers: { accept: 'application/octet-stream' } }) console.info(res.headers.get('accept')) // application/octet-stream ``` Make sure the content-type matches the accept header: ```typescript const res = await verifiedFetch(cid, { headers: { accept: 'application/vnd.ipld.raw' } }) console.info(res.headers.get('accept')) // application/vnd.ipld.raw ``` Support multiple values, match the first one: ```typescript const res = await verifiedFetch(cid, { headers: { accept: 'application/vnd.ipld.raw, application/octet-stream, */*' } }) console.info(res.headers.get('accept')) // application/vnd.ipld.raw ``` If they specify an Accept header we can't honor, return a [406](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/406): ```typescript const res = await verifiedFetch(cid, { headers: { accept: 'application/what-even-is-this' } }) console.info(res.status) // 406 ``` --- .../verified-fetch/src/utils/get-format.ts | 118 ++++++++++++++ packages/verified-fetch/src/verified-fetch.ts | 148 ++++++++++++------ .../verified-fetch/test/accept-header.spec.ts | 96 ++++++++++++ packages/verified-fetch/test/fixtures/cids.ts | 6 + .../test/utils/get-format.spec.ts | 66 ++++++++ .../test/verified-fetch.spec.ts | 106 +++++++++++-- 6 files changed, 480 insertions(+), 60 deletions(-) create mode 100644 packages/verified-fetch/src/utils/get-format.ts create mode 100644 packages/verified-fetch/test/accept-header.spec.ts create mode 100644 packages/verified-fetch/test/fixtures/cids.ts create mode 100644 packages/verified-fetch/test/utils/get-format.spec.ts diff --git a/packages/verified-fetch/src/utils/get-format.ts b/packages/verified-fetch/src/utils/get-format.ts new file mode 100644 index 000000000..6e2c149bc --- /dev/null +++ b/packages/verified-fetch/src/utils/get-format.ts @@ -0,0 +1,118 @@ +import { code as dagCborCode } from '@ipld/dag-cbor' +import { code as dagJsonCode } from '@ipld/dag-json' +import type { CID } from 'multiformats/cid' + +export type FORMAT = 'raw' | 'car' | 'dag-json' | 'dag-cbor' | 'json' | 'cbor' | 'ipns-record' | 'tar' + +const FORMATS: string[] = [ + 'raw', 'car', 'dag-json', 'dag-cbor', 'json', 'cbor', 'ipns-record', 'tar' +] + +function isSupportedFormat (format: string): format is FORMAT { + return FORMATS.includes(format) +} + +const FORMAT_MAP: Record = { + // https://www.iana.org/assignments/media-types/application/vnd.ipld.raw + 'application/vnd.ipld.raw': 'raw', + 'application/octet-stream': 'raw', + + // https://www.iana.org/assignments/media-types/application/vnd.ipld.car + 'application/vnd.ipld.car': 'car', + + // https://www.iana.org/assignments/media-types/application/vnd.ipld.dag-json + 'application/vnd.ipld.dag-json': 'dag-json', + + // https://www.iana.org/assignments/media-types/application/vnd.ipld.dag-cbor + 'application/vnd.ipld.dag-cbor': 'dag-cbor', + 'application/json': 'json', + 'application/cbor': 'cbor', + + // https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record + 'application/vnd.ipfs.ipns-record': 'ipns-record', + 'application/x-tar': 'tar' +} + +const MIME_TYPE_MAP: Record = { + raw: 'application/octet-stream', + car: 'application/vnd.ipld.car', + 'dag-json': 'application/vnd.ipld.dag-json', + 'dag-cbor': 'application/vnd.ipld.dag-cbor', + json: 'application/json', + cbor: 'application/cbor', + 'ipns-record': 'application/vnd.ipfs.ipns-record', + tar: 'application/x-tar' +} + +interface UserFormat { + format: FORMAT + mimeType: string +} + +/** + * Determines the format requested by the client either by an `Accept` header or + * a `format` query string arg. + * + * @see https://specs.ipfs.tech/http-gateways/path-gateway/#format-request-query-parameter + */ +export function getFormat ({ cid, headerFormat, queryFormat }: { cid: CID, headerFormat?: string | null, queryFormat?: string | null }): UserFormat | undefined { + let output: UserFormat | undefined + + if (headerFormat != null) { + output = getFormatFromHeader(headerFormat) + } else if (queryFormat != null && isSupportedFormat(queryFormat)) { + output = { + format: queryFormat, + mimeType: MIME_TYPE_MAP[queryFormat] + } + } + + // special case: if the CID is dag-json or dag-cbor but we'd use the regular + // json handler, use the dag-json/dag-cbor one instead but retain the + // application/json mime type - the requested mime type will be passed through + // to the handler which will ensure the decoded object can actually be + // represented as plain JSON + if (output?.mimeType === 'application/json') { + if (cid.code === dagCborCode) { + output.format = 'dag-cbor' + } + + if (cid.code === dagJsonCode) { + output.format = 'dag-json' + } + } + + return output +} + +/** + * Match one of potentially many `Accept` header values or wildcards + */ +function getFormatFromHeader (accept: string): UserFormat | undefined { + const headerFormats = accept + .split(',') + .map(s => s.split(';')[0]) + .map(s => s.trim()) + .sort() + + let foundWildcard = false + + for (const [mimeType, format] of Object.entries(FORMAT_MAP)) { + for (const headerFormat of headerFormats) { + if (headerFormat.includes(mimeType)) { + return { format, mimeType } + } + + if (headerFormat.startsWith('*/') || headerFormat.endsWith('/*')) { + foundWildcard = true + } + } + } + + if (foundWildcard) { + return { + format: 'raw', + mimeType: '*/*' + } + } +} diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index a8894ecf3..778ec5139 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -9,6 +9,7 @@ import { code as rawCode } from 'multiformats/codecs/raw' import { identity } from 'multiformats/hashes/identity' import { CustomProgressEvent } from 'progress-events' import { dagCborToSafeJSON } from './utils/dag-cbor-to-safe-json.js' +import { getFormat } from './utils/get-format.js' import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js' import { parseResource } from './utils/parse-resource.js' import { walkPath, type PathWalkerFn } from './utils/walk-path.js' @@ -37,6 +38,12 @@ interface FetchHandlerFunctionArg { path: string terminalElement?: UnixFSEntry options?: Omit & AbortOptions + + /** + * If present, the user has sent an accept header with this value - if the + * content cannot be represented in this format a 406 should be returned + */ + accept?: string } interface FetchHandlerFunction { @@ -72,6 +79,47 @@ function notSupportedResponse (body?: BodyInit | null): Response { }) } +function notAcceptableResponse (body?: BodyInit | null): Response { + return new Response(body, { + status: 406, + statusText: '406 Not Acceptable' + }) +} + +/** + * These are Accept header values that will cause content type sniffing to be + * skipped and set to these values. + */ +const RAW_HEADERS = [ + 'application/vnd.ipld.raw', + 'application/octet-stream' +] + +/** + * if the user has specified an `Accept` header, and it's in our list of + * allowable "raw" format headers, use that instead of detecting the content + * type, to avoid the user signalling that they will Accepting one mime type + * and then receiving something different. + */ +function getOverridenRawContentType (headers?: HeadersInit): string | undefined { + const acceptHeader = new Headers(headers).get('accept') ?? '' + + // e.g. "Accept: text/html, application/xhtml+xml, application/xml;q=0.9, image/webp, */*;q=0.8" + const acceptHeaders = acceptHeader.split(',') + .map(s => s.split(';')[0]) + .map(s => s.trim()) + + for (const mimeType of acceptHeaders) { + if (mimeType === '*/*') { + return + } + + if (RAW_HEADERS.includes(mimeType ?? '')) { + return mimeType + } + } +} + export class VerifiedFetch { private readonly helia: Helia private readonly ipns: IPNS @@ -112,17 +160,17 @@ export class VerifiedFetch { private async handleJson ({ cid, path, options }: FetchHandlerFunctionArg): Promise { this.log.trace('fetching %c/%s', cid, path) options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { cid, path })) - const result = await this.helia.blockstore.get(cid, { + const block = await this.helia.blockstore.get(cid, { signal: options?.signal, onProgress: options?.onProgress }) - const response = okResponse(result) + const response = okResponse(block) response.headers.set('content-type', 'application/json') options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid, path })) return response } - private async handleDagCbor ({ cid, path, options }: FetchHandlerFunctionArg): Promise { + private async handleDagCbor ({ cid, path, accept, options }: FetchHandlerFunctionArg): Promise { this.log.trace('fetching %c/%s', cid, path) options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { cid, path })) // return body as binary @@ -132,6 +180,12 @@ export class VerifiedFetch { try { body = dagCborToSafeJSON(block) } catch (err) { + if (accept === 'application/json') { + this.log('could not decode DAG-CBOR as JSON-safe, but the client sent "Accept: application/json"', err) + + return notAcceptableResponse() + } + this.log('could not decode DAG-CBOR as JSON-safe, falling back to `application/octet-stream`', err) body = block } @@ -193,7 +247,16 @@ export class VerifiedFetch { options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { cid, path })) const result = await this.helia.blockstore.get(cid) const response = okResponse(result) - await this.setContentType(result, path, response) + + // if the user has specified an `Accept` header that corresponds to a raw + // type, honour that header, so they don't request `vnd.ipld.raw` and get + // `octet-stream` or vice versa + const overriddenContentType = getOverridenRawContentType(options?.headers) + if (overriddenContentType != null) { + response.headers.set('content-type', overriddenContentType) + } else { + await this.setContentType(result, path, response) + } options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid, path })) return response @@ -225,52 +288,27 @@ export class VerifiedFetch { response.headers.set('content-type', contentType) } - /** - * Determines the format requested by the client, defaults to `null` if no format is requested. - * - * @see https://specs.ipfs.tech/http-gateways/path-gateway/#format-request-query-parameter - * @default 'raw' - */ - private getFormat ({ headerFormat, queryFormat }: { headerFormat: string | null, queryFormat: string | null }): string | null { - const formatMap: Record = { - 'vnd.ipld.raw': 'raw', - 'vnd.ipld.car': 'car', - 'application/x-tar': 'tar', - 'application/vnd.ipld.dag-json': 'dag-json', - 'application/vnd.ipld.dag-cbor': 'dag-cbor', - 'application/json': 'json', - 'application/cbor': 'cbor', - 'vnd.ipfs.ipns-record': 'ipns-record' - } - - if (headerFormat != null) { - for (const format in formatMap) { - if (headerFormat.includes(format)) { - return formatMap[format] - } - } - } else if (queryFormat != null) { - return queryFormat - } - - return null - } - /** * Map of format to specific handlers for that format. - * These format handlers should adjust the response headers as specified in https://specs.ipfs.tech/http-gateways/path-gateway/#response-headers + * + * These format handlers should adjust the response headers as specified in + * https://specs.ipfs.tech/http-gateways/path-gateway/#response-headers */ private readonly formatHandlers: Record = { - raw: async () => notSupportedResponse('application/vnd.ipld.raw support is not implemented'), + raw: this.handleRaw, car: this.handleIPLDCar, 'ipns-record': this.handleIPNSRecord, tar: async () => notSupportedResponse('application/x-tar support is not implemented'), - 'dag-json': async () => notSupportedResponse('application/vnd.ipld.dag-json support is not implemented'), - 'dag-cbor': async () => notSupportedResponse('application/vnd.ipld.dag-cbor support is not implemented'), - json: async () => notSupportedResponse('application/json support is not implemented'), - cbor: async () => notSupportedResponse('application/cbor support is not implemented') + 'dag-json': this.handleJson, + 'dag-cbor': this.handleDagCbor, + json: this.handleJson, + cbor: this.handleDagCbor } + /** + * If the user has not specified an Accept header or format query string arg, + * use the CID codec to choose an appropriate handler for the block data. + */ private readonly codecHandlers: Record = { [dagPbCode]: this.handleDagPb, [dagJsonCode]: this.handleJson, @@ -281,19 +319,32 @@ export class VerifiedFetch { } async fetch (resource: Resource, opts?: VerifiedFetchOptions): Promise { + this.log('fetch', resource) + const options = convertOptions(opts) const { path, query, ...rest } = await parseResource(resource, { ipns: this.ipns, logger: this.helia.logger }, options) const cid = rest.cid let response: Response | undefined - const format = this.getFormat({ headerFormat: new Headers(options?.headers).get('accept'), queryFormat: query.format ?? null }) + const acceptHeader = new Headers(options?.headers).get('accept') + this.log('accept header %s', acceptHeader) + + const format = getFormat({ cid, headerFormat: acceptHeader, queryFormat: query.format ?? null }) + this.log('format %s, mime type %s', format?.format, format?.mimeType) + + if (format == null && acceptHeader != null) { + this.log('no format found for accept header %s', acceptHeader) + + // user specified an Accept header but we had no handler for it + return notAcceptableResponse() + } if (format != null) { // TODO: These should be handled last when they're returning something other than 501 - const formatHandler = this.formatHandlers[format] + const formatHandler = this.formatHandlers[format.format] if (formatHandler != null) { - response = await formatHandler.call(this, { cid, path, options }) + response = await formatHandler.call(this, { cid, path, accept: format.mimeType, options }) if (response.status === 501) { return response @@ -323,6 +374,15 @@ export class VerifiedFetch { } } + const contentType = response.headers.get('content-type') + + if (format != null && format.mimeType !== '*/*' && contentType !== format.mimeType) { + // the user requested a specific, non-wildcard representation type, but + // the data cannot be represented as that type so return a + // "Not Acceptable" response + return notAcceptableResponse() + } + response.headers.set('etag', cid.toString()) // https://specs.ipfs.tech/http-gateways/path-gateway/#etag-response-header response.headers.set('cache-control', 'public, max-age=29030400, immutable') response.headers.set('X-Ipfs-Path', resource.toString()) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-path-response-header diff --git a/packages/verified-fetch/test/accept-header.spec.ts b/packages/verified-fetch/test/accept-header.spec.ts new file mode 100644 index 000000000..d08949b32 --- /dev/null +++ b/packages/verified-fetch/test/accept-header.spec.ts @@ -0,0 +1,96 @@ +/* eslint-env mocha */ +import { dagCbor } from '@helia/dag-cbor' +import * as ipldDagCbor from '@ipld/dag-cbor' +import { stop } from '@libp2p/interface' +import { expect } from 'aegir/chai' +import { VerifiedFetch } from '../src/verified-fetch.js' +import { createHelia } from './fixtures/create-offline-helia.js' +import type { Helia } from '@helia/interface' + +describe('accept header', () => { + let helia: Helia + let verifiedFetch: VerifiedFetch + + beforeEach(async () => { + helia = await createHelia() + verifiedFetch = new VerifiedFetch({ + helia + }) + }) + + afterEach(async () => { + await stop(helia, verifiedFetch) + }) + + it('should allow specifying application/vnd.ipld.raw accept header to skip data decoding', async () => { + // JSON-compliant CBOR - if decoded would otherwise cause `Content-Type` to + // be set to `application/json` + const obj = { + hello: 'world' + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const resp = await verifiedFetch.fetch(cid, { + headers: { + accept: 'application/vnd.ipld.raw' + } + }) + expect(resp.headers.get('content-type')).to.equal('application/vnd.ipld.raw') + + const output = ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) + expect(output).to.deep.equal(obj) + }) + + it('should allow specifying application/octet-stream accept header to skip data decoding', async () => { + // JSON-compliant CBOR - if decoded would otherwise cause `Content-Type` to + // be set to `application/json` + const obj = { + hello: 'world' + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const resp = await verifiedFetch.fetch(cid, { + headers: { + accept: 'application/octet-stream' + } + }) + expect(resp.headers.get('content-type')).to.equal('application/octet-stream') + + const output = ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) + expect(output).to.deep.equal(obj) + }) + + it('should return 406 Not Acceptable if the accept header cannot be honoured', async () => { + const obj = { + hello: 'world' + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const resp = await verifiedFetch.fetch(cid, { + headers: { + accept: 'application/what-even-is-this' + } + }) + expect(resp.status).to.equal(406) + expect(resp.statusText).to.equal('406 Not Acceptable') + }) + + it('should suuport wildcards', async () => { + const obj = { + hello: 'world' + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const resp = await verifiedFetch.fetch(cid, { + headers: { + accept: 'application/what-even-is-this, */*' + } + }) + expect(resp.status).to.equal(200) + expect(resp.headers.get('content-type')).to.equal('application/octet-stream') + }) +}) diff --git a/packages/verified-fetch/test/fixtures/cids.ts b/packages/verified-fetch/test/fixtures/cids.ts new file mode 100644 index 000000000..dc4d73765 --- /dev/null +++ b/packages/verified-fetch/test/fixtures/cids.ts @@ -0,0 +1,6 @@ +import { CID } from 'multiformats/cid' + +export const cids: Record = { + file: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), + dagCbor: CID.parse('bafyreicnokmhmrnlp2wjhyk2haep4tqxiptwfrp2rrs7rzq7uk766chqvq') +} diff --git a/packages/verified-fetch/test/utils/get-format.spec.ts b/packages/verified-fetch/test/utils/get-format.spec.ts new file mode 100644 index 000000000..ac4e74e37 --- /dev/null +++ b/packages/verified-fetch/test/utils/get-format.spec.ts @@ -0,0 +1,66 @@ +import { expect } from 'aegir/chai' +import { getFormat } from '../../src/utils/get-format.js' +import { cids } from '../fixtures/cids.js' + +describe('get-format', () => { + it('should override query format with Accept header if available', () => { + const format = getFormat({ + cid: cids.file, + headerFormat: 'application/vnd.ipld.car', + queryFormat: 'raw' + }) + + expect(format).to.have.property('format', 'car') + expect(format).to.have.property('mimeType', 'application/vnd.ipld.car') + }) + + it('should default wildcards to raw format', () => { + const format = getFormat({ + cid: cids.file, + headerFormat: '*/*' + }) + + expect(format).to.have.property('format', 'raw') + expect(format).to.have.property('mimeType', '*/*') + }) + + it('should use specific type before wildcard', () => { + const format = getFormat({ + cid: cids.file, + headerFormat: '*/*, application/x-tar' + }) + + expect(format).to.have.property('format', 'tar') + expect(format).to.have.property('mimeType', 'application/x-tar') + }) + + it('should use specific type before wildcard', () => { + const format = getFormat({ + cid: cids.file, + headerFormat: 'application/x-tar, */*' + }) + + expect(format).to.have.property('format', 'tar') + expect(format).to.have.property('mimeType', 'application/x-tar') + }) + + it('should support partial wildcard', () => { + const format = getFormat({ + cid: cids.file, + headerFormat: 'application/*' + }) + + expect(format).to.have.property('format', 'raw') + expect(format).to.have.property('mimeType', '*/*') + }) + + it('should support partial wildcard', () => { + const format = getFormat({ + cid: cids.file, + headerFormat: '*/' + }) + + expect(format).to.have.property('format', 'raw') + expect(format).to.have.property('mimeType', '*/*') + }) +}) diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index d495c0313..304c3a1d2 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -20,11 +20,9 @@ import Sinon from 'sinon' import { stubInterface } from 'sinon-ts' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { VerifiedFetch } from '../src/verified-fetch.js' +import { cids } from './fixtures/cids.js' import { createHelia } from './fixtures/create-offline-helia.js' import type { Helia } from '@helia/interface' -import type { Logger, ComponentLogger } from '@libp2p/interface' - -const testCID = CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') describe('@helia/verifed-fetch', () => { let helia: Helia @@ -63,15 +61,13 @@ describe('@helia/verifed-fetch', () => { before(async () => { verifiedFetch = new VerifiedFetch({ helia: stubInterface({ - logger: stubInterface({ - forComponent: () => stubInterface() - }) + logger: defaultLogger() }), ipns: stubInterface({ resolveDns: async (dnsLink: string) => { expect(dnsLink).to.equal('mydomain.com') return { - cid: testCID, + cid: cids.file, path: '' } } @@ -85,14 +81,7 @@ describe('@helia/verifed-fetch', () => { }) const formatsAndAcceptHeaders = [ - ['raw', 'application/vnd.ipld.raw'], - ['car', 'application/vnd.ipld.car'], - ['tar', 'application/x-tar'], - ['dag-json', 'application/vnd.ipld.dag-json'], - ['dag-cbor', 'application/vnd.ipld.dag-cbor'], - ['json', 'application/json'], - ['cbor', 'application/cbor'], - ['ipns-record', 'application/vnd.ipfs.ipns-record'] + ['tar', 'application/x-tar'] ] for (const [format, acceptHeader] of formatsAndAcceptHeaders) { @@ -101,7 +90,7 @@ describe('@helia/verifed-fetch', () => { const resp = await verifiedFetch.fetch(`ipns://mydomain.com?format=${format}`) expect(resp).to.be.ok() expect(resp.status).to.equal(501) - const resp2 = await verifiedFetch.fetch(testCID, { + const resp2 = await verifiedFetch.fetch(cids.file, { headers: { accept: acceptHeader } @@ -523,4 +512,89 @@ describe('@helia/verifed-fetch', () => { await expect(resp.text()).to.eventually.equal('hello world') }) }) + + describe('accept', () => { + let helia: Helia + let verifiedFetch: VerifiedFetch + let contentTypeParser: Sinon.SinonStub + + beforeEach(async () => { + contentTypeParser = Sinon.stub() + helia = await createHelia() + verifiedFetch = new VerifiedFetch({ + helia + }, { + contentTypeParser + }) + }) + + afterEach(async () => { + await stop(helia, verifiedFetch) + }) + + it('should allow specifying an accept header', async () => { + const obj = { + hello: 'world' + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const resp = await verifiedFetch.fetch(cid, { + headers: { + accept: 'application/octet-stream' + } + }) + expect(resp.headers.get('content-type')).to.equal('application/octet-stream') + const output = ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) + expect(output).to.deep.equal(obj) + }) + + it('should return a 406 if the content cannot be represented by the mime type in the accept header', async () => { + const obj = { + hello: 'world', + // fails to parse as JSON + link: CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN') + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const resp = await verifiedFetch.fetch(cid, { + headers: { + accept: 'application/json' + } + }) + expect(resp.status).to.equal(406) + }) + + it('should return a 406 if the content type parser returns a different value to the accept header', async () => { + contentTypeParser.returns('text/plain') + + const fs = unixfs(helia) + const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3, 4])) + + const resp = await verifiedFetch.fetch(cid, { + headers: { + accept: 'image/jpeg' + } + }) + expect(resp.status).to.equal(406) + }) + + it('should allow specifying an accept as raw', async () => { + const obj = { + hello: 'world' + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const resp = await verifiedFetch.fetch(cid, { + headers: { + accept: 'application/vnd.ipld.raw' + } + }) + expect(resp.headers.get('content-type')).to.equal('application/vnd.ipld.raw') + const output = ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) + expect(output).to.deep.equal(obj) + }) + }) }) From 67ddf99a5fae0a9d97f062f1550625160f8e7240 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 16 Feb 2024 16:28:15 +0000 Subject: [PATCH 02/10] chore: add docs --- packages/verified-fetch/README.md | 33 ++++++++++++++++++++++++++++ packages/verified-fetch/src/index.ts | 33 ++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/packages/verified-fetch/README.md b/packages/verified-fetch/README.md index b355422ce..40e78641f 100644 --- a/packages/verified-fetch/README.md +++ b/packages/verified-fetch/README.md @@ -347,6 +347,39 @@ if (res.headers.get('Content-Type') === 'application/json') { console.info(obj) // ... ``` +## The `Accept` header + +The `Accept` header can be passed to override certain response processing, or to ensure that the final `Content-Type` of the response is the one that is expected. + +If the final `Content-Type` does not match the `Accept` header, or if the content cannot be represented in the format dictated by the `Accept` header, or you have configured a custom content type parser, and that parser returns a value that isn't in the accept header, a [406: Not Acceptible](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/406) response will be returned: + +```typescript +import { verifiedFetch } from '@helia/verified-fetch' + +const res = await verifiedFetch('ipfs://bafyJPEGImageCID', { + headers: { + accept: 'image/png' + } +}) + +console.info(res.status) // 406 - the image was a JPEG but we specified PNG as the accept header +``` + +It can also be used to skip processing the data from some formats such as `DAG-CBOR` if you wish to handle decoding it yourself: + +```typescript +import { verifiedFetch } from '@helia/verified-fetch' + +const res = await verifiedFetch('ipfs://bafyDAGCBORCID', { + headers: { + accept: 'application/octet-stream' + } +}) + +console.info(res.headers.get('accept')) // application/octet-stream +const buf = await res.arrayBuffer() // raw bytes, not processed as JSON +``` + ## Comparison to fetch This module attempts to act as similarly to the `fetch()` API as possible. diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index 786f25735..d5f9c6fbb 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -320,6 +320,39 @@ * console.info(obj) // ... * ``` * + * ## The `Accept` header + * + * The `Accept` header can be passed to override certain response processing, or to ensure that the final `Content-Type` of the response is the one that is expected. + * + * If the final `Content-Type` does not match the `Accept` header, or if the content cannot be represented in the format dictated by the `Accept` header, or you have configured a custom content type parser, and that parser returns a value that isn't in the accept header, a [406: Not Acceptible](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/406) response will be returned: + * + * ```typescript + * import { verifiedFetch } from '@helia/verified-fetch' + * + * const res = await verifiedFetch('ipfs://bafyJPEGImageCID', { + * headers: { + * accept: 'image/png' + * } + * }) + * + * console.info(res.status) // 406 - the image was a JPEG but we specified PNG as the accept header + * ``` + * + * It can also be used to skip processing the data from some formats such as `DAG-CBOR` if you wish to handle decoding it yourself: + * + * ```typescript + * import { verifiedFetch } from '@helia/verified-fetch' + * + * const res = await verifiedFetch('ipfs://bafyDAGCBORCID', { + * headers: { + * accept: 'application/octet-stream' + * } + * }) + * + * console.info(res.headers.get('accept')) // application/octet-stream + * const buf = await res.arrayBuffer() // raw bytes, not processed as JSON + * ``` + * * ## Comparison to fetch * * This module attempts to act as similarly to the `fetch()` API as possible. From 19b04a6815ba3c15c64800445e2c3b2d1fcfed94 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 20 Feb 2024 13:19:25 +0000 Subject: [PATCH 03/10] feat: convert between types --- packages/verified-fetch/src/index.ts | 4 + .../utils/get-content-disposition-filename.ts | 18 ++ .../verified-fetch/src/utils/get-format.ts | 117 ------- .../src/utils/parse-url-string.ts | 12 +- .../src/utils/select-output-type.ts | 163 ++++++++++ packages/verified-fetch/src/verified-fetch.ts | 297 +++++++++++------- .../verified-fetch/test/accept-header.spec.ts | 184 ++++++++++- packages/verified-fetch/test/fixtures/cids.ts | 16 +- .../test/fixtures/memory-car.ts | 33 ++ packages/verified-fetch/test/index.spec.ts | 1 - .../get-content-disposition-filename.spec.ts | 16 + .../test/utils/get-format.spec.ts | 66 ---- .../test/utils/select-output-type.spec.ts | 41 +++ .../test/verified-fetch.spec.ts | 12 +- 14 files changed, 666 insertions(+), 314 deletions(-) create mode 100644 packages/verified-fetch/src/utils/get-content-disposition-filename.ts delete mode 100644 packages/verified-fetch/src/utils/get-format.ts create mode 100644 packages/verified-fetch/src/utils/select-output-type.ts create mode 100644 packages/verified-fetch/test/fixtures/memory-car.ts create mode 100644 packages/verified-fetch/test/utils/get-content-disposition-filename.spec.ts delete mode 100644 packages/verified-fetch/test/utils/get-format.spec.ts create mode 100644 packages/verified-fetch/test/utils/select-output-type.spec.ts diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index d5f9c6fbb..222578f83 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -482,6 +482,10 @@ import type { ProgressEvent, ProgressOptions } from 'progress-events' */ export type Resource = string | CID +export interface ResourceDetail { + resource: Resource +} + export interface CIDDetail { cid: CID path: string diff --git a/packages/verified-fetch/src/utils/get-content-disposition-filename.ts b/packages/verified-fetch/src/utils/get-content-disposition-filename.ts new file mode 100644 index 000000000..cee3ee1f2 --- /dev/null +++ b/packages/verified-fetch/src/utils/get-content-disposition-filename.ts @@ -0,0 +1,18 @@ +/** + * Takes a filename URL param and returns a string for use in a + * `Content-Disposition` header + */ +export function getContentDispositionFilename (filename: string): string { + const asciiOnly = replaceNonAsciiCharacters(filename) + + if (asciiOnly === filename) { + return `filename="${filename}"` + } + + return `filename="${asciiOnly}"; filename*=UTF-8''${encodeURIComponent(filename)}` +} + +function replaceNonAsciiCharacters (filename: string): string { + // eslint-disable-next-line no-control-regex + return filename.replace(/[^\x00-\x7F]/g, '_') +} diff --git a/packages/verified-fetch/src/utils/get-format.ts b/packages/verified-fetch/src/utils/get-format.ts deleted file mode 100644 index 64ab56597..000000000 --- a/packages/verified-fetch/src/utils/get-format.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { code as dagCborCode } from '@ipld/dag-cbor' -import { code as dagJsonCode } from '@ipld/dag-json' -import type { RequestFormatShorthand } from '../types.js' -import type { CID } from 'multiformats/cid' - -const FORMATS: string[] = [ - 'raw', 'car', 'dag-json', 'dag-cbor', 'json', 'cbor', 'ipns-record', 'tar' -] - -function isSupportedFormat (format: string): format is RequestFormatShorthand { - return FORMATS.includes(format) -} - -const FORMAT_MAP: Record = { - // https://www.iana.org/assignments/media-types/application/vnd.ipld.raw - 'application/vnd.ipld.raw': 'raw', - 'application/octet-stream': 'raw', - - // https://www.iana.org/assignments/media-types/application/vnd.ipld.car - 'application/vnd.ipld.car': 'car', - - // https://www.iana.org/assignments/media-types/application/vnd.ipld.dag-json - 'application/vnd.ipld.dag-json': 'dag-json', - - // https://www.iana.org/assignments/media-types/application/vnd.ipld.dag-cbor - 'application/vnd.ipld.dag-cbor': 'dag-cbor', - 'application/json': 'json', - 'application/cbor': 'cbor', - - // https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record - 'application/vnd.ipfs.ipns-record': 'ipns-record', - 'application/x-tar': 'tar' -} - -const MIME_TYPE_MAP: Record = { - raw: 'application/octet-stream', - car: 'application/vnd.ipld.car', - 'dag-json': 'application/vnd.ipld.dag-json', - 'dag-cbor': 'application/vnd.ipld.dag-cbor', - json: 'application/json', - cbor: 'application/cbor', - 'ipns-record': 'application/vnd.ipfs.ipns-record', - tar: 'application/x-tar' -} - -interface UserFormat { - format: RequestFormatShorthand - mimeType: string -} - -/** - * Determines the format requested by the client either by an `Accept` header or - * a `format` query string arg. - * - * @see https://specs.ipfs.tech/http-gateways/path-gateway/#format-request-query-parameter - */ -export function getFormat ({ cid, headerFormat, queryFormat }: { cid: CID, headerFormat?: string | null, queryFormat?: string | null }): UserFormat | undefined { - let output: UserFormat | undefined - - if (headerFormat != null) { - output = getFormatFromHeader(headerFormat) - } else if (queryFormat != null && isSupportedFormat(queryFormat)) { - output = { - format: queryFormat, - mimeType: MIME_TYPE_MAP[queryFormat] - } - } - - // special case: if the CID is dag-json or dag-cbor but we'd use the regular - // json handler, use the dag-json/dag-cbor one instead but retain the - // application/json mime type - the requested mime type will be passed through - // to the handler which will ensure the decoded object can actually be - // represented as plain JSON - if (output?.mimeType === 'application/json') { - if (cid.code === dagCborCode) { - output.format = 'dag-cbor' - } - - if (cid.code === dagJsonCode) { - output.format = 'dag-json' - } - } - - return output -} - -/** - * Match one of potentially many `Accept` header values or wildcards - */ -function getFormatFromHeader (accept: string): UserFormat | undefined { - const headerFormats = accept - .split(',') - .map(s => s.split(';')[0]) - .map(s => s.trim()) - .sort() - - let foundWildcard = false - - for (const [mimeType, format] of Object.entries(FORMAT_MAP)) { - for (const headerFormat of headerFormats) { - if (headerFormat.includes(mimeType)) { - return { format, mimeType } - } - - if (headerFormat.startsWith('*/') || headerFormat.endsWith('/*')) { - foundWildcard = true - } - } - } - - if (foundWildcard) { - return { - format: 'raw', - mimeType: '*/*' - } - } -} diff --git a/packages/verified-fetch/src/utils/parse-url-string.ts b/packages/verified-fetch/src/utils/parse-url-string.ts index 4c0789734..d869f77c6 100644 --- a/packages/verified-fetch/src/utils/parse-url-string.ts +++ b/packages/verified-fetch/src/utils/parse-url-string.ts @@ -19,6 +19,8 @@ export interface ParseUrlStringOptions extends ProgressOptions { format?: RequestFormatShorthand + download?: boolean + filename?: string } export interface ParsedUrlStringResults { @@ -109,7 +111,7 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin } // parse query string - const query: Record = {} + const query: Record = {} if (queryString != null && queryString.length > 0) { const queryParts = queryString.split('&') @@ -117,6 +119,14 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin const [key, value] = part.split('=') query[key] = decodeURIComponent(value) } + + if (query.download != null) { + query.download = query.download === 'true' + } + + if (query.filename != null) { + query.filename = query.filename.toString() + } } /** diff --git a/packages/verified-fetch/src/utils/select-output-type.ts b/packages/verified-fetch/src/utils/select-output-type.ts new file mode 100644 index 000000000..3ec9af68c --- /dev/null +++ b/packages/verified-fetch/src/utils/select-output-type.ts @@ -0,0 +1,163 @@ +import { code as dagCborCode } from '@ipld/dag-cbor' +import { code as dagJsonCode } from '@ipld/dag-json' +import { code as dagPbCode } from '@ipld/dag-pb' +import { code as jsonCode } from 'multiformats/codecs/json' +import { code as rawCode } from 'multiformats/codecs/raw' +import type { RequestFormatShorthand } from '../types.js' +import type { CID } from 'multiformats/cid' + +const CID_TYPE_MAP: Record = { + [dagCborCode]: [ + 'application/json', + 'application/vnd.ipld.dag-cbor', + 'application/cbor', + 'application/vnd.ipld.dag-json', + 'application/octet-stream', + 'application/vnd.ipld.raw', + 'application/vnd.ipfs.ipns-record', + 'application/vnd.ipld.car' + ], + [dagJsonCode]: [ + 'application/json', + 'application/vnd.ipld.dag-cbor', + 'application/cbor', + 'application/vnd.ipld.dag-json', + 'application/octet-stream', + 'application/vnd.ipld.raw', + 'application/vnd.ipfs.ipns-record', + 'application/vnd.ipld.car' + ], + [jsonCode]: [ + 'application/json', + 'application/vnd.ipld.dag-cbor', + 'application/cbor', + 'application/vnd.ipld.dag-json', + 'application/octet-stream', + 'application/vnd.ipld.raw', + 'application/vnd.ipfs.ipns-record', + 'application/vnd.ipld.car' + ], + [dagPbCode]: [ + 'application/octet-stream', + 'application/json', + 'application/vnd.ipld.dag-cbor', + 'application/cbor', + 'application/vnd.ipld.dag-json', + 'application/vnd.ipld.raw', + 'application/vnd.ipfs.ipns-record', + 'application/vnd.ipld.car', + 'application/x-tar' + ], + [rawCode]: [ + 'application/octet-stream', + 'application/vnd.ipld.raw', + 'application/vnd.ipfs.ipns-record', + 'application/vnd.ipld.car' + ] +} + +/** + * Selects an output mime-type based on the CID and a passed `Accept` header + */ +export function selectOutputType (cid: CID, accept?: string): string | undefined { + const cidMimeTypes = CID_TYPE_MAP[cid.code] + + if (accept != null) { + return chooseMimeType(accept, cidMimeTypes) + } +} + +function chooseMimeType (accept: string, validMimeTypes: string[]): string | undefined { + const requestedMimeTypes = accept + .split(',') + .map(s => { + const parts = s.trim().split(';') + + return { + mimeType: `${parts[0]}`.trim(), + weight: parseQFactor(parts[1]) + } + }) + .sort((a, b) => { + if (a.weight === b.weight) { + return 0 + } + + if (a.weight > b.weight) { + return -1 + } + + return 1 + }) + .map(s => s.mimeType) + + for (const headerFormat of requestedMimeTypes) { + for (const mimeType of validMimeTypes) { + if (headerFormat.includes(mimeType)) { + return mimeType + } + + if (headerFormat === '*/*') { + return mimeType + } + + if (headerFormat.startsWith('*/') && mimeType.split('/')[1] === headerFormat.split('/')[1]) { + return mimeType + } + + if (headerFormat.endsWith('/*') && mimeType.split('/')[0] === headerFormat.split('/')[0]) { + return mimeType + } + } + } +} + +/** + * Parses q-factor weighting from the accept header to allow letting some mime + * types take precedence over others. + * + * If the q-factor for an acceptable mime representation is omitted it defaults + * to `1`. + * + * All specified values should be in the range 0-1. + * + * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept#q + */ +function parseQFactor (str?: string): number { + if (str != null) { + str = str.trim() + } + + if (str == null || !str.startsWith('q=')) { + return 1 + } + + const factor = parseFloat(str.replace('q=', '')) + + if (isNaN(factor)) { + return 0 + } + + return factor +} + +const FORMAT_TO_MIME_TYPE: Record = { + raw: 'application/vnd.ipld.raw', + car: 'application/vnd.ipld.car', + 'dag-json': 'application/vnd.ipld.dag-json', + 'dag-cbor': 'application/vnd.ipld.dag-cbor', + json: 'application/json', + cbor: 'application/cbor', + 'ipns-record': 'application/vnd.ipfs.ipns-record', + tar: 'application/x-tar' +} + +/** + * Converts a `format=...` query param to a mime type as would be found in the + * `Accept` header, if a valid mapping is available + */ +export function queryFormatToAcceptHeader (format?: RequestFormatShorthand): string | undefined { + if (format != null) { + return FORMAT_TO_MIME_TYPE[format] + } +} diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index 0f471902e..f31c363c3 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -1,20 +1,22 @@ import { ipns as heliaIpns, type IPNS } from '@helia/ipns' import { dnsJsonOverHttps } from '@helia/ipns/dns-resolvers' import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs, type UnixFSStats } from '@helia/unixfs' -import { code as dagCborCode } from '@ipld/dag-cbor' -import { code as dagJsonCode } from '@ipld/dag-json' +import * as ipldDagCbor from '@ipld/dag-cbor' +import * as ipldDagJson from '@ipld/dag-json' import { code as dagPbCode } from '@ipld/dag-pb' import { code as jsonCode } from 'multiformats/codecs/json' import { code as rawCode } from 'multiformats/codecs/raw' import { identity } from 'multiformats/hashes/identity' import { CustomProgressEvent } from 'progress-events' import { dagCborToSafeJSON } from './utils/dag-cbor-to-safe-json.js' +import { getContentDispositionFilename } from './utils/get-content-disposition-filename.js' import { getETag } from './utils/get-e-tag.js' -import { getFormat } from './utils/get-format.js' import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js' import { parseResource } from './utils/parse-resource.js' -import { walkPath, type PathWalkerFn } from './utils/walk-path.js' +import { selectOutputType, queryFormatToAcceptHeader } from './utils/select-output-type.js' +import { walkPath } from './utils/walk-path.js' import type { CIDDetail, ContentTypeParser, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js' +import type { RequestFormatShorthand } from './types.js' import type { Helia } from '@helia/interface' import type { AbortOptions, Logger } from '@libp2p/interface' import type { UnixFSEntry } from 'ipfs-unixfs-exporter' @@ -24,7 +26,6 @@ interface VerifiedFetchComponents { helia: Helia ipns?: IPNS unixfs?: HeliaUnixFs - pathWalker?: PathWalkerFn } /** @@ -37,7 +38,6 @@ interface VerifiedFetchInit { interface FetchHandlerFunctionArg { cid: CID path: string - terminalElement?: UnixFSEntry options?: Omit & AbortOptions /** @@ -74,10 +74,12 @@ function okResponse (body?: BodyInit | null): Response { } function notSupportedResponse (body?: BodyInit | null): Response { - return new Response(body, { + const response = new Response(body, { status: 501, statusText: 'Not Implemented' }) + response.headers.set('X-Content-Type-Options', 'nosniff') // see https://specs.ipfs.tech/http-gateways/path-gateway/#x-content-type-options-response-header + return response } function notAcceptableResponse (body?: BodyInit | null): Response { @@ -125,11 +127,10 @@ export class VerifiedFetch { private readonly helia: Helia private readonly ipns: IPNS private readonly unixfs: HeliaUnixFs - private readonly pathWalker: PathWalkerFn private readonly log: Logger private readonly contentTypeParser: ContentTypeParser | undefined - constructor ({ helia, ipns, unixfs, pathWalker }: VerifiedFetchComponents, init?: VerifiedFetchInit) { + constructor ({ helia, ipns, unixfs }: VerifiedFetchComponents, init?: VerifiedFetchInit) { this.helia = helia this.log = helia.logger.forComponent('helia:verified-fetch') this.ipns = ipns ?? heliaIpns(helia, { @@ -139,66 +140,123 @@ export class VerifiedFetch { ] }) this.unixfs = unixfs ?? heliaUnixFs(helia) - this.pathWalker = pathWalker ?? walkPath this.contentTypeParser = init?.contentTypeParser this.log.trace('created VerifiedFetch instance') } - // handle vnd.ipfs.ipns-record - private async handleIPNSRecord ({ cid, path, options }: FetchHandlerFunctionArg): Promise { - const response = notSupportedResponse('vnd.ipfs.ipns-record support is not implemented') - response.headers.set('X-Content-Type-Options', 'nosniff') // see https://specs.ipfs.tech/http-gateways/path-gateway/#x-content-type-options-response-header - return response + /** + * Accepts an `ipns://...` URL as a string and returns a `Response` containing + * a raw IPNS record. + */ + private async handleIPNSRecord (resource: string, opts?: VerifiedFetchOptions): Promise { + return notSupportedResponse('vnd.ipfs.ipns-record support is not implemented') } - // handle vnd.ipld.car - private async handleIPLDCar ({ cid, path, options }: FetchHandlerFunctionArg): Promise { - const response = notSupportedResponse('vnd.ipld.car support is not implemented') - response.headers.set('X-Content-Type-Options', 'nosniff') // see https://specs.ipfs.tech/http-gateways/path-gateway/#x-content-type-options-response-header - return response + /** + * Accepts a `CID` and returns a `Response` with a body stream that is a CAR + * of the `DAG` referenced by the `CID`. + */ + private async handleCar ({ cid, path, options }: FetchHandlerFunctionArg): Promise { + return notSupportedResponse('vnd.ipld.car support is not implemented') + } + + /** + * Accepts a UnixFS `CID` and returns a `.tar` file containing the file or + * directory structure referenced by the `CID`. + */ + private async handleTar ({ cid, path, options }: FetchHandlerFunctionArg): Promise { + if (cid.code !== dagPbCode) { + return notAcceptableResponse('only dag-pb CIDs can be returned in TAR files') + } + + return notSupportedResponse('application/tar support is not implemented') } - private async handleJson ({ cid, path, options }: FetchHandlerFunctionArg): Promise { + private async handleJson ({ cid, path, accept, options }: FetchHandlerFunctionArg): Promise { this.log.trace('fetching %c/%s', cid, path) - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { cid, path })) - const block = await this.helia.blockstore.get(cid, { - signal: options?.signal, - onProgress: options?.onProgress - }) - const response = okResponse(block) - response.headers.set('content-type', 'application/json') - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid, path })) + const block = await this.helia.blockstore.get(cid, options) + let body: string | Uint8Array + + if (accept === 'application/vnd.ipld.dag-cbor' || accept === 'application/cbor') { + try { + // if vnd.ipld.dag-cbor has been specified, convert to the format - note + // that this supports more data types than regular JSON, the content-type + // response header is set so the user knows to process it differently + const obj = ipldDagJson.decode(block) + body = ipldDagCbor.encode(obj) + } catch (err) { + this.log.error('could not transform %c to application/vnd.ipld.dag-cbor', err) + return notAcceptableResponse() + } + } else { + // skip decoding + body = block + } + + const response = okResponse(body) + response.headers.set('content-type', accept ?? 'application/json') return response } private async handleDagCbor ({ cid, path, accept, options }: FetchHandlerFunctionArg): Promise { this.log.trace('fetching %c/%s', cid, path) - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { cid, path })) - // return body as binary - const block = await this.helia.blockstore.get(cid) - let body: string | Uint8Array - try { - body = dagCborToSafeJSON(block) - } catch (err) { - if (accept === 'application/json') { - this.log('could not decode DAG-CBOR as JSON-safe, but the client sent "Accept: application/json"', err) + const block = await this.helia.blockstore.get(cid, options) + let body: string | Uint8Array + if (accept === 'application/octet-stream' || accept === 'application/vnd.ipld.dag-cbor' || accept === 'application/cbor') { + // skip decoding + body = block + } else if (accept === 'application/vnd.ipld.dag-json') { + try { + // if vnd.ipld.dag-json has been specified, convert to the format - note + // that this supports more data types than regular JSON, the content-type + // response header is set so the user knows to process it differently + const obj = ipldDagCbor.decode(block) + body = ipldDagJson.encode(obj) + } catch (err) { + this.log.error('could not transform %c to application/vnd.ipld.dag-json', err) return notAcceptableResponse() } + } else { + try { + body = dagCborToSafeJSON(block) + } catch (err) { + if (accept === 'application/json') { + this.log('could not decode DAG-CBOR as JSON-safe, but the client sent "Accept: application/json"', err) - this.log('could not decode DAG-CBOR as JSON-safe, falling back to `application/octet-stream`', err) - body = block + return notAcceptableResponse() + } + + this.log('could not decode DAG-CBOR as JSON-safe, falling back to `application/octet-stream`', err) + body = block + } } const response = okResponse(body) - response.headers.set('content-type', body instanceof Uint8Array ? 'application/octet-stream' : 'application/json') - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid, path })) + + if (accept == null) { + accept = body instanceof Uint8Array ? 'application/octet-stream' : 'application/json' + } + + response.headers.set('content-type', accept) + return response } - private async handleDagPb ({ cid, path, options, terminalElement }: FetchHandlerFunctionArg): Promise { - this.log.trace('fetching %c/%s', cid, path) + private async handleDagPb ({ cid, path, options }: FetchHandlerFunctionArg): Promise { + let terminalElement: UnixFSEntry | undefined + let ipfsRoots: CID[] | undefined + + try { + const pathDetails = await walkPath(this.helia.blockstore, `${cid.toString()}/${path}`, options) + ipfsRoots = pathDetails.ipfsRoots + terminalElement = pathDetails.terminalElement + } catch (err) { + this.log.error('Error walking path %s', path, err) + // return new Response(`Error walking path: ${(err as Error).message}`, { status: 500 }) + } + let resolvedCID = terminalElement?.cid ?? cid let stat: UnixFSStats if (terminalElement?.type === 'directory') { @@ -207,7 +265,6 @@ export class VerifiedFetch { const rootFilePath = 'index.html' try { this.log.trace('found directory at %c/%s, looking for index.html', cid, path) - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { cid: dirCid, path: rootFilePath })) stat = await this.unixfs.stat(dirCid, { path: rootFilePath, signal: options?.signal, @@ -225,7 +282,6 @@ export class VerifiedFetch { } } - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { cid: resolvedCID, path: '' })) const asyncIter = this.unixfs.cat(resolvedCID, { signal: options?.signal, onProgress: options?.onProgress @@ -238,20 +294,20 @@ export class VerifiedFetch { const response = okResponse(stream) await this.setContentType(firstChunk, path, response) - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid: resolvedCID, path: '' })) + if (ipfsRoots != null) { + response.headers.set('X-Ipfs-Roots', ipfsRoots.map(cid => cid.toV1().toString()).join(',')) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-roots-response-header + } return response } private async handleRaw ({ cid, path, options }: FetchHandlerFunctionArg): Promise { - this.log.trace('fetching %c/%s', cid, path) - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { cid, path })) - const result = await this.helia.blockstore.get(cid) + const result = await this.helia.blockstore.get(cid, options) const response = okResponse(result) // if the user has specified an `Accept` header that corresponds to a raw - // type, honour that header, so they don't request `vnd.ipld.raw` and get - // `octet-stream` or vice versa + // type, honour that header, so for example they don't request + // `application/vnd.ipld.raw` but get `application/octet-stream` const overriddenContentType = getOverridenRawContentType(options?.headers) if (overriddenContentType != null) { response.headers.set('content-type', overriddenContentType) @@ -259,7 +315,6 @@ export class VerifiedFetch { await this.setContentType(result, path, response) } - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid, path })) return response } @@ -289,109 +344,113 @@ export class VerifiedFetch { response.headers.set('content-type', contentType) } - /** - * Map of format to specific handlers for that format. - * - * These format handlers should adjust the response headers as specified in - * https://specs.ipfs.tech/http-gateways/path-gateway/#response-headers - */ - private readonly formatHandlers: Record = { - raw: this.handleRaw, - car: this.handleIPLDCar, - 'ipns-record': this.handleIPNSRecord, - tar: async () => notSupportedResponse('application/x-tar support is not implemented'), - 'dag-json': this.handleJson, - 'dag-cbor': this.handleDagCbor, - json: this.handleJson, - cbor: this.handleDagCbor - } - /** * If the user has not specified an Accept header or format query string arg, * use the CID codec to choose an appropriate handler for the block data. */ private readonly codecHandlers: Record = { [dagPbCode]: this.handleDagPb, - [dagJsonCode]: this.handleJson, + [ipldDagJson.code]: this.handleJson, [jsonCode]: this.handleJson, - [dagCborCode]: this.handleDagCbor, + [ipldDagCbor.code]: this.handleDagCbor, [rawCode]: this.handleRaw, [identity.code]: this.handleRaw } async fetch (resource: Resource, opts?: VerifiedFetchOptions): Promise { - this.log('fetch', resource) + this.log('fetch %s', resource) const options = convertOptions(opts) - const { path, query, ...rest } = await parseResource(resource, { ipns: this.ipns, logger: this.helia.logger }, options) - const cid = rest.cid - let response: Response | undefined - const acceptHeader = new Headers(options?.headers).get('accept') - this.log('accept header %s', acceptHeader) + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { resource })) - const format = getFormat({ cid, headerFormat: acceptHeader, queryFormat: query.format ?? null }) - this.log('format %s, mime type %s', format?.format, format?.mimeType) + // resolve the CID/path from the requested resource + const { path, query, cid } = await parseResource(resource, { ipns: this.ipns, logger: this.helia.logger }, options) - if (format == null && acceptHeader != null) { - this.log('no format found for accept header %s', acceptHeader) + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:resolve', { cid, path })) - // user specified an Accept header but we had no handler for it - return notAcceptableResponse() - } + const requestHeaders = new Headers(options?.headers) + const incomingAcceptHeader = requestHeaders.get('accept') - if (format != null) { - // TODO: These should be handled last when they're returning something other than 501 - const formatHandler = this.formatHandlers[format.format] + if (incomingAcceptHeader != null) { + this.log('incoming accept header "%s"', incomingAcceptHeader) + } - if (formatHandler != null) { - response = await formatHandler.call(this, { cid, path, accept: format.mimeType, options }) + const queryFormatMapping = queryFormatToAcceptHeader(query.format) - if (response.status === 501) { - return response - } - } + if (query.format != null) { + this.log('incoming query format "%s", mapped to %s', query.format, queryFormatMapping) } - let terminalElement: UnixFSEntry | undefined - let ipfsRoots: CID[] | undefined + const acceptHeader = incomingAcceptHeader ?? queryFormatMapping + const accept = selectOutputType(cid, acceptHeader) + this.log('output type %s', accept) - try { - const pathDetails = await this.pathWalker(this.helia.blockstore, `${cid.toString()}/${path}`, options) - ipfsRoots = pathDetails.ipfsRoots - terminalElement = pathDetails.terminalElement - } catch (err) { - this.log.error('Error walking path %s', path, err) - // return new Response(`Error walking path: ${(err as Error).message}`, { status: 500 }) + if (acceptHeader != null && accept == null) { + return notAcceptableResponse() } - if (response == null) { + let response: Response + let reqFormat: RequestFormatShorthand | undefined + + if (accept === 'application/vnd.ipfs.ipns-record') { + // the user requested a raw IPNS record + reqFormat = 'ipns-record' + response = await this.handleIPNSRecord(resource.toString(), options) + } else if (accept === 'application/vnd.ipld.car') { + // the user requested a CAR file + reqFormat = 'car' + query.download = true + query.filename = query.filename ?? `${cid.toString()}.car` + response = await this.handleCar({ cid, path, options }) + } else if (accept === 'application/vnd.ipld.raw') { + // the user requested a raw block + reqFormat = 'raw' + query.download = true + query.filename = query.filename ?? `${cid.toString()}.bin` + response = await this.handleRaw({ cid, path, options }) + } else if (accept === 'application/x-tar') { + // the user requested a TAR file + reqFormat = 'tar' + response = await this.handleTar({ cid, path, options }) + } else { + // derive the handler from the CID type const codecHandler = this.codecHandlers[cid.code] - if (codecHandler != null) { - response = await codecHandler.call(this, { cid, path, options, terminalElement }) - } else { + if (codecHandler == null) { return notSupportedResponse(`Support for codec with code ${cid.code} is not yet implemented. Please open an issue at https://github.com/ipfs/helia/issues/new`) } + + response = await codecHandler.call(this, { cid, path, accept, options }) } - const contentType = response.headers.get('content-type') + response.headers.set('etag', getETag({ cid, reqFormat, weak: false })) + response.headers.set('cache-control', 'public, max-age=29030400, immutable') + // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-path-response-header + response.headers.set('X-Ipfs-Path', resource.toString()) - if (format != null && format.mimeType !== '*/*' && contentType !== format.mimeType) { - // the user requested a specific, non-wildcard representation type, but - // the data cannot be represented as that type so return a - // "Not Acceptable" response - return notAcceptableResponse() + // set Content-Disposition header + let contentDisposition: string | undefined + + // force download if requested + if (query.download === true) { + contentDisposition = 'attachment' } - response.headers.set('etag', getETag({ cid, reqFormat: format?.format, weak: false })) - response.headers.set('cache-control', 'public, max-age=29030400, immutable') - response.headers.set('X-Ipfs-Path', resource.toString()) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-path-response-header + // override filename if requested + if (query.filename != null) { + if (contentDisposition == null) { + contentDisposition = 'inline' + } - if (ipfsRoots != null) { - response.headers.set('X-Ipfs-Roots', ipfsRoots.map(cid => cid.toV1().toString()).join(',')) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-roots-response-header + contentDisposition = `${contentDisposition}; ${getContentDispositionFilename(query.filename)}` + } + + if (contentDisposition != null) { + response.headers.set('Content-Disposition', contentDisposition) } - // response.headers.set('Content-Disposition', `TODO`) // https://specs.ipfs.tech/http-gateways/path-gateway/#content-disposition-response-header + + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid, path })) return response } diff --git a/packages/verified-fetch/test/accept-header.spec.ts b/packages/verified-fetch/test/accept-header.spec.ts index d08949b32..0e008afe8 100644 --- a/packages/verified-fetch/test/accept-header.spec.ts +++ b/packages/verified-fetch/test/accept-header.spec.ts @@ -1,10 +1,16 @@ -/* eslint-env mocha */ +import { car } from '@helia/car' import { dagCbor } from '@helia/dag-cbor' +import { dagJson } from '@helia/dag-json' +import { ipns } from '@helia/ipns' import * as ipldDagCbor from '@ipld/dag-cbor' +import * as ipldDagJson from '@ipld/dag-json' import { stop } from '@libp2p/interface' +import { createEd25519PeerId } from '@libp2p/peer-id-factory' import { expect } from 'aegir/chai' +import { marshal } from 'ipns' import { VerifiedFetch } from '../src/verified-fetch.js' import { createHelia } from './fixtures/create-offline-helia.js' +import { memoryCarWriter } from './fixtures/memory-car.js' import type { Helia } from '@helia/interface' describe('accept header', () => { @@ -62,6 +68,78 @@ describe('accept header', () => { expect(output).to.deep.equal(obj) }) + it('should transform DAG-CBOR to DAG-JSON', async () => { + const obj = { + hello: 'world' + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const resp = await verifiedFetch.fetch(cid, { + headers: { + accept: 'application/vnd.ipld.dag-json' + } + }) + expect(resp.headers.get('content-type')).to.equal('application/vnd.ipld.dag-json') + + const output = ipldDagJson.decode(new Uint8Array(await resp.arrayBuffer())) + expect(output).to.deep.equal(obj) + }) + + it('should transform DAG-CBOR to JSON', async () => { + const obj = { + hello: 'world' + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const resp = await verifiedFetch.fetch(cid, { + headers: { + accept: 'application/json' + } + }) + expect(resp.headers.get('content-type')).to.equal('application/json') + + const output = ipldDagJson.decode(new Uint8Array(await resp.arrayBuffer())) + expect(output).to.deep.equal(obj) + }) + + it('should transform DAG-JSON to DAG-CBOR', async () => { + const obj = { + hello: 'world' + } + const j = dagJson(helia) + const cid = await j.add(obj) + + const resp = await verifiedFetch.fetch(cid, { + headers: { + accept: 'application/vnd.ipld.dag-cbor' + } + }) + expect(resp.headers.get('content-type')).to.equal('application/vnd.ipld.dag-cbor') + + const output = ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) + expect(output).to.deep.equal(obj) + }) + + it('should transform DAG-JSON to CBOR', async () => { + const obj = { + hello: 'world' + } + const j = dagJson(helia) + const cid = await j.add(obj) + + const resp = await verifiedFetch.fetch(cid, { + headers: { + accept: 'application/cbor' + } + }) + expect(resp.headers.get('content-type')).to.equal('application/cbor') + + const output = ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) + expect(output).to.deep.equal(obj) + }) + it('should return 406 Not Acceptable if the accept header cannot be honoured', async () => { const obj = { hello: 'world' @@ -78,7 +156,23 @@ describe('accept header', () => { expect(resp.statusText).to.equal('406 Not Acceptable') }) - it('should suuport wildcards', async () => { + it('should support wildcards', async () => { + const obj = { + hello: 'world' + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const resp = await verifiedFetch.fetch(cid, { + headers: { + accept: 'application/what-even-is-this, */*, application/vnd.ipld.raw' + } + }) + expect(resp.status).to.equal(200) + expect(resp.headers.get('content-type')).to.equal('application/json') + }) + + it('should support type wildcards', async () => { const obj = { hello: 'world' } @@ -87,10 +181,94 @@ describe('accept header', () => { const resp = await verifiedFetch.fetch(cid, { headers: { - accept: 'application/what-even-is-this, */*' + accept: '*/json, application/vnd.ipld.raw' + } + }) + expect(resp.status).to.equal(200) + expect(resp.headers.get('content-type')).to.equal('application/json') + }) + + it('should support subtype wildcards', async () => { + const obj = { + hello: 'world' + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const resp = await verifiedFetch.fetch(cid, { + headers: { + accept: 'application/*, application/vnd.ipld.raw' + } + }) + expect(resp.status).to.equal(200) + expect(resp.headers.get('content-type')).to.equal('application/json') + }) + + it('should support q-factor weighting', async () => { + const obj = { + hello: 'world' + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const resp = await verifiedFetch.fetch(cid, { + headers: { + // these all match, application/json would be chosen as it is first but + // application/octet-stream has a higher weighting so it should win + accept: [ + 'application/json;q=0.1', + 'application/application/vnd.ipld.raw;q=0.5', + 'application/octet-stream;q=0.8' + ].join(', ') } }) expect(resp.status).to.equal(200) expect(resp.headers.get('content-type')).to.equal('application/octet-stream') }) + + it.skip('should support fetching IPNS records', async () => { + const peerId = await createEd25519PeerId() + const obj = { + hello: 'world' + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const i = ipns(helia) + const record = await i.publish(peerId, cid) + + const resp = await verifiedFetch.fetch(`ipns://${peerId}`, { + headers: { + accept: 'application/vnd.ipfs.ipns-record' + } + }) + expect(resp.status).to.equal(200) + expect(resp.headers.get('content-type')).to.equal('application/vnd.ipfs.ipns-record') + const buf = await resp.arrayBuffer() + + expect(buf).to.equalBytes(marshal(record)) + }) + + it.skip('should support fetching a CAR file', async () => { + const obj = { + hello: 'world' + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const ca = car(helia) + const writer = memoryCarWriter(cid) + await ca.export(cid, writer) + + const resp = await verifiedFetch.fetch(cid, { + headers: { + accept: 'application/vnd.ipld.car' + } + }) + expect(resp.status).to.equal(200) + expect(resp.headers.get('content-type')).to.equal('application/vnd.ipld.car; version=1') + const buf = await resp.arrayBuffer() + + expect(buf).to.equalBytes(await writer.bytes()) + }) }) diff --git a/packages/verified-fetch/test/fixtures/cids.ts b/packages/verified-fetch/test/fixtures/cids.ts index dc4d73765..03ab9f0cf 100644 --- a/packages/verified-fetch/test/fixtures/cids.ts +++ b/packages/verified-fetch/test/fixtures/cids.ts @@ -1,6 +1,18 @@ +import * as dagCbor from '@ipld/dag-cbor' +import * as dagJson from '@ipld/dag-json' +import * as dagPb from '@ipld/dag-pb' import { CID } from 'multiformats/cid' +import * as json from 'multiformats/codecs/json' +import * as raw from 'multiformats/codecs/raw' + +// 112 = dag-pb, 18 = sha256, 0 = CIDv0 +const mh = CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr').multihash export const cids: Record = { - file: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), - dagCbor: CID.parse('bafyreicnokmhmrnlp2wjhyk2haep4tqxiptwfrp2rrs7rzq7uk766chqvq') + filev0: CID.createV0(mh), + file: CID.createV1(dagPb.code, mh), + dagCbor: CID.createV1(dagCbor.code, mh), + dagJson: CID.createV1(dagJson.code, mh), + json: CID.createV1(json.code, mh), + raw: CID.createV1(raw.code, mh) } diff --git a/packages/verified-fetch/test/fixtures/memory-car.ts b/packages/verified-fetch/test/fixtures/memory-car.ts new file mode 100644 index 000000000..ad210cb36 --- /dev/null +++ b/packages/verified-fetch/test/fixtures/memory-car.ts @@ -0,0 +1,33 @@ +import { CarWriter } from '@ipld/car' +import toBuffer from 'it-to-buffer' +import defer from 'p-defer' +import type { CID } from 'multiformats/cid' + +export interface MemoryCar extends Pick { + bytes(): Promise +} + +export function memoryCarWriter (root: CID | CID[]): MemoryCar { + const deferred = defer() + const { writer, out } = CarWriter.create(Array.isArray(root) ? root : [root]) + + Promise.resolve() + .then(async () => { + deferred.resolve(await toBuffer(out)) + }) + .catch(err => { + deferred.reject(err) + }) + + return { + async put (block: { cid: CID, bytes: Uint8Array }): Promise { + await writer.put(block) + }, + async close (): Promise { + await writer.close() + }, + async bytes (): Promise { + return deferred.promise + } + } +} diff --git a/packages/verified-fetch/test/index.spec.ts b/packages/verified-fetch/test/index.spec.ts index e79d04371..90cd1054d 100644 --- a/packages/verified-fetch/test/index.spec.ts +++ b/packages/verified-fetch/test/index.spec.ts @@ -1,4 +1,3 @@ -/* eslint-env mocha */ import { createHeliaHTTP } from '@helia/http' import { expect } from 'aegir/chai' import { createHelia } from 'helia' diff --git a/packages/verified-fetch/test/utils/get-content-disposition-filename.spec.ts b/packages/verified-fetch/test/utils/get-content-disposition-filename.spec.ts new file mode 100644 index 000000000..4fb328d03 --- /dev/null +++ b/packages/verified-fetch/test/utils/get-content-disposition-filename.spec.ts @@ -0,0 +1,16 @@ +import { expect } from 'aegir/chai' +import { getContentDispositionFilename } from '../../src/utils/get-content-disposition-filename.js' + +describe('get-content-disposition-filename', () => { + it('should support ascii-only filenames', () => { + expect( + getContentDispositionFilename('foo.txt') + ).to.equal('filename="foo.txt"') + }) + + it('should remove non-ascii characters from filenames', () => { + expect( + getContentDispositionFilename('testтест.jpg') + ).to.equal('filename="test____.jpg"; filename*=UTF-8\'\'test%D1%82%D0%B5%D1%81%D1%82.jpg') + }) +}) diff --git a/packages/verified-fetch/test/utils/get-format.spec.ts b/packages/verified-fetch/test/utils/get-format.spec.ts deleted file mode 100644 index ac4e74e37..000000000 --- a/packages/verified-fetch/test/utils/get-format.spec.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { expect } from 'aegir/chai' -import { getFormat } from '../../src/utils/get-format.js' -import { cids } from '../fixtures/cids.js' - -describe('get-format', () => { - it('should override query format with Accept header if available', () => { - const format = getFormat({ - cid: cids.file, - headerFormat: 'application/vnd.ipld.car', - queryFormat: 'raw' - }) - - expect(format).to.have.property('format', 'car') - expect(format).to.have.property('mimeType', 'application/vnd.ipld.car') - }) - - it('should default wildcards to raw format', () => { - const format = getFormat({ - cid: cids.file, - headerFormat: '*/*' - }) - - expect(format).to.have.property('format', 'raw') - expect(format).to.have.property('mimeType', '*/*') - }) - - it('should use specific type before wildcard', () => { - const format = getFormat({ - cid: cids.file, - headerFormat: '*/*, application/x-tar' - }) - - expect(format).to.have.property('format', 'tar') - expect(format).to.have.property('mimeType', 'application/x-tar') - }) - - it('should use specific type before wildcard', () => { - const format = getFormat({ - cid: cids.file, - headerFormat: 'application/x-tar, */*' - }) - - expect(format).to.have.property('format', 'tar') - expect(format).to.have.property('mimeType', 'application/x-tar') - }) - - it('should support partial wildcard', () => { - const format = getFormat({ - cid: cids.file, - headerFormat: 'application/*' - }) - - expect(format).to.have.property('format', 'raw') - expect(format).to.have.property('mimeType', '*/*') - }) - - it('should support partial wildcard', () => { - const format = getFormat({ - cid: cids.file, - headerFormat: '*/' - }) - - expect(format).to.have.property('format', 'raw') - expect(format).to.have.property('mimeType', '*/*') - }) -}) diff --git a/packages/verified-fetch/test/utils/select-output-type.spec.ts b/packages/verified-fetch/test/utils/select-output-type.spec.ts new file mode 100644 index 000000000..244d9c87f --- /dev/null +++ b/packages/verified-fetch/test/utils/select-output-type.spec.ts @@ -0,0 +1,41 @@ +import { expect } from 'aegir/chai' +import { selectOutputType } from '../../src/utils/select-output-type.js' +import { cids } from '../fixtures/cids.js' + +describe('select-output-type', () => { + it('should return undefined if no accept header passed', () => { + const format = selectOutputType(cids.file) + + expect(format).to.be.undefined() + }) + + it('should override query format with Accept header if available', () => { + const format = selectOutputType(cids.file, 'application/vnd.ipld.car') + + expect(format).to.equal('application/vnd.ipld.car') + }) + + it('should match accept headers with equal weighting in definition order', () => { + const format = selectOutputType(cids.file, 'application/x-tar, */*') + + expect(format).to.equal('application/x-tar') + }) + + it('should match accept headers in weighting order', () => { + const format = selectOutputType(cids.file, 'application/x-tar;q=0.1, application/octet-stream;q=0.5, text/html') + + expect(format).to.equal('application/octet-stream') + }) + + it('should support partial type wildcard', () => { + const format = selectOutputType(cids.file, '*/json') + + expect(format).to.equal('application/json') + }) + + it('should support partial subtype wildcard', () => { + const format = selectOutputType(cids.file, 'application/*') + + expect(format).to.equal('application/octet-stream') + }) +}) diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index 304c3a1d2..56d0e38ae 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -1,4 +1,3 @@ -/* eslint-env mocha */ import { dagCbor } from '@helia/dag-cbor' import { dagJson } from '@helia/dag-json' import { type IPNS } from '@helia/ipns' @@ -138,15 +137,18 @@ describe('@helia/verifed-fetch', () => { onProgress }) - expect(onProgress.callCount).to.equal(3) + expect(onProgress.callCount).to.equal(4) const onProgressEvents = onProgress.getCalls().map(call => call.args[0]) - expect(onProgressEvents[0]).to.include({ type: 'blocks:get:blockstore:get' }).and.to.have.property('detail').that.deep.equals(cid) - expect(onProgressEvents[1]).to.include({ type: 'verified-fetch:request:start' }).and.to.have.property('detail').that.deep.equals({ + expect(onProgressEvents[0]).to.include({ type: 'verified-fetch:request:start' }).and.to.have.property('detail').that.deep.equals({ + resource: `ipfs://${cid}` + }) + expect(onProgressEvents[1]).to.include({ type: 'verified-fetch:request:resolve' }).and.to.have.property('detail').that.deep.equals({ cid, path: '' }) - expect(onProgressEvents[2]).to.include({ type: 'verified-fetch:request:end' }).and.to.have.property('detail').that.deep.equals({ + expect(onProgressEvents[2]).to.include({ type: 'blocks:get:blockstore:get' }).and.to.have.property('detail').that.deep.equals(cid) + expect(onProgressEvents[3]).to.include({ type: 'verified-fetch:request:end' }).and.to.have.property('detail').that.deep.equals({ cid, path: '' }) From 0ad29ea6095d945d49ce9b268e40dfc68a0a1d5a Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 20 Feb 2024 13:24:01 +0000 Subject: [PATCH 04/10] chore: update dev deps --- packages/verified-fetch/package.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index c710cf616..fda1a6809 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -159,10 +159,12 @@ "progress-events": "^1.0.0" }, "devDependencies": { + "@helia/car": "^3.0.0", "@helia/dag-cbor": "^3.0.0", "@helia/dag-json": "^3.0.0", "@helia/json": "^3.0.0", "@helia/utils": "^0.0.1", + "@ipld/car": "^5.2.6", "@libp2p/logger": "^4.0.5", "@libp2p/peer-id-factory": "^4.0.5", "@sgtpooki/file-type": "^1.0.1", @@ -171,9 +173,11 @@ "blockstore-core": "^4.4.0", "datastore-core": "^9.2.8", "helia": "^4.0.1", + "ipns": "^9.0.0", "it-last": "^3.0.4", "it-to-buffer": "^4.0.5", "magic-bytes.js": "^1.8.0", + "p-defer": "^4.0.0", "sinon": "^17.0.1", "sinon-ts": "^2.0.0", "uint8arrays": "^5.0.1" From 9a1849c966e1aa2857960f98446fd1b78afdf498 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 20 Feb 2024 17:38:29 +0000 Subject: [PATCH 05/10] feat: support downloading car files from @helia/verified-fetch Adds support for the `application/vnd.ipld.car` accept header to allow downloading CAR files of DAGs. --- packages/verified-fetch/package.json | 1 + packages/verified-fetch/src/verified-fetch.ts | 25 ++++++- .../verified-fetch/test/accept-header.spec.ts | 25 ------- packages/verified-fetch/test/car.spec.ts | 69 +++++++++++++++++++ 4 files changed, 94 insertions(+), 26 deletions(-) create mode 100644 packages/verified-fetch/test/car.spec.ts diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index fda1a6809..c6ecdc7ab 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -155,6 +155,7 @@ "cborg": "^4.0.9", "hashlru": "^2.3.0", "ipfs-unixfs-exporter": "^13.5.0", + "it-to-browser-readablestream": "^2.0.6", "multiformats": "^13.1.0", "progress-events": "^1.0.0" }, diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index f31c363c3..508b5bb39 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -1,9 +1,12 @@ +import { car } from '@helia/car' import { ipns as heliaIpns, type IPNS } from '@helia/ipns' import { dnsJsonOverHttps } from '@helia/ipns/dns-resolvers' import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs, type UnixFSStats } from '@helia/unixfs' +import { CarWriter } from '@ipld/car' import * as ipldDagCbor from '@ipld/dag-cbor' import * as ipldDagJson from '@ipld/dag-json' import { code as dagPbCode } from '@ipld/dag-pb' +import toBrowserReadableStream from 'it-to-browser-readablestream' import { code as jsonCode } from 'multiformats/codecs/json' import { code as rawCode } from 'multiformats/codecs/raw' import { identity } from 'multiformats/hashes/identity' @@ -157,7 +160,27 @@ export class VerifiedFetch { * of the `DAG` referenced by the `CID`. */ private async handleCar ({ cid, path, options }: FetchHandlerFunctionArg): Promise { - return notSupportedResponse('vnd.ipld.car support is not implemented') + const c = car(this.helia) + const { writer, out } = CarWriter.create(cid) + + const stream = toBrowserReadableStream(async function * () { + yield * out + }()) + + // write the DAG behind `cid` into the writer + c.export(cid, writer, options) + .catch(err => { + this.log.error('could not write car', err) + stream.cancel(err) + .catch(err => { + this.log.error('could not cancel stream after car export error', err) + }) + }) + + const response = okResponse(stream) + response.headers.set('content-type', 'application/vnd.ipld.car; version=1') + + return response } /** diff --git a/packages/verified-fetch/test/accept-header.spec.ts b/packages/verified-fetch/test/accept-header.spec.ts index 0e008afe8..74b852bf9 100644 --- a/packages/verified-fetch/test/accept-header.spec.ts +++ b/packages/verified-fetch/test/accept-header.spec.ts @@ -1,4 +1,3 @@ -import { car } from '@helia/car' import { dagCbor } from '@helia/dag-cbor' import { dagJson } from '@helia/dag-json' import { ipns } from '@helia/ipns' @@ -10,7 +9,6 @@ import { expect } from 'aegir/chai' import { marshal } from 'ipns' import { VerifiedFetch } from '../src/verified-fetch.js' import { createHelia } from './fixtures/create-offline-helia.js' -import { memoryCarWriter } from './fixtures/memory-car.js' import type { Helia } from '@helia/interface' describe('accept header', () => { @@ -248,27 +246,4 @@ describe('accept header', () => { expect(buf).to.equalBytes(marshal(record)) }) - - it.skip('should support fetching a CAR file', async () => { - const obj = { - hello: 'world' - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const ca = car(helia) - const writer = memoryCarWriter(cid) - await ca.export(cid, writer) - - const resp = await verifiedFetch.fetch(cid, { - headers: { - accept: 'application/vnd.ipld.car' - } - }) - expect(resp.status).to.equal(200) - expect(resp.headers.get('content-type')).to.equal('application/vnd.ipld.car; version=1') - const buf = await resp.arrayBuffer() - - expect(buf).to.equalBytes(await writer.bytes()) - }) }) diff --git a/packages/verified-fetch/test/car.spec.ts b/packages/verified-fetch/test/car.spec.ts new file mode 100644 index 000000000..8e6927043 --- /dev/null +++ b/packages/verified-fetch/test/car.spec.ts @@ -0,0 +1,69 @@ +import { car } from '@helia/car' +import { dagCbor } from '@helia/dag-cbor' +import { stop } from '@libp2p/interface' +import { expect } from 'aegir/chai' +import { VerifiedFetch } from '../src/verified-fetch.js' +import { createHelia } from './fixtures/create-offline-helia.js' +import { memoryCarWriter } from './fixtures/memory-car.js' +import type { Helia } from '@helia/interface' + +describe('car files', () => { + let helia: Helia + let verifiedFetch: VerifiedFetch + + beforeEach(async () => { + helia = await createHelia() + verifiedFetch = new VerifiedFetch({ + helia + }) + }) + + afterEach(async () => { + await stop(helia, verifiedFetch) + }) + + it('should support fetching a CAR file', async () => { + const obj = { + hello: 'world' + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const ca = car(helia) + const writer = memoryCarWriter(cid) + await ca.export(cid, writer) + + const resp = await verifiedFetch.fetch(cid, { + headers: { + accept: 'application/vnd.ipld.car' + } + }) + expect(resp.status).to.equal(200) + expect(resp.headers.get('content-type')).to.equal('application/vnd.ipld.car; version=1') + expect(resp.headers.get('content-disposition')).to.equal(`attachment; filename="${cid.toString()}.car"`) + const buf = new Uint8Array(await resp.arrayBuffer()) + + expect(buf).to.equalBytes(await writer.bytes()) + }) + + it('should support specify a filename for a CAR file', async () => { + const obj = { + hello: 'world' + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const ca = car(helia) + const writer = memoryCarWriter(cid) + await ca.export(cid, writer) + + const resp = await verifiedFetch.fetch(`ipfs://${cid}?filename=foo.bar`, { + headers: { + accept: 'application/vnd.ipld.car' + } + }) + expect(resp.status).to.equal(200) + expect(resp.headers.get('content-type')).to.equal('application/vnd.ipld.car; version=1') + expect(resp.headers.get('content-disposition')).to.equal('attachment; filename="foo.bar"') + }) +}) From 8356cf42274c4abe9b6d796823ae786f36887f8e Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 21 Feb 2024 07:16:30 +0000 Subject: [PATCH 06/10] chore: fix deps --- packages/verified-fetch/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index c6ecdc7ab..8f5acdbc1 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -141,12 +141,14 @@ "release": "aegir release" }, "dependencies": { + "@helia/car": "^3.0.0", "@helia/block-brokers": "^2.0.1", "@helia/http": "^1.0.1", "@helia/interface": "^4.0.0", "@helia/ipns": "^6.0.0", "@helia/routers": "^1.0.0", "@helia/unixfs": "^3.0.0", + "@ipld/car": "^5.2.6", "@ipld/dag-cbor": "^9.2.0", "@ipld/dag-json": "^10.2.0", "@ipld/dag-pb": "^4.1.0", @@ -160,12 +162,10 @@ "progress-events": "^1.0.0" }, "devDependencies": { - "@helia/car": "^3.0.0", "@helia/dag-cbor": "^3.0.0", "@helia/dag-json": "^3.0.0", "@helia/json": "^3.0.0", "@helia/utils": "^0.0.1", - "@ipld/car": "^5.2.6", "@libp2p/logger": "^4.0.5", "@libp2p/peer-id-factory": "^4.0.5", "@sgtpooki/file-type": "^1.0.1", From 4bd955960af59875094ef37469a3190b56f8bffc Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Wed, 21 Feb 2024 07:48:54 +0000 Subject: [PATCH 07/10] chore: typo Co-authored-by: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> --- packages/verified-fetch/test/car.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/verified-fetch/test/car.spec.ts b/packages/verified-fetch/test/car.spec.ts index 8e6927043..01937dbf0 100644 --- a/packages/verified-fetch/test/car.spec.ts +++ b/packages/verified-fetch/test/car.spec.ts @@ -46,7 +46,7 @@ describe('car files', () => { expect(buf).to.equalBytes(await writer.bytes()) }) - it('should support specify a filename for a CAR file', async () => { + it('should support specifying a filename for a CAR file', async () => { const obj = { hello: 'world' } From 06f39f439400805e2b8d7e40975fb1302e91e236 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 21 Feb 2024 15:51:08 +0000 Subject: [PATCH 08/10] chore: add another comment --- packages/verified-fetch/src/verified-fetch.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index 508b5bb39..3865fe8e3 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -159,15 +159,14 @@ export class VerifiedFetch { * Accepts a `CID` and returns a `Response` with a body stream that is a CAR * of the `DAG` referenced by the `CID`. */ - private async handleCar ({ cid, path, options }: FetchHandlerFunctionArg): Promise { + private async handleCar ({ cid, options }: FetchHandlerFunctionArg): Promise { const c = car(this.helia) const { writer, out } = CarWriter.create(cid) - const stream = toBrowserReadableStream(async function * () { - yield * out - }()) + // convert AsyncIterable -> AsyncIterator -> ReadableStream + const stream = toBrowserReadableStream(out[Symbol.asyncIterator]()) - // write the DAG behind `cid` into the writer + // write all blocks from the DAG into the car writer c.export(cid, writer, options) .catch(err => { this.log.error('could not write car', err) From ffc2fb6a2252f128aa9e5f6b59e48541d0e1770f Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 21 Feb 2024 16:17:16 +0000 Subject: [PATCH 09/10] chore: simplify --- packages/verified-fetch/package.json | 2 +- packages/verified-fetch/src/verified-fetch.ts | 16 +--------------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index 8f5acdbc1..bdf392db6 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -148,7 +148,6 @@ "@helia/ipns": "^6.0.0", "@helia/routers": "^1.0.0", "@helia/unixfs": "^3.0.0", - "@ipld/car": "^5.2.6", "@ipld/dag-cbor": "^9.2.0", "@ipld/dag-json": "^10.2.0", "@ipld/dag-pb": "^4.1.0", @@ -166,6 +165,7 @@ "@helia/dag-json": "^3.0.0", "@helia/json": "^3.0.0", "@helia/utils": "^0.0.1", + "@ipld/car": "^5.2.6", "@libp2p/logger": "^4.0.5", "@libp2p/peer-id-factory": "^4.0.5", "@sgtpooki/file-type": "^1.0.1", diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index 3865fe8e3..ee9b07679 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -2,7 +2,6 @@ import { car } from '@helia/car' import { ipns as heliaIpns, type IPNS } from '@helia/ipns' import { dnsJsonOverHttps } from '@helia/ipns/dns-resolvers' import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs, type UnixFSStats } from '@helia/unixfs' -import { CarWriter } from '@ipld/car' import * as ipldDagCbor from '@ipld/dag-cbor' import * as ipldDagJson from '@ipld/dag-json' import { code as dagPbCode } from '@ipld/dag-pb' @@ -161,20 +160,7 @@ export class VerifiedFetch { */ private async handleCar ({ cid, options }: FetchHandlerFunctionArg): Promise { const c = car(this.helia) - const { writer, out } = CarWriter.create(cid) - - // convert AsyncIterable -> AsyncIterator -> ReadableStream - const stream = toBrowserReadableStream(out[Symbol.asyncIterator]()) - - // write all blocks from the DAG into the car writer - c.export(cid, writer, options) - .catch(err => { - this.log.error('could not write car', err) - stream.cancel(err) - .catch(err => { - this.log.error('could not cancel stream after car export error', err) - }) - }) + const stream = toBrowserReadableStream(c.stream(cid, options)) const response = okResponse(stream) response.headers.set('content-type', 'application/vnd.ipld.car; version=1') From 4a151fc8b1f6776315c99cad0f0b7fe5b7079c35 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 21 Feb 2024 18:31:38 +0000 Subject: [PATCH 10/10] chore: generic type can be derived --- packages/verified-fetch/src/verified-fetch.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index ee9b07679..16eaee612 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -160,7 +160,7 @@ export class VerifiedFetch { */ private async handleCar ({ cid, options }: FetchHandlerFunctionArg): Promise { const c = car(this.helia) - const stream = toBrowserReadableStream(c.stream(cid, options)) + const stream = toBrowserReadableStream(c.stream(cid, options)) const response = okResponse(stream) response.headers.set('content-type', 'application/vnd.ipld.car; version=1')