fixup! add tests

Signed-off-by: Miroslav Bajtoš <oss@bajtos.net>

Author: bajtos

lib/spark.js

@@ -83,15 +83,7 @@ export default class Spark {
     stats.protocol = provider.protocol
     stats.providerAddress = provider.address
 
-    try {
-      await this.fetchCAR(provider.protocol, provider.address, retrieval.cid, stats)
-    } catch (err) {
-      console.error(`Failed to fetch ${retrieval.cid} from ${provider.address} using ${provider.protocol}`)
-      console.error(err)
-      if (!stats.statusCode) {
-        stats.statusCode = mapErrorToStatusCode(err)
-      }
-    }
+    await this.fetchCAR(provider.protocol, provider.address, retrieval.cid, stats)
   }
 
   async fetchCAR (protocol, address, cid, stats) {
@@ -159,6 +151,12 @@ export default class Spark {
         console.error('Retrieval failed with status code %s: %s',
           res.status, (await res.text()).trimEnd())
       }
+    } catch (err) {
+      console.error(`Failed to fetch ${cid} from ${address} using ${protocol}`)
+      console.error(err)
+      if (!stats.statusCode || stats.statusCode === 200) {
+        stats.statusCode = mapErrorToStatusCode(err)
+      }
     } finally {
       clearTimeout(timeout)
     }
@@ -194,18 +192,7 @@ export default class Spark {
   async nextRetrieval () {
     const { id: retrievalId, ...retrieval } = await this.getRetrieval()
 
-    const stats = {
-      timeout: false,
-      startAt: new Date(),
-      firstByteAt: null,
-      endAt: null,
-      carTooLarge: false,
-      byteLength: 0,
-      carChecksum: null,
-      statusCode: null,
-      providerId: null,
-      indexerResult: null
-    }
+    const stats = newStats()
 
     await this.executeRetrievalCheck(retrieval, stats)
 
@@ -244,6 +231,19 @@ export default class Spark {
   }
 }
 
+export function newStats () {
+  return {
+    timeout: false,
+    startAt: new Date(),
+    firstByteAt: null,
+    endAt: null,
+    carTooLarge: false,
+    byteLength: 0,
+    carChecksum: null,
+    statusCode: null
+  }
+}
+
 function getRetrievalUrl (protocol, address, cid) {
   if (protocol === 'http') {
     const baseUrl = multiaddrToHttpUrl(address)

test/spark.js

@@ -1,6 +1,6 @@
 /* global Zinnia */
 
-import Spark from '../lib/spark.js'
+import Spark, { newStats } from '../lib/spark.js'
 import { test } from 'zinnia:test'
 import { assertInstanceOf, assertEquals, assertArrayIncludes } from 'zinnia:assert'
 import { SPARK_VERSION } from '../lib/constants.js'
@@ -47,24 +47,15 @@ test('getRetrieval', async () => {
   }])
 })
 
-// TODO: test more cases
-test('fetchCAR', async () => {
+test('fetchCAR - http', async () => {
   const requests = []
-  const mockedFetch = async url => {
-    requests.push(url.toString())
-    return fetch(url)
-  }
-  const spark = new Spark({ fetch: mockedFetch })
-  const stats = {
-    timeout: false,
-    startAt: new Date(),
-    firstByteAt: null,
-    endAt: null,
-    carTooLarge: false,
-    byteLength: 0,
-    carChecksum: null,
-    statusCode: null
-  }
+  const spark = new Spark({
+    fetch: async (url) => {
+      requests.push(url.toString())
+      return fetch(url)
+    }
+  })
+  const stats = newStats()
   await spark.fetchCAR('http', '/dns/frisbii.fly.dev/tcp/443/https', KNOWN_CID, stats)
   assertEquals(stats.statusCode, 200, 'stats.statusCode')
   assertEquals(stats.timeout, false, 'stats.timeout')
@@ -77,6 +68,35 @@ test('fetchCAR', async () => {
   assertEquals(requests, [`https://frisbii.fly.dev/ipfs/${KNOWN_CID}?dag-scope=block`])
 })
 
+test('fetchCAR - graphsync', async () => {
+  // This test relies on data stored as part of a Filecoin deal which will eventually expire.
+  // Also the storage provider may decide to stop serving Graphsync retrievals.
+  // When that happens, this test will start failing, and we will need to find different
+  // content that can be retrieved over Graphsync.
+  // Hopefully, we will no longer support Graphsync by that time.
+  const cid = 'bafybeiepi56qxfcwqgpstg25r6sonig7y3pzd37lwambzmlcmbnujjri4a'
+  const addr = '/dns/f010479.twinquasar.io/tcp/42002/p2p/12D3KooWHKeaNCnYByQUMS2n5PAZ1KZ9xKXqsb4bhpxVJ6bBJg5V'
+
+  const requests = []
+  const spark = new Spark({
+    fetch: async (url) => {
+      requests.push(url.toString())
+      return fetch(url)
+    }
+  })
+  const stats = newStats()
+  await spark.fetchCAR('graphsync', addr, cid, stats)
+  assertEquals(stats.statusCode, 200, 'stats.statusCode')
+  assertEquals(stats.timeout, false, 'stats.timeout')
+  assertInstanceOf(stats.startAt, Date)
+  assertInstanceOf(stats.firstByteAt, Date)
+  assertInstanceOf(stats.endAt, Date)
+  assertEquals(stats.carTooLarge, false, 'stats.carTooLarge')
+  assertEquals(stats.byteLength, 217, 'stats.byteLength')
+  assertEquals(stats.carChecksum, '1220a8d765159d8829f2bca7df05e5cd46eb88bdaa30905d3d08c6295562ea072f0f', 'stats.carChecksum')
+  assertEquals(requests, [`ipfs://${cid}?dag-scope=block&protocols=graphsync&providers=${encodeURIComponent(addr)}`])
+})
+
 /* Disabled as long as we are fetching the top-level block only
 test('fetchCAR exceeding MAX_CAR_SIZE', async () => {
   const fetch = async url => {
@@ -91,13 +111,7 @@ test('fetchCAR exceeding MAX_CAR_SIZE', async () => {
     }
   }
   const spark = new Spark({ fetch })
-  const stats = {
-    timeout: false,
-    carTooLarge: false,
-    byteLength: 0,
-    carChecksum: null,
-    statusCode: null
-  }
+  const stats = newStats()
   await spark.fetchCAR('http', '/ip4/127.0.0.1/tcp/80/http', 'bafy', stats)
   assertEquals(stats.timeout, false)
   assertEquals(stats.carTooLarge, true)
@@ -107,6 +121,108 @@ test('fetchCAR exceeding MAX_CAR_SIZE', async () => {
 })
 */
 
+test('fetchCAR fails with statusCode=701 (unsupported host type)', async () => {
+  const spark = new Spark()
+  const stats = newStats()
+  await spark.fetchCAR('http', '/ip99/1.2.3.4.5/tcp/80/http', KNOWN_CID, stats)
+  assertEquals(stats.statusCode, 701, 'stats.statusCode')
+})
+
+test('fetchCAR fails with statusCode=702 (protocol is not tcp)', async () => {
+  const spark = new Spark()
+  const stats = newStats()
+  await spark.fetchCAR('http', '/ip4/1.2.3.4/udp/80/http', KNOWN_CID, stats)
+  assertEquals(stats.statusCode, 702, 'stats.statusCode')
+})
+
+test('fetchCAR fails with statusCode=703 (scheme is not http/https)', async () => {
+  const spark = new Spark()
+  const stats = newStats()
+  await spark.fetchCAR('http', '/ip4/1.2.3.4/tcp/80/ldap', KNOWN_CID, stats)
+  assertEquals(stats.statusCode, 703, 'stats.statusCode')
+})
+
+test('fetchCAR fails with statusCode=704 (multiaddr has too many parts)', async () => {
+  const spark = new Spark()
+  const stats = newStats()
+  await spark.fetchCAR('http', '/ip4/1.2.3.4/tcp/80/http/p2p/pubkey', KNOWN_CID, stats)
+  assertEquals(stats.statusCode, 704, 'stats.statusCode')
+})
+
+test('fetchCAR fails with statusCode=801 (DNS error)', async () => {
+  const spark = new Spark()
+  const stats = newStats()
+  await spark.fetchCAR('http', '/dns/invalid.example.com/tcp/80/http', KNOWN_CID, stats)
+  assertEquals(stats.statusCode, 801, 'stats.statusCode')
+})
+
+test('fetchCAR fails with statusCode=802 (TCP connection refused)', async () => {
+  const spark = new Spark()
+  const stats = newStats()
+  await spark.fetchCAR('http', '/ip4/127.0.0.1/tcp/79/http', KNOWN_CID, stats)
+  assertEquals(stats.statusCode, 802, 'stats.statusCode')
+})
+
+test('fetchCAR fails with statusCode=802 (TCP connection refused)', async () => {
+  const spark = new Spark()
+  const stats = newStats()
+  await spark.fetchCAR('http', '/ip4/127.0.0.1/tcp/79/http', KNOWN_CID, stats)
+  assertEquals(stats.statusCode, 802, 'stats.statusCode')
+})
+
+// TODO:
+// statusCode=901 - unsupported hash algorithm
+
+test('fetchCAR fails with statusCode=902 (hash mismatch)', async () => {
+  const spark = new Spark({
+    fetch: async (url) => {
+      const res = await fetch(url)
+      return {
+        status: res.status,
+        ok: res.ok,
+        body: (async function * () {
+          const bytes = new Uint8Array(await res.arrayBuffer())
+          // manipulate one byte inside the CAR block
+          bytes[bytes.length - 1] = bytes[bytes.length - 1] ^ 0x88
+          yield bytes
+        })()
+      }
+    }
+  })
+  const stats = newStats()
+  await spark.fetchCAR('http', '/dns/frisbii.fly.dev/tcp/443/https', KNOWN_CID, stats)
+  assertEquals(stats.statusCode, 902, 'stats.statusCode')
+})
+
+test('fetchCAR fails with statusCode=903 (unexpected CAR block)', async () => {
+  const spark = new Spark({
+    // Fetch the root block of a different CID
+    fetch: (_url) => fetch(
+      'https://frisbii.fly.dev/ipfs/bafkreih5zasorm4tlfga4ztwvm2dlnw6jxwwuvgnokyt3mjamfn3svvpyy?dag-scope=block'
+    )
+  })
+  const stats = newStats()
+  await spark.fetchCAR('http', '/ip4/127.0.0.1/tcp/80/http', KNOWN_CID, stats)
+  assertEquals(stats.statusCode, 903, 'stats.statusCode')
+})
+
+test('fetchCAR fails with statusCode=904 (cannot parse CAR)', async () => {
+  const spark = new Spark({
+    fetch: async (_url) => {
+      return {
+        status: 200,
+        ok: true,
+        body: (async function * () {
+          yield new Uint8Array([1, 2, 3])
+        })()
+      }
+    }
+  })
+  const stats = newStats()
+  await spark.fetchCAR('http', '/ip4/127.0.0.1/tcp/80/http', KNOWN_CID, stats)
+  assertEquals(stats.statusCode, 904, 'stats.statusCode')
+})
+
 test('submitRetrieval', async () => {
   const requests = []
   const fetch = async (url, allOpts) => {