From e4d7a15dc77f43ce1ce891aeefce708595ba0b48 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Tue, 5 Aug 2025 11:08:03 -0400 Subject: [PATCH 1/6] fix: do not fetch each item in directories --- packages/verified-fetch/package.json | 6 +++--- .../verified-fetch/src/plugins/plugin-handle-dag-pb.ts | 8 +++++--- packages/verified-fetch/src/utils/dir-index-html.ts | 2 +- packages/verified-fetch/src/utils/walk-path.ts | 5 ++++- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index ddeaa242..2bedb7f4 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -170,7 +170,7 @@ "@helia/interface": "^5.3.1", "@helia/ipns": "^8.2.2", "@helia/routers": "^3.1.1", - "@helia/unixfs": "^5.0.2", + "@helia/unixfs": "^5.1.0", "@ipld/car": "^5.4.2", "@ipld/dag-cbor": "^9.2.3", "@ipld/dag-json": "^10.2.4", @@ -187,7 +187,7 @@ "helia": "^5.4.1", "interface-blockstore": "^5.3.1", "interface-datastore": "^8.3.1", - "ipfs-unixfs-exporter": "^13.6.2", + "ipfs-unixfs-exporter": "^13.7.2", "ipns": "^10.0.2", "it-map": "^3.1.3", "it-pipe": "^3.0.1", @@ -211,7 +211,7 @@ "browser-readablestream-to-it": "^2.0.9", "datastore-core": "^10.0.2", "helia": "^5.4.1", - "ipfs-unixfs-importer": "^15.3.2", + "ipfs-unixfs-importer": "^15.4.0", "it-all": "^3.0.8", "it-last": "^3.0.8", "it-to-buffer": "^4.0.9", diff --git a/packages/verified-fetch/src/plugins/plugin-handle-dag-pb.ts b/packages/verified-fetch/src/plugins/plugin-handle-dag-pb.ts index 66453ae4..cdbc4ab6 100644 --- a/packages/verified-fetch/src/plugins/plugin-handle-dag-pb.ts +++ b/packages/verified-fetch/src/plugins/plugin-handle-dag-pb.ts @@ -95,7 +95,8 @@ export class DagPbPlugin extends BasePlugin { const entry = await handleServerTiming('exporter-dir', '', async () => exporter(`/ipfs/${dirCid}/${rootFilePath}`, helia.blockstore, { signal: options?.signal, - onProgress: options?.onProgress + onProgress: options?.onProgress, + // extended: false }), withServerTiming) log.trace('found root file at %c/%s with cid %c', dirCid, rootFilePath, entry.cid) @@ -111,7 +112,7 @@ export class DagPbPlugin extends BasePlugin { context.modified++ this.log.trace('attempting to get directory entries because index.html was not found') try { - for await (const dirItem of fs.ls(dirCid, { signal: options?.signal, onProgress: options?.onProgress })) { + for await (const dirItem of fs.ls(dirCid, { signal: options?.signal, onProgress: options?.onProgress, extended: false })) { context.directoryEntries.push(dirItem) } // dir-index-html plugin or dir-index-json (future idea?) plugin should handle this @@ -138,7 +139,8 @@ export class DagPbPlugin extends BasePlugin { try { const entry = await handleServerTiming('exporter-file', '', async () => exporter(resolvedCID, helia.blockstore, { signal: options?.signal, - onProgress: options?.onProgress + onProgress: options?.onProgress, + // extended: false }), withServerTiming) let firstChunk: Uint8Array diff --git a/packages/verified-fetch/src/utils/dir-index-html.ts b/packages/verified-fetch/src/utils/dir-index-html.ts index 0800250d..48f60d73 100644 --- a/packages/verified-fetch/src/utils/dir-index-html.ts +++ b/packages/verified-fetch/src/utils/dir-index-html.ts @@ -165,7 +165,7 @@ export const dirIndexHtml = (dir: UnixFSEntry, items: UnixFSEntry[], { gatewayUR }, listing: items.map((item) => { return { - size: item.size.toString(), + size: item.size?.toString() ?? '?', name: item.name, path: getItemPath(item), hash: item.cid.toString(), diff --git a/packages/verified-fetch/src/utils/walk-path.ts b/packages/verified-fetch/src/utils/walk-path.ts index a2dfbf7b..fa6e3678 100644 --- a/packages/verified-fetch/src/utils/walk-path.ts +++ b/packages/verified-fetch/src/utils/walk-path.ts @@ -24,7 +24,10 @@ async function walkPath (blockstore: ReadableStorage, path: string, options?: Pa const ipfsRoots: CID[] = [] let terminalElement: UnixFSEntry | undefined - for await (const entry of exporterWalk(path, blockstore, options)) { + for await (const entry of exporterWalk(path, blockstore, { + ...options, + // extended: false + })) { ipfsRoots.push(entry.cid) terminalElement = entry } From 12f75b3755aa14f6aa036300b76711c19a13f631 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Tue, 5 Aug 2025 12:06:37 -0400 Subject: [PATCH 2/6] chore: remove unused extended: false --- packages/verified-fetch/src/plugins/plugin-handle-dag-pb.ts | 3 +-- packages/verified-fetch/src/utils/walk-path.ts | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/verified-fetch/src/plugins/plugin-handle-dag-pb.ts b/packages/verified-fetch/src/plugins/plugin-handle-dag-pb.ts index cdbc4ab6..7a83d54e 100644 --- a/packages/verified-fetch/src/plugins/plugin-handle-dag-pb.ts +++ b/packages/verified-fetch/src/plugins/plugin-handle-dag-pb.ts @@ -139,8 +139,7 @@ export class DagPbPlugin extends BasePlugin { try { const entry = await handleServerTiming('exporter-file', '', async () => exporter(resolvedCID, helia.blockstore, { signal: options?.signal, - onProgress: options?.onProgress, - // extended: false + onProgress: options?.onProgress }), withServerTiming) let firstChunk: Uint8Array diff --git a/packages/verified-fetch/src/utils/walk-path.ts b/packages/verified-fetch/src/utils/walk-path.ts index fa6e3678..13551038 100644 --- a/packages/verified-fetch/src/utils/walk-path.ts +++ b/packages/verified-fetch/src/utils/walk-path.ts @@ -25,8 +25,7 @@ async function walkPath (blockstore: ReadableStorage, path: string, options?: Pa let terminalElement: UnixFSEntry | undefined for await (const entry of exporterWalk(path, blockstore, { - ...options, - // extended: false + ...options })) { ipfsRoots.push(entry.cid) terminalElement = entry From 290cbe24e30401c4d226df6a708bd715df69272d Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Tue, 5 Aug 2025 13:13:11 -0400 Subject: [PATCH 3/6] fix: createOfflineHelia fixture passes through types --- packages/verified-fetch/test/fixtures/create-offline-helia.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/verified-fetch/test/fixtures/create-offline-helia.ts b/packages/verified-fetch/test/fixtures/create-offline-helia.ts index 27dc0543..361b07bd 100644 --- a/packages/verified-fetch/test/fixtures/create-offline-helia.ts +++ b/packages/verified-fetch/test/fixtures/create-offline-helia.ts @@ -2,9 +2,9 @@ import { createHeliaHTTP } from '@helia/http' import { MemoryBlockstore } from 'blockstore-core' import { IdentityBlockstore } from 'blockstore-core/identity' import { MemoryDatastore } from 'datastore-core' -import type { HeliaInit } from 'helia' -export async function createHelia (init: Partial = {}): Promise> { +export async function createHelia (...args: Parameters): Promise> { + const [init] = args const datastore = new MemoryDatastore() const blockstore = new IdentityBlockstore(new MemoryBlockstore()) From a02e04a8ae1a35addccc2826f4db158160832479 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Tue, 5 Aug 2025 13:19:58 -0400 Subject: [PATCH 4/6] chore: remove comma dangle --- packages/verified-fetch/src/plugins/plugin-handle-dag-pb.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/verified-fetch/src/plugins/plugin-handle-dag-pb.ts b/packages/verified-fetch/src/plugins/plugin-handle-dag-pb.ts index 7a83d54e..ad757870 100644 --- a/packages/verified-fetch/src/plugins/plugin-handle-dag-pb.ts +++ b/packages/verified-fetch/src/plugins/plugin-handle-dag-pb.ts @@ -95,7 +95,7 @@ export class DagPbPlugin extends BasePlugin { const entry = await handleServerTiming('exporter-dir', '', async () => exporter(`/ipfs/${dirCid}/${rootFilePath}`, helia.blockstore, { signal: options?.signal, - onProgress: options?.onProgress, + onProgress: options?.onProgress // extended: false }), withServerTiming) From f25fa118db9e661707d77700931bd6fe17a4c324 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Tue, 5 Aug 2025 13:25:42 -0400 Subject: [PATCH 5/6] Update packages/verified-fetch/src/plugins/plugin-handle-dag-pb.ts --- packages/verified-fetch/src/plugins/plugin-handle-dag-pb.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/verified-fetch/src/plugins/plugin-handle-dag-pb.ts b/packages/verified-fetch/src/plugins/plugin-handle-dag-pb.ts index ad757870..e1e172a7 100644 --- a/packages/verified-fetch/src/plugins/plugin-handle-dag-pb.ts +++ b/packages/verified-fetch/src/plugins/plugin-handle-dag-pb.ts @@ -96,7 +96,6 @@ export class DagPbPlugin extends BasePlugin { const entry = await handleServerTiming('exporter-dir', '', async () => exporter(`/ipfs/${dirCid}/${rootFilePath}`, helia.blockstore, { signal: options?.signal, onProgress: options?.onProgress - // extended: false }), withServerTiming) log.trace('found root file at %c/%s with cid %c', dirCid, rootFilePath, entry.cid) From 12f5a993893e2454ae86d85a9986ac22d5ead0f6 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Tue, 5 Aug 2025 13:25:48 -0400 Subject: [PATCH 6/6] Update packages/verified-fetch/src/utils/walk-path.ts --- packages/verified-fetch/src/utils/walk-path.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/verified-fetch/src/utils/walk-path.ts b/packages/verified-fetch/src/utils/walk-path.ts index 13551038..a2dfbf7b 100644 --- a/packages/verified-fetch/src/utils/walk-path.ts +++ b/packages/verified-fetch/src/utils/walk-path.ts @@ -24,9 +24,7 @@ async function walkPath (blockstore: ReadableStorage, path: string, options?: Pa const ipfsRoots: CID[] = [] let terminalElement: UnixFSEntry | undefined - for await (const entry of exporterWalk(path, blockstore, { - ...options - })) { + for await (const entry of exporterWalk(path, blockstore, options)) { ipfsRoots.push(entry.cid) terminalElement = entry }