Allow save to disk for mod zip >2GB

Refs TS-2082

Author: ebkr

src/providers/generic/file/FsProvider.ts

@@ -16,6 +16,7 @@ export default abstract class FsProvider {
     }
 
     public abstract writeFile(path: string, content: string | Buffer): Promise<void>;
+    public abstract writeStreamToFile(path: string, content: ReadableStream): Promise<void>;
     public abstract readFile(path: string): Promise<Buffer>;
     public abstract readdir(path: string): Promise<string[]>;
     public abstract rmdir(path: string): Promise<void>;

src/providers/generic/file/NodeFs.ts

@@ -91,6 +91,25 @@ export default class NodeFs extends FsProvider {
         })
     }
 
+    async writeStreamToFile(path: string, content: ReadableStream): Promise<void> {
+        return new Promise((resolve, reject) => {
+            NodeFs.lock.acquire(path, async () => {
+                const writeStream = fs.createWriteStream(path, { flags : 'w' });
+                writeStream.on('finish', () => resolve());
+
+                const reader = content.getReader();
+                while (true) {
+                    const line = await reader.read();
+                    if (line.done) {
+                        break;
+                    }
+                    writeStream.write(line.value);
+                }
+                writeStream.end();
+            }).catch(reject);
+        });
+    }
+
     async rename(path: string, newPath: string): Promise<void> {
         return new Promise((resolve, reject) => {
             NodeFs.lock.acquire([path, newPath], async () => {

src/providers/ror2/downloading/ThunderstoreDownloaderProvider.ts

@@ -106,7 +106,7 @@ export default abstract class ThunderstoreDownloaderProvider {
      * @param combo     The mod being downloaded.
      * @param callback  Callback on if saving and extracting has been performed correctly. An error is provided if success is false.
      */
-    public abstract saveToFile(response: Buffer, combo: ThunderstoreCombo, callback: (success: boolean, error?: R2Error) => void): void;
+    public abstract saveToFile(response: ReadableStream, combo: ThunderstoreCombo, callback: (success: boolean, error?: R2Error) => void): void;
 
     /**
      * Check the cache to see if the mod has already been downloaded.

src/r2mm/downloading/BetterThunderstoreDownloader.ts

@@ -239,18 +239,18 @@ export default class BetterThunderstoreDownloader extends ThunderstoreDownloader
             onDownloadProgress: progress => {
                 callback((progress.loaded / progress.total) * 100, StatusEnum.PENDING, null);
             },
-            responseType: 'arraybuffer',
+            responseType: 'blob',
             headers: {
                 'Content-Type': 'application/zip',
                 'Access-Control-Allow-Origin': '*'
-            }
+            },
         });
     }
 
     private async _saveDownloadResponse(response: AxiosResponse, combo: ThunderstoreCombo, callback: (progress: number, status: number, err: R2Error | null) => void): Promise<void> {
-        const buf: Buffer = Buffer.from(response.data)
+        const dataStream = response.data.stream() as ReadableStream;
         callback(100, StatusEnum.PENDING, null);
-        await this.saveToFile(buf, combo, (success: boolean, error?: R2Error) => {
+        await this.saveToFile(dataStream, combo, (success: boolean, error?: R2Error) => {
             if (success) {
                 callback(100, StatusEnum.SUCCESS, error || null);
             } else {
@@ -259,27 +259,28 @@ export default class BetterThunderstoreDownloader extends ThunderstoreDownloader
         });
     }
 
-    public async saveToFile(response: Buffer, combo: ThunderstoreCombo, callback: (success: boolean, error?: R2Error) => void) {
+    public async saveToFile(data: ReadableStream, combo: ThunderstoreCombo, callback: (success: boolean, error?: R2Error) => void) {
         const fs = FsProvider.instance;
         const cacheDirectory = path.join(PathResolver.MOD_ROOT, 'cache');
         try {
             if (! await fs.exists(path.join(cacheDirectory, combo.getMod().getFullName()))) {
                 await fs.mkdirs(path.join(cacheDirectory, combo.getMod().getFullName()));
             }
-            await fs.writeFile(path.join(
+            await fs.writeStreamToFile(path.join(
                 cacheDirectory,
                 combo.getMod().getFullName(),
                 combo.getVersion().getVersionNumber().toString() + '.zip'
-            ), response);
+            ), data);
             await ZipExtract.extractAndDelete(
                 path.join(cacheDirectory, combo.getMod().getFullName()),
                 combo.getVersion().getVersionNumber().toString() + '.zip',
                 combo.getVersion().getVersionNumber().toString(),
                 callback
             );
         } catch(e) {
+            const err = e as Error;
             callback(false, new FileWriteError(
-                'File write error',
+                err.name,
                 `Failed to write downloaded zip of ${combo.getMod().getFullName()} cache folder. \nReason: ${(e as Error).message}`,
                 `Try running ${ManagerInformation.APP_NAME} as an administrator`
             ));

test/jest/__tests__/stubs/providers/InMemory.FsProvider.ts

@@ -225,6 +225,9 @@ export default class InMemoryFsProvider extends FsProvider {
         parent.nodes.push(newFile);
     }
 
+    async writeStreamToFile(path: string, content: ReadableStream): Promise<void> {
+        throw new Error("writeStreamToFile: not implemented");
+    }
 
     async setModifiedTime(file: string, time: Date): Promise<void> {
         const found = this.findFileType(file);

test/jest/__tests__/stubs/providers/stub.FsProvider.ts

@@ -1,69 +1,77 @@
 import FsProvider from '../../../../../src/providers/generic/file/FsProvider';
-import StatInterface from '../../../../../src/providers/generic/file/StatInterface';
+import Dexie from 'dexie';
+import { types } from 'sass';
+import * as Buffer from 'buffer';
+import Error = types.Error;
+import Promise = Dexie.Promise;
 
 export default class StubFsProvider extends FsProvider {
 
-    async base64FromZip(path: string): Promise<string> {
+    base64FromZip = async (path: string) => {
         throw new Error("Stub access must be mocked or spied");
-    }
+    };
 
-    async chmod(path: string, mode: string | number): Promise<void> {
+    chmod = async (path: string, mode: string | number) => {
         throw new Error("Stub access must be mocked or spied");
-    }
+    };
 
-    async copyFile(from: string, to: string): Promise<void> {
+    copyFile = async (from: string, to: string) => {
         throw new Error("Stub access must be mocked or spied");
-    }
+    };
 
-    async copyFolder(from: string, to: string): Promise<void> {
+    copyFolder = async (from: string, to: string) => {
         throw new Error("Stub access must be mocked or spied");
-    }
+    };
 
-    async exists(path: string): Promise<boolean> {
+    exists = async (path: string) => {
         throw new Error("Stub access must be mocked or spied");
-    }
+    };
 
-    async lstat(path: string): Promise<StatInterface> {
+    lstat = async (path: string) => {
         throw new Error("Stub access must be mocked or spied");
-    }
+    };
 
-    async mkdirs(path: string): Promise<void> {
+    mkdirs = async (path: string) => {
         throw new Error("Stub access must be mocked or spied");
-    }
+    };
 
-    async readFile(path: string): Promise<Buffer> {
+    readFile = async (path: string) => {
         throw new Error("Stub access must be mocked or spied");
-    }
+    };
 
-    async readdir(path: string): Promise<string[]> {
+    readdir = async (path: string) => {
         throw new Error("Stub access must be mocked or spied");
-    }
+    };
 
-    async realpath(path: string): Promise<string> {
+    realpath = async (path: string) => {
         throw new Error("Stub access must be mocked or spied");
-    }
+    };
 
-    async rename(path: string, newPath: string): Promise<void> {
+    rename = async (path: string, newPath: string) => {
         throw new Error("Stub access must be mocked or spied");
-    }
+    };
 
-    async rmdir(path: string): Promise<void> {
+    rmdir = async (path: string) => {
         throw new Error("Stub access must be mocked or spied");
-    }
+    };
 
-    async stat(path: string): Promise<StatInterface> {
+    stat = async (path: string) => {
         throw new Error("Stub access must be mocked or spied");
-    }
+    };
 
-    async unlink(path: string): Promise<void> {
+    unlink = async (path: string) => {
         throw new Error("Stub access must be mocked or spied");
-    }
+    };
 
-    async writeFile(path: string, content: string | Buffer): Promise<void> {
+    writeFile = async (path: string, content: string | Buffer) => {
         throw new Error("Stub access must be mocked or spied");
-    }
+    };
 
-    async setModifiedTime(path: string, time: Date): Promise<void> {
+    writeStreamToFile = async (path: string, content: ReadableStream) => {
         throw new Error("Stub access must be mocked or spied");
     }
+
+    setModifiedTime = async (path: string, time: Date) => {
+        throw new Error("Stub access must be mocked or spied");
+    };
 }