feat: support stream read large-file reduce memory usage

packages/editor/src/browser/preference/schema.ts

@@ -1655,6 +1655,11 @@ const customEditorSchema: PreferenceSchemaProperties = {
     default: 4 * 1024 * 1024 * 1024, // 4096 MB
     description: '%editor.configuration.largeFileSize%',
   },
+  'editor.streamLargeFile': {
+    type: 'boolean',
+    default: true,
+    description: '%editor.configuration.streamLargeFile%',
+  },
   'editor.quickSuggestionsDelay': {
     type: 'integer',
     default: 100,

packages/file-service/__tests__/browser/file-service-client.test.ts

@@ -2,24 +2,69 @@ import fs from 'fs-extra';
 import temp from 'temp';
 
 import { WSChannelHandler } from '@opensumi/ide-connection/lib/browser';
-import { DisposableCollection, FileChangeType, FileUri, UTF8 } from '@opensumi/ide-core-common';
+import { PreferenceService } from '@opensumi/ide-core-browser';
+import {
+  DisposableCollection,
+  Event,
+  FileChangeType,
+  FileSystemProviderCapabilities,
+  FileUri,
+  UTF8,
+} from '@opensumi/ide-core-common';
+import { FileStat } from '@opensumi/ide-core-common/lib/types/file';
 import { createBrowserInjector } from '@opensumi/ide-dev-tool/src/injector-helper';
 import { FileService } from '@opensumi/ide-file-service/lib/node';
 import { DiskFileSystemProvider } from '@opensumi/ide-file-service/lib/node/disk-file-system.provider';
 import { WatcherProcessManagerToken } from '@opensumi/ide-file-service/lib/node/watcher-process-manager';
+import { SumiReadableStream } from '@opensumi/ide-utils/lib/stream';
 
 import { FileServicePath, IDiskFileProvider, IFileServiceClient } from '../../src';
 import { FileServiceClientModule } from '../../src/browser';
 import { RecursiveFileSystemWatcher } from '../../src/node/hosted/recursive/file-service-watcher';
 
+function createMockStreamProvider() {
+  return {
+    capabilities: FileSystemProviderCapabilities.FileReadWrite,
+    onDidChangeCapabilities: Event.None,
+    onDidChangeFile: Event.None,
+    watch: jest.fn(() => 1),
+    stat: jest.fn(),
+    readFile: jest.fn(),
+    readFileStream: jest.fn(),
+  };
+}
+
+function createReadableStream(chunks: Uint8Array[]) {
+  const stream = new SumiReadableStream<Uint8Array>();
+  setTimeout(() => {
+    chunks.forEach((chunk) => stream.emitData(chunk));
+    stream.end();
+  }, 0);
+  return stream;
+}
+
+function createStreamStat(uri: string, size: number): FileStat {
+  return {
+    uri,
+    lastModification: Date.now(),
+    isDirectory: false,
+    size,
+  };
+}
+
 describe('FileServiceClient should be work', () => {
   jest.setTimeout(10000);
 
   const injector = createBrowserInjector([FileServiceClientModule]);
   const toDispose = new DisposableCollection();
   let fileServiceClient: IFileServiceClient;
+  let streamProvider: ReturnType<typeof createMockStreamProvider>;
   const track = temp.track();
   const tempDir = FileUri.create(fs.realpathSync(temp.mkdirSync('file-service-client-test')));
+  const preferenceGetValidMock = jest.fn();
+  const preferenceServiceMock = {
+    getValid: preferenceGetValidMock,
+  };
 
   injector.overrideProviders(
     {
@@ -46,13 +91,19 @@ describe('FileServiceClient should be work', () => {
         setWatcherFileExcludes: () => void 0,
       },
     },
+    {
+      token: PreferenceService,
+      useValue: preferenceServiceMock,
+    },
   );
 
   beforeAll(() => {
     // @ts-ignore
     injector.mock(RecursiveFileSystemWatcher, 'shouldUseNSFW', () => Promise.resolve(false));
     fileServiceClient = injector.get(IFileServiceClient);
     toDispose.push(fileServiceClient.registerProvider('file', injector.get(IDiskFileProvider)));
+    streamProvider = createMockStreamProvider();
+    toDispose.push(fileServiceClient.registerProvider('stream', streamProvider as any));
   });
 
   afterAll(async () => {
@@ -232,4 +283,97 @@ describe('FileServiceClient should be work', () => {
     const encoding = await fileServiceClient.getEncoding(tempDir.toString());
     expect(encoding).toBe(UTF8);
   });
+
+  describe('large file stream reading', () => {
+    const streamResourceUri = 'stream://test/large-file.txt';
+
+    const configureStreamPreferences = (threshold: number, enabled: boolean) => {
+      preferenceGetValidMock.mockImplementation((key: string, defaultValue: any) => {
+        if (key === 'editor.largeFile') {
+          return threshold;
+        }
+        if (key === 'editor.streamLargeFile') {
+          return enabled;
+        }
+        return defaultValue;
+      });
+    };
+
+    beforeEach(() => {
+      preferenceGetValidMock.mockReset();
+      streamProvider.stat.mockReset();
+      streamProvider.readFile.mockReset();
+      streamProvider.readFileStream.mockReset();
+    });
+
+    it('reads via stream when preference enabled and size exceeds threshold', async () => {
+      configureStreamPreferences(10, true);
+      streamProvider.stat.mockResolvedValue(createStreamStat(streamResourceUri, 20));
+      const chunkOne = new Uint8Array([1, 2]);
+      const chunkTwo = new Uint8Array([3, 4]);
+      streamProvider.readFileStream.mockImplementation(async () => createReadableStream([chunkOne, chunkTwo]));
+      streamProvider.readFile.mockResolvedValue(new Uint8Array([9]));
+
+      const result = await fileServiceClient.readFile(streamResourceUri);
+
+      expect(Array.from(result.content.buffer)).toEqual([1, 2, 3, 4]);
+      expect(streamProvider.readFileStream).toHaveBeenCalledTimes(1);
+      expect(streamProvider.readFile).not.toHaveBeenCalled();
+    });
+
+    it('reads via stream when size equals threshold', async () => {
+      configureStreamPreferences(10, true);
+      streamProvider.stat.mockResolvedValue(createStreamStat(streamResourceUri, 10));
+      const chunk = new Uint8Array([11]);
+      streamProvider.readFileStream.mockImplementation(async () => createReadableStream([chunk]));
+
+      const result = await fileServiceClient.readFile(streamResourceUri);
+
+      expect(streamProvider.readFileStream).toHaveBeenCalledTimes(1);
+      expect(streamProvider.readFile).not.toHaveBeenCalled();
+      expect(Array.from(result.content.buffer)).toEqual([11]);
+    });
+
+    it('falls back to readFile when stream reading fails', async () => {
+      configureStreamPreferences(10, true);
+      streamProvider.stat.mockResolvedValue(createStreamStat(streamResourceUri, 50));
+      streamProvider.readFileStream.mockImplementation(async () => {
+        throw new Error('stream error');
+      });
+      const fallbackContent = new Uint8Array([7]);
+      streamProvider.readFile.mockResolvedValue(fallbackContent);
+
+      const result = await fileServiceClient.readFile(streamResourceUri);
+
+      expect(streamProvider.readFileStream).toHaveBeenCalledTimes(1);
+      expect(streamProvider.readFile).toHaveBeenCalledTimes(1);
+      expect(Array.from(result.content.buffer)).toEqual([7]);
+    });
+
+    it('skips stream path when preference disables it', async () => {
+      configureStreamPreferences(10, false);
+      streamProvider.stat.mockResolvedValue(createStreamStat(streamResourceUri, 20));
+      const fallbackContent = new Uint8Array([5, 6]);
+      streamProvider.readFile.mockResolvedValue(fallbackContent);
+
+      const result = await fileServiceClient.readFile(streamResourceUri);
+
+      expect(streamProvider.readFileStream).not.toHaveBeenCalled();
+      expect(streamProvider.readFile).toHaveBeenCalledTimes(1);
+      expect(Array.from(result.content.buffer)).toEqual([5, 6]);
+    });
+
+    it('uses readFile when size is below threshold even if streaming enabled', async () => {
+      configureStreamPreferences(10, true);
+      streamProvider.stat.mockResolvedValue(createStreamStat(streamResourceUri, 5));
+      const smallContent = new Uint8Array([8, 9]);
+      streamProvider.readFile.mockResolvedValue(smallContent);
+
+      const result = await fileServiceClient.readFile(streamResourceUri);
+
+      expect(streamProvider.readFileStream).not.toHaveBeenCalled();
+      expect(streamProvider.readFile).toHaveBeenCalledTimes(1);
+      expect(Array.from(result.content.buffer)).toEqual([8, 9]);
+    });
+  });
 });

packages/file-service/src/browser/file-service-client.ts

@@ -15,14 +15,15 @@ import {
   FilesChangeEvent,
   IDisposable,
   ParsedPattern,
+  PreferenceService,
   URI,
   Uri,
   parseGlob,
 } from '@opensumi/ide-core-browser';
-import { CorePreferences } from '@opensumi/ide-core-browser/lib/core-preferences';
 import { FileSystemProviderCapabilities, IEventBus, ILogger, Schemes, isUndefined } from '@opensumi/ide-core-common';
 import { IElectronMainUIService } from '@opensumi/ide-core-common/lib/electron';
 import { IApplicationService } from '@opensumi/ide-core-common/lib/types/application';
+import { IReadableStream, listenReadable } from '@opensumi/ide-utils/lib/stream';
 import { Iterable } from '@opensumi/monaco-editor-core/esm/vs/base/common/iterator';
 
 import {
@@ -168,7 +169,8 @@ export class FileServiceClient implements IFileServiceClient, IDisposable {
     this.userHomeDeferred.resolve(userHome);
   }
 
-  corePreferences: CorePreferences;
+  @Autowired(PreferenceService)
+  private readonly preference: PreferenceService;
 
   handlesScheme(scheme: string) {
     return this.registry.providers.has(scheme) || this.fsProviders.has(scheme);
@@ -189,16 +191,16 @@ export class FileServiceClient implements IFileServiceClient, IDisposable {
     const provider = await this.getProvider(_uri.scheme);
     const rawContent = await provider.readFile(_uri.codeUri);
     const data = (rawContent as any).data || rawContent;
-    const buffer = BinaryBuffer.wrap(Uint8Array.from(data));
+    const buffer = BinaryBuffer.wrap(data instanceof Uint8Array ? data : Uint8Array.from(data));
     return { content: buffer.toString(options?.encoding) };
   }
 
   async readFile(uri: string) {
     const _uri = this.convertUri(uri);
     const provider = await this.getProvider(_uri.scheme);
-    const rawContent = await provider.readFile(_uri.codeUri);
+    const rawContent = await this.doReadFile(provider, _uri.codeUri);
     const data = (rawContent as any).data || rawContent;
-    const buffer = BinaryBuffer.wrap(Uint8Array.from(data));
+    const buffer = BinaryBuffer.wrap(data instanceof Uint8Array ? data : Uint8Array.from(data));
     return { content: buffer };
   }
 
@@ -622,6 +624,93 @@ export class FileServiceClient implements IFileServiceClient, IDisposable {
     return _uri;
   }
 
+  private async doReadFile(provider: FileSystemProvider, uri: Uri) {
+    const shouldStream = await this.shouldUseReadStream(provider, uri);
+    if (shouldStream && provider.readFileStream) {
+      try {
+        const stream = await provider.readFileStream(uri);
+        return await this.collectReadableStream(stream);
+      } catch (error) {
+        this.logger?.warn('[FileServiceClient] readFileStream failed, fallback to readFile.', error);
+      }
+    }
+
+    return await provider.readFile(uri);
+  }
+
+  private async shouldUseReadStream(provider: FileSystemProvider, uri: Uri): Promise<boolean> {
+    if (!provider.readFileStream || !this.isLargeFileStreamEnabled()) {
+      return false;
+    }
+
+    const threshold = this.getLargeFileStreamThreshold();
+    if (!threshold) {
+      return false;
+    }
+
+    try {
+      const stat = await provider.stat(uri);
+      if (stat && typeof stat.size === 'number' && stat.size >= threshold) {
+        return true;
+      }
+    } catch (error) {
+      this.logger?.warn(
+        '[FileServiceClient] stat failed when deciding readFile strategy, fallback to readFile.',
+        error,
+      );
+    }
+
+    return false;
+  }
+
+  private collectReadableStream(stream: IReadableStream<Uint8Array>): Promise<Uint8Array> {
+    return new Promise((resolve, reject) => {
+      const chunks: Uint8Array[] = [];
+      let totalLength = 0;
+
+      listenReadable(stream, {
+        onData: (chunk) => {
+          const data = chunk instanceof Uint8Array ? chunk : Uint8Array.from(chunk as any);
+          chunks.push(data);
+          totalLength += data.byteLength;
+        },
+        onError: (error) => reject(error),
+        onEnd: () => {
+          if (chunks.length === 0) {
+            resolve(new Uint8Array(0));
+            return;
+          }
+
+          if (chunks.length === 1) {
+            resolve(chunks[0]);
+            return;
+          }
+
+          const merged = new Uint8Array(totalLength);
+          let offset = 0;
+          for (const chunk of chunks) {
+            merged.set(chunk, offset);
+            offset += chunk.byteLength;
+          }
+          resolve(merged);
+        },
+      });
+    });
+  }
+
+  private getLargeFileStreamThreshold(): number | undefined {
+    const threshold = this.preference?.getValid<number>('editor.largeFile', 4 * 1024 * 1024 * 1024);
+    if (typeof threshold === 'number' && threshold > 0) {
+      return threshold;
+    }
+    return undefined;
+  }
+
+  private isLargeFileStreamEnabled(): boolean {
+    const enabled = this.preference?.getValid<boolean>('editor.streamLargeFile', true);
+    return typeof enabled === 'undefined' ? true : enabled;
+  }
+
   private updateExcludeMatcher() {
     this.filesExcludes.forEach((str) => {
       if (this.workspaceRoots.length > 0) {

packages/i18n/src/common/en-US.lang.ts

@@ -565,6 +565,7 @@ export const localizationBundle = {
     'preference.editor.maxTokenizationLineLength': 'Max Tokenization Line Length',
     'preference.editor.quickSuggestionsDelay': 'Quick suggestions show delay (in ms) Defaults to 10 (ms)',
     'preference.editor.largeFile': 'Large File Size',
+    'preference.editor.streamLargeFile': 'Use Stream For Large File',
     'preference.editor.formatOnPaste': 'Format On Paste',
     'preference.files.eol': 'EOL',
     'preference.files.trimFinalNewlines': 'Trim Final Newlines',
@@ -607,6 +608,8 @@ export const localizationBundle = {
       "Controls the delay in ms after which a dirty file is saved automatically. Only applies when `#editor.formatOnSave#` is set to 'Save After Delay'.",
     'editor.configuration.forceReadOnly': 'If Enable readOnly',
     'editor.configuration.largeFileSize': 'Custom size of the large file (B)',
+    'editor.configuration.streamLargeFile':
+      'Read large files via stream when file size exceeds the configured threshold.',
     'editor.configuration.fontFamily': 'Controls the font family.',
     'editor.configuration.fontWeight':
       'Controls the font weight. Accepts "normal" and "bold" keywords or numbers between 1 and 1000.',

packages/i18n/src/common/zh-CN.lang.ts

@@ -518,6 +518,7 @@ export const localizationBundle = {
     'preference.editor.maxTokenizationLineLength': '最大解析标识长度',
     'preference.editor.quickSuggestionsDelay': '智能提示延迟（毫秒）',
     'preference.editor.largeFile': '超大文件尺寸',
+    'preference.editor.streamLargeFile': '大文件使用流式读取',
     'preference.files.eol': '文件行尾字符',
     'preference.files.trimFinalNewlines': '移除最后的换行符',
     'preference.files.trimTrailingWhitespace': '移除结尾空格',
@@ -1097,6 +1098,7 @@ export const localizationBundle = {
       '控制 Tab 缩进等于的空格数。若启用 `#editor.detectIndentation#`，该设置可能会被覆盖',
     'editor.configuration.fontWeight': '控制字体粗细，接收 "normal" 和 "bold" 关键词或者 1 到 1000 数值。',
     'editor.configuration.largeFileSize': '控制超大文件的自定义体积。(单位：B)',
+    'editor.configuration.streamLargeFile': '当文件超过阈值时，使用流式读取以降低一次性内存占用。',
     'editor.configuration.preferredFormatter': '配置优先使用的格式化器。',
     'editor.configuration.wrapTab': '控制当编辑器 Tab 超过可用空间时，是否使用换行来代替滚动模式。',
     'editor.configuration.enablePreviewFromCodeNavigation': '控制当代码导航从其出发时，编辑器是否仍处于预览模式。',

packages/preferences/src/browser/preference-settings.service.ts

@@ -730,6 +730,7 @@ export const defaultSettingSections: {
         // {id: 'editor.forceReadOnly', localized: 'preference.editor.forceReadOnly'},
         { id: 'editor.maxTokenizationLineLength', localized: 'preference.editor.maxTokenizationLineLength' },
         { id: 'editor.largeFile', localized: 'preference.editor.largeFile' },
+        { id: 'editor.streamLargeFile', localized: 'preference.editor.streamLargeFile' },
         { id: 'editor.readonlyFiles', localized: 'preference.editor.readonlyFiles' },
         {
           id: 'editor.bracketPairColorization.enabled',