diff --git a/packages/editor/src/browser/preference/schema.ts b/packages/editor/src/browser/preference/schema.ts index d574f2ab7f..9bc1ec31ab 100644 --- a/packages/editor/src/browser/preference/schema.ts +++ b/packages/editor/src/browser/preference/schema.ts @@ -1655,6 +1655,11 @@ const customEditorSchema: PreferenceSchemaProperties = { default: 4 * 1024 * 1024 * 1024, // 4096 MB description: '%editor.configuration.largeFileSize%', }, + 'editor.streamLargeFile': { + type: 'boolean', + default: true, + description: '%editor.configuration.streamLargeFile%', + }, 'editor.quickSuggestionsDelay': { type: 'integer', default: 100, diff --git a/packages/file-service/__tests__/browser/file-service-client.test.ts b/packages/file-service/__tests__/browser/file-service-client.test.ts index fef70a855b..51d032a0c7 100644 --- a/packages/file-service/__tests__/browser/file-service-client.test.ts +++ b/packages/file-service/__tests__/browser/file-service-client.test.ts @@ -2,24 +2,69 @@ import fs from 'fs-extra'; import temp from 'temp'; import { WSChannelHandler } from '@opensumi/ide-connection/lib/browser'; -import { DisposableCollection, FileChangeType, FileUri, UTF8 } from '@opensumi/ide-core-common'; +import { PreferenceService } from '@opensumi/ide-core-browser'; +import { + DisposableCollection, + Event, + FileChangeType, + FileSystemProviderCapabilities, + FileUri, + UTF8, +} from '@opensumi/ide-core-common'; +import { FileStat } from '@opensumi/ide-core-common/lib/types/file'; import { createBrowserInjector } from '@opensumi/ide-dev-tool/src/injector-helper'; import { FileService } from '@opensumi/ide-file-service/lib/node'; import { DiskFileSystemProvider } from '@opensumi/ide-file-service/lib/node/disk-file-system.provider'; import { WatcherProcessManagerToken } from '@opensumi/ide-file-service/lib/node/watcher-process-manager'; +import { SumiReadableStream } from '@opensumi/ide-utils/lib/stream'; import { FileServicePath, IDiskFileProvider, IFileServiceClient } from '../../src'; import { FileServiceClientModule } from '../../src/browser'; import { RecursiveFileSystemWatcher } from '../../src/node/hosted/recursive/file-service-watcher'; +function createMockStreamProvider() { + return { + capabilities: FileSystemProviderCapabilities.FileReadWrite, + onDidChangeCapabilities: Event.None, + onDidChangeFile: Event.None, + watch: jest.fn(() => 1), + stat: jest.fn(), + readFile: jest.fn(), + readFileStream: jest.fn(), + }; +} + +function createReadableStream(chunks: Uint8Array[]) { + const stream = new SumiReadableStream(); + setTimeout(() => { + chunks.forEach((chunk) => stream.emitData(chunk)); + stream.end(); + }, 0); + return stream; +} + +function createStreamStat(uri: string, size: number): FileStat { + return { + uri, + lastModification: Date.now(), + isDirectory: false, + size, + }; +} + describe('FileServiceClient should be work', () => { jest.setTimeout(10000); const injector = createBrowserInjector([FileServiceClientModule]); const toDispose = new DisposableCollection(); let fileServiceClient: IFileServiceClient; + let streamProvider: ReturnType; const track = temp.track(); const tempDir = FileUri.create(fs.realpathSync(temp.mkdirSync('file-service-client-test'))); + const preferenceGetValidMock = jest.fn(); + const preferenceServiceMock = { + getValid: preferenceGetValidMock, + }; injector.overrideProviders( { @@ -46,6 +91,10 @@ describe('FileServiceClient should be work', () => { setWatcherFileExcludes: () => void 0, }, }, + { + token: PreferenceService, + useValue: preferenceServiceMock, + }, ); beforeAll(() => { @@ -53,6 +102,8 @@ describe('FileServiceClient should be work', () => { injector.mock(RecursiveFileSystemWatcher, 'shouldUseNSFW', () => Promise.resolve(false)); fileServiceClient = injector.get(IFileServiceClient); toDispose.push(fileServiceClient.registerProvider('file', injector.get(IDiskFileProvider))); + streamProvider = createMockStreamProvider(); + toDispose.push(fileServiceClient.registerProvider('stream', streamProvider as any)); }); afterAll(async () => { @@ -232,4 +283,97 @@ describe('FileServiceClient should be work', () => { const encoding = await fileServiceClient.getEncoding(tempDir.toString()); expect(encoding).toBe(UTF8); }); + + describe('large file stream reading', () => { + const streamResourceUri = 'stream://test/large-file.txt'; + + const configureStreamPreferences = (threshold: number, enabled: boolean) => { + preferenceGetValidMock.mockImplementation((key: string, defaultValue: any) => { + if (key === 'editor.largeFile') { + return threshold; + } + if (key === 'editor.streamLargeFile') { + return enabled; + } + return defaultValue; + }); + }; + + beforeEach(() => { + preferenceGetValidMock.mockReset(); + streamProvider.stat.mockReset(); + streamProvider.readFile.mockReset(); + streamProvider.readFileStream.mockReset(); + }); + + it('reads via stream when preference enabled and size exceeds threshold', async () => { + configureStreamPreferences(10, true); + streamProvider.stat.mockResolvedValue(createStreamStat(streamResourceUri, 20)); + const chunkOne = new Uint8Array([1, 2]); + const chunkTwo = new Uint8Array([3, 4]); + streamProvider.readFileStream.mockImplementation(async () => createReadableStream([chunkOne, chunkTwo])); + streamProvider.readFile.mockResolvedValue(new Uint8Array([9])); + + const result = await fileServiceClient.readFile(streamResourceUri); + + expect(Array.from(result.content.buffer)).toEqual([1, 2, 3, 4]); + expect(streamProvider.readFileStream).toHaveBeenCalledTimes(1); + expect(streamProvider.readFile).not.toHaveBeenCalled(); + }); + + it('reads via stream when size equals threshold', async () => { + configureStreamPreferences(10, true); + streamProvider.stat.mockResolvedValue(createStreamStat(streamResourceUri, 10)); + const chunk = new Uint8Array([11]); + streamProvider.readFileStream.mockImplementation(async () => createReadableStream([chunk])); + + const result = await fileServiceClient.readFile(streamResourceUri); + + expect(streamProvider.readFileStream).toHaveBeenCalledTimes(1); + expect(streamProvider.readFile).not.toHaveBeenCalled(); + expect(Array.from(result.content.buffer)).toEqual([11]); + }); + + it('falls back to readFile when stream reading fails', async () => { + configureStreamPreferences(10, true); + streamProvider.stat.mockResolvedValue(createStreamStat(streamResourceUri, 50)); + streamProvider.readFileStream.mockImplementation(async () => { + throw new Error('stream error'); + }); + const fallbackContent = new Uint8Array([7]); + streamProvider.readFile.mockResolvedValue(fallbackContent); + + const result = await fileServiceClient.readFile(streamResourceUri); + + expect(streamProvider.readFileStream).toHaveBeenCalledTimes(1); + expect(streamProvider.readFile).toHaveBeenCalledTimes(1); + expect(Array.from(result.content.buffer)).toEqual([7]); + }); + + it('skips stream path when preference disables it', async () => { + configureStreamPreferences(10, false); + streamProvider.stat.mockResolvedValue(createStreamStat(streamResourceUri, 20)); + const fallbackContent = new Uint8Array([5, 6]); + streamProvider.readFile.mockResolvedValue(fallbackContent); + + const result = await fileServiceClient.readFile(streamResourceUri); + + expect(streamProvider.readFileStream).not.toHaveBeenCalled(); + expect(streamProvider.readFile).toHaveBeenCalledTimes(1); + expect(Array.from(result.content.buffer)).toEqual([5, 6]); + }); + + it('uses readFile when size is below threshold even if streaming enabled', async () => { + configureStreamPreferences(10, true); + streamProvider.stat.mockResolvedValue(createStreamStat(streamResourceUri, 5)); + const smallContent = new Uint8Array([8, 9]); + streamProvider.readFile.mockResolvedValue(smallContent); + + const result = await fileServiceClient.readFile(streamResourceUri); + + expect(streamProvider.readFileStream).not.toHaveBeenCalled(); + expect(streamProvider.readFile).toHaveBeenCalledTimes(1); + expect(Array.from(result.content.buffer)).toEqual([8, 9]); + }); + }); }); diff --git a/packages/file-service/src/browser/file-service-client.ts b/packages/file-service/src/browser/file-service-client.ts index 7e75446006..b4bbff2186 100644 --- a/packages/file-service/src/browser/file-service-client.ts +++ b/packages/file-service/src/browser/file-service-client.ts @@ -15,14 +15,15 @@ import { FilesChangeEvent, IDisposable, ParsedPattern, + PreferenceService, URI, Uri, parseGlob, } from '@opensumi/ide-core-browser'; -import { CorePreferences } from '@opensumi/ide-core-browser/lib/core-preferences'; import { FileSystemProviderCapabilities, IEventBus, ILogger, Schemes, isUndefined } from '@opensumi/ide-core-common'; import { IElectronMainUIService } from '@opensumi/ide-core-common/lib/electron'; import { IApplicationService } from '@opensumi/ide-core-common/lib/types/application'; +import { IReadableStream, listenReadable } from '@opensumi/ide-utils/lib/stream'; import { Iterable } from '@opensumi/monaco-editor-core/esm/vs/base/common/iterator'; import { @@ -168,7 +169,8 @@ export class FileServiceClient implements IFileServiceClient, IDisposable { this.userHomeDeferred.resolve(userHome); } - corePreferences: CorePreferences; + @Autowired(PreferenceService) + private readonly preference: PreferenceService; handlesScheme(scheme: string) { return this.registry.providers.has(scheme) || this.fsProviders.has(scheme); @@ -189,16 +191,16 @@ export class FileServiceClient implements IFileServiceClient, IDisposable { const provider = await this.getProvider(_uri.scheme); const rawContent = await provider.readFile(_uri.codeUri); const data = (rawContent as any).data || rawContent; - const buffer = BinaryBuffer.wrap(Uint8Array.from(data)); + const buffer = BinaryBuffer.wrap(data instanceof Uint8Array ? data : Uint8Array.from(data)); return { content: buffer.toString(options?.encoding) }; } async readFile(uri: string) { const _uri = this.convertUri(uri); const provider = await this.getProvider(_uri.scheme); - const rawContent = await provider.readFile(_uri.codeUri); + const rawContent = await this.doReadFile(provider, _uri.codeUri); const data = (rawContent as any).data || rawContent; - const buffer = BinaryBuffer.wrap(Uint8Array.from(data)); + const buffer = BinaryBuffer.wrap(data instanceof Uint8Array ? data : Uint8Array.from(data)); return { content: buffer }; } @@ -622,6 +624,93 @@ export class FileServiceClient implements IFileServiceClient, IDisposable { return _uri; } + private async doReadFile(provider: FileSystemProvider, uri: Uri) { + const shouldStream = await this.shouldUseReadStream(provider, uri); + if (shouldStream && provider.readFileStream) { + try { + const stream = await provider.readFileStream(uri); + return await this.collectReadableStream(stream); + } catch (error) { + this.logger?.warn('[FileServiceClient] readFileStream failed, fallback to readFile.', error); + } + } + + return await provider.readFile(uri); + } + + private async shouldUseReadStream(provider: FileSystemProvider, uri: Uri): Promise { + if (!provider.readFileStream || !this.isLargeFileStreamEnabled()) { + return false; + } + + const threshold = this.getLargeFileStreamThreshold(); + if (!threshold) { + return false; + } + + try { + const stat = await provider.stat(uri); + if (stat && typeof stat.size === 'number' && stat.size >= threshold) { + return true; + } + } catch (error) { + this.logger?.warn( + '[FileServiceClient] stat failed when deciding readFile strategy, fallback to readFile.', + error, + ); + } + + return false; + } + + private collectReadableStream(stream: IReadableStream): Promise { + return new Promise((resolve, reject) => { + const chunks: Uint8Array[] = []; + let totalLength = 0; + + listenReadable(stream, { + onData: (chunk) => { + const data = chunk instanceof Uint8Array ? chunk : Uint8Array.from(chunk as any); + chunks.push(data); + totalLength += data.byteLength; + }, + onError: (error) => reject(error), + onEnd: () => { + if (chunks.length === 0) { + resolve(new Uint8Array(0)); + return; + } + + if (chunks.length === 1) { + resolve(chunks[0]); + return; + } + + const merged = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + merged.set(chunk, offset); + offset += chunk.byteLength; + } + resolve(merged); + }, + }); + }); + } + + private getLargeFileStreamThreshold(): number | undefined { + const threshold = this.preference?.getValid('editor.largeFile', 4 * 1024 * 1024 * 1024); + if (typeof threshold === 'number' && threshold > 0) { + return threshold; + } + return undefined; + } + + private isLargeFileStreamEnabled(): boolean { + const enabled = this.preference?.getValid('editor.streamLargeFile', true); + return typeof enabled === 'undefined' ? true : enabled; + } + private updateExcludeMatcher() { this.filesExcludes.forEach((str) => { if (this.workspaceRoots.length > 0) { diff --git a/packages/i18n/src/common/en-US.lang.ts b/packages/i18n/src/common/en-US.lang.ts index 722a064c0c..af1ec605f1 100644 --- a/packages/i18n/src/common/en-US.lang.ts +++ b/packages/i18n/src/common/en-US.lang.ts @@ -565,6 +565,7 @@ export const localizationBundle = { 'preference.editor.maxTokenizationLineLength': 'Max Tokenization Line Length', 'preference.editor.quickSuggestionsDelay': 'Quick suggestions show delay (in ms) Defaults to 10 (ms)', 'preference.editor.largeFile': 'Large File Size', + 'preference.editor.streamLargeFile': 'Use Stream For Large File', 'preference.editor.formatOnPaste': 'Format On Paste', 'preference.files.eol': 'EOL', 'preference.files.trimFinalNewlines': 'Trim Final Newlines', @@ -607,6 +608,8 @@ export const localizationBundle = { "Controls the delay in ms after which a dirty file is saved automatically. Only applies when `#editor.formatOnSave#` is set to 'Save After Delay'.", 'editor.configuration.forceReadOnly': 'If Enable readOnly', 'editor.configuration.largeFileSize': 'Custom size of the large file (B)', + 'editor.configuration.streamLargeFile': + 'Read large files via stream when file size exceeds the configured threshold.', 'editor.configuration.fontFamily': 'Controls the font family.', 'editor.configuration.fontWeight': 'Controls the font weight. Accepts "normal" and "bold" keywords or numbers between 1 and 1000.', diff --git a/packages/i18n/src/common/zh-CN.lang.ts b/packages/i18n/src/common/zh-CN.lang.ts index 32045bc712..9f658f169c 100644 --- a/packages/i18n/src/common/zh-CN.lang.ts +++ b/packages/i18n/src/common/zh-CN.lang.ts @@ -518,6 +518,7 @@ export const localizationBundle = { 'preference.editor.maxTokenizationLineLength': '最大解析标识长度', 'preference.editor.quickSuggestionsDelay': '智能提示延迟（毫秒）', 'preference.editor.largeFile': '超大文件尺寸', + 'preference.editor.streamLargeFile': '大文件使用流式读取', 'preference.files.eol': '文件行尾字符', 'preference.files.trimFinalNewlines': '移除最后的换行符', 'preference.files.trimTrailingWhitespace': '移除结尾空格', @@ -1097,6 +1098,7 @@ export const localizationBundle = { '控制 Tab 缩进等于的空格数。若启用 `#editor.detectIndentation#`，该设置可能会被覆盖', 'editor.configuration.fontWeight': '控制字体粗细，接收 "normal" 和 "bold" 关键词或者 1 到 1000 数值。', 'editor.configuration.largeFileSize': '控制超大文件的自定义体积。(单位：B)', + 'editor.configuration.streamLargeFile': '当文件超过阈值时，使用流式读取以降低一次性内存占用。', 'editor.configuration.preferredFormatter': '配置优先使用的格式化器。', 'editor.configuration.wrapTab': '控制当编辑器 Tab 超过可用空间时，是否使用换行来代替滚动模式。', 'editor.configuration.enablePreviewFromCodeNavigation': '控制当代码导航从其出发时，编辑器是否仍处于预览模式。', diff --git a/packages/preferences/src/browser/preference-settings.service.ts b/packages/preferences/src/browser/preference-settings.service.ts index a3b497c7f9..955147b68b 100644 --- a/packages/preferences/src/browser/preference-settings.service.ts +++ b/packages/preferences/src/browser/preference-settings.service.ts @@ -730,6 +730,7 @@ export const defaultSettingSections: { // {id: 'editor.forceReadOnly', localized: 'preference.editor.forceReadOnly'}, { id: 'editor.maxTokenizationLineLength', localized: 'preference.editor.maxTokenizationLineLength' }, { id: 'editor.largeFile', localized: 'preference.editor.largeFile' }, + { id: 'editor.streamLargeFile', localized: 'preference.editor.streamLargeFile' }, { id: 'editor.readonlyFiles', localized: 'preference.editor.readonlyFiles' }, { id: 'editor.bracketPairColorization.enabled',