diff --git a/.changeset/cli-image-paste-support.md b/.changeset/cli-image-paste-support.md new file mode 100644 index 00000000000..6352b455d90 --- /dev/null +++ b/.changeset/cli-image-paste-support.md @@ -0,0 +1,9 @@ +--- +"@kilocode/cli": patch +--- + +Add image paste support to CLI + +- Allow Ctrl+V in the CLI to paste clipboard images, attach them as [Image #N], and send them with messages (macOS only, with status feedback and cleanup) +- Add image mention parsing (@path and [Image #N]) so pasted or referenced images are included when sending messages +- Split media code into a dedicated module with platform-specific clipboard handlers and image utilities diff --git a/.kilocode/rules/rules.md b/.kilocode/rules/rules.md index e8221236f20..8449b115f66 100644 --- a/.kilocode/rules/rules.md +++ b/.kilocode/rules/rules.md @@ -12,7 +12,9 @@ - Backend tests: `cd src && pnpm test path/to/test-file` (don't include `src/` in path) - UI tests: `cd webview-ui && pnpm test src/path/to/test-file` - Example: For `src/tests/user.spec.ts`, run `cd src && pnpm test tests/user.spec.ts` NOT `pnpm test src/tests/user.spec.ts` - - **Test File Naming Convention**: Use `.spec.ts` or `.spec.tsx` for test files, NOT `.test.ts` or `.test.tsx`. This is the project standard. + - **Test File Naming Convention**: + - Monorepo default: `.spec.ts` / `.spec.tsx` + - CLI package exception: `.test.ts` / `.test.tsx` (match existing CLI convention) 2. Lint Rules: diff --git a/cli/src/media/__tests__/atMentionParser.test.ts b/cli/src/media/__tests__/atMentionParser.test.ts new file mode 100644 index 00000000000..144e2298777 --- /dev/null +++ b/cli/src/media/__tests__/atMentionParser.test.ts @@ -0,0 +1,271 @@ +import { + parseAtMentions, + extractImagePaths, + removeImageMentions, + reconstructText, + type ParsedSegment, +} from "../atMentionParser" + +describe("atMentionParser", () => { + describe("parseAtMentions", () => { + it("should parse simple @ mentions", () => { + const result = parseAtMentions("Check @./image.png please") + + expect(result.paths).toEqual(["./image.png"]) + expect(result.imagePaths).toEqual(["./image.png"]) + expect(result.otherPaths).toEqual([]) + expect(result.segments).toHaveLength(3) + }) + + it("should parse multiple @ mentions", () => { + const result = parseAtMentions("Look at @./first.png and @./second.jpg") + + expect(result.paths).toEqual(["./first.png", "./second.jpg"]) + expect(result.imagePaths).toEqual(["./first.png", "./second.jpg"]) + }) + + it("should distinguish image and non-image paths", () => { + const result = parseAtMentions("Check @./code.ts and @./screenshot.png") + + expect(result.paths).toEqual(["./code.ts", "./screenshot.png"]) + expect(result.imagePaths).toEqual(["./screenshot.png"]) + expect(result.otherPaths).toEqual(["./code.ts"]) + }) + + it("should handle quoted paths with spaces", () => { + const result = parseAtMentions('Look at @"path with spaces/image.png"') + + expect(result.paths).toEqual(["path with spaces/image.png"]) + expect(result.imagePaths).toEqual(["path with spaces/image.png"]) + }) + + it("should handle single-quoted paths", () => { + const result = parseAtMentions("Look at @'path with spaces/image.png'") + + expect(result.paths).toEqual(["path with spaces/image.png"]) + }) + + it("should handle escaped spaces in paths", () => { + const result = parseAtMentions("Look at @path\\ with\\ spaces/image.png") + + expect(result.paths).toEqual(["path with spaces/image.png"]) + }) + + it("should stop at path terminators", () => { + const result = parseAtMentions("Check @./image.png, then @./other.jpg") + + expect(result.paths).toEqual(["./image.png", "./other.jpg"]) + }) + + it("should handle @ at end of string", () => { + const result = parseAtMentions("End with @") + + expect(result.paths).toEqual([]) + expect(result.segments).toHaveLength(1) + }) + + it("should handle text without @ mentions", () => { + const result = parseAtMentions("Just regular text without mentions") + + expect(result.paths).toEqual([]) + expect(result.segments).toHaveLength(1) + expect(result.segments[0]).toMatchObject({ + type: "text", + content: "Just regular text without mentions", + }) + }) + + it("should handle absolute paths", () => { + const result = parseAtMentions("Check @/absolute/path/image.png") + + expect(result.paths).toEqual(["/absolute/path/image.png"]) + }) + + it("should handle relative paths with parent directory", () => { + const result = parseAtMentions("Check @../parent/image.png") + + expect(result.paths).toEqual(["../parent/image.png"]) + }) + + it("should preserve segment positions", () => { + const input = "Start @./image.png end" + const result = parseAtMentions(input) + + expect(result.segments[0]).toMatchObject({ + type: "text", + content: "Start ", + startIndex: 0, + endIndex: 6, + }) + expect(result.segments[1]).toMatchObject({ + type: "atPath", + content: "./image.png", + startIndex: 6, + endIndex: 18, + }) + expect(result.segments[2]).toMatchObject({ + type: "text", + content: " end", + startIndex: 18, + endIndex: 22, + }) + }) + + it("should handle @ in email addresses (not a file path)", () => { + // @ followed by typical email pattern should be parsed but not as an image + const result = parseAtMentions("Email: test@example.com") + + // It will try to parse but example.com is not an image + expect(result.imagePaths).toEqual([]) + }) + + it("should handle multiple @ mentions consecutively", () => { + const result = parseAtMentions("@./a.png@./b.png") + + // Without whitespace separator, @ is part of the path + // This is expected behavior - paths need whitespace separation + expect(result.paths).toHaveLength(1) + expect(result.paths[0]).toBe("./a.png@./b.png") + }) + + it("should ignore trailing punctuation when parsing image paths", () => { + const result = parseAtMentions("Check @./image.png? please and @./second.jpg.") + + expect(result.imagePaths).toEqual(["./image.png", "./second.jpg"]) + expect(result.otherPaths).toEqual([]) + }) + }) + + describe("extractImagePaths", () => { + it("should extract only image paths", () => { + const paths = extractImagePaths("Check @./code.ts and @./image.png and @./doc.md") + + expect(paths).toEqual(["./image.png"]) + }) + + it("should return empty array for text without images", () => { + const paths = extractImagePaths("No images here, just @./file.ts") + + expect(paths).toEqual([]) + }) + + it("should handle all supported image formats", () => { + const paths = extractImagePaths("@./a.png @./b.jpg @./c.jpeg @./d.webp") + + expect(paths).toEqual(["./a.png", "./b.jpg", "./c.jpeg", "./d.webp"]) + }) + }) + + describe("removeImageMentions", () => { + it("should remove image mentions from text", () => { + const result = removeImageMentions("Check @./image.png please") + + expect(result).toBe("Check please") + }) + + it("should preserve non-image mentions", () => { + const result = removeImageMentions("Check @./code.ts and @./image.png") + + expect(result).toBe("Check @./code.ts and ") + }) + + it("should use custom placeholder", () => { + const result = removeImageMentions("Check @./image.png please", "[image]") + + expect(result).toBe("Check [image] please") + }) + + it("should handle multiple image mentions", () => { + const result = removeImageMentions("@./a.png and @./b.jpg here") + + expect(result).toBe(" and here") + }) + + it("should not collapse newlines or indentation", () => { + const input = "Line1\n @./img.png\nLine3" + const result = removeImageMentions(input) + + expect(result).toBe("Line1\n \nLine3") + }) + }) + + describe("reconstructText", () => { + it("should reconstruct text from segments", () => { + const segments: ParsedSegment[] = [ + { type: "text", content: "Hello ", startIndex: 0, endIndex: 6 }, + { type: "atPath", content: "./image.png", startIndex: 6, endIndex: 18 }, + { type: "text", content: " world", startIndex: 18, endIndex: 24 }, + ] + + const result = reconstructText(segments) + + expect(result).toBe("Hello @./image.png world") + }) + + it("should apply transform function", () => { + const segments: ParsedSegment[] = [ + { type: "text", content: "Check ", startIndex: 0, endIndex: 6 }, + { type: "atPath", content: "./image.png", startIndex: 6, endIndex: 18 }, + ] + + const result = reconstructText(segments, (seg) => { + if (seg.type === "atPath") { + return `[IMG: ${seg.content}]` + } + return seg.content + }) + + expect(result).toBe("Check [IMG: ./image.png]") + }) + }) + + describe("edge cases", () => { + it("should handle empty string", () => { + const result = parseAtMentions("") + + expect(result.paths).toEqual([]) + expect(result.segments).toHaveLength(0) + }) + + it("should handle only @", () => { + const result = parseAtMentions("@") + + expect(result.paths).toEqual([]) + }) + + it("should handle @ followed by space", () => { + const result = parseAtMentions("@ space") + + expect(result.paths).toEqual([]) + }) + + it("should handle unclosed quotes", () => { + const result = parseAtMentions('Check @"unclosed quote') + + // Should still extract what it can + expect(result.paths).toHaveLength(1) + }) + + it("should handle escaped backslash in path", () => { + const result = parseAtMentions("@path\\\\with\\\\backslash.png") + + expect(result.paths).toEqual(["path\\with\\backslash.png"]) + }) + + it("should handle various path terminators", () => { + const tests = [ + { input: "@./img.png)", expected: "./img.png" }, + { input: "@./img.png]", expected: "./img.png" }, + { input: "@./img.png}", expected: "./img.png" }, + { input: "@./img.png>", expected: "./img.png" }, + { input: "@./img.png|", expected: "./img.png" }, + { input: "@./img.png&", expected: "./img.png" }, + ] + + for (const { input, expected } of tests) { + const result = parseAtMentions(input) + expect(result.paths).toEqual([expected]) + } + }) + }) +}) diff --git a/cli/src/media/__tests__/clipboard.test.ts b/cli/src/media/__tests__/clipboard.test.ts new file mode 100644 index 00000000000..41a8530fd58 --- /dev/null +++ b/cli/src/media/__tests__/clipboard.test.ts @@ -0,0 +1,170 @@ +import { + isClipboardSupported, + // Domain logic functions (exported for testing) + parseClipboardInfo, + detectImageFormat, + buildDataUrl, + getUnsupportedClipboardPlatformMessage, + getClipboardDir, + generateClipboardFilename, +} from "../clipboard" + +describe("clipboard utility", () => { + describe("parseClipboardInfo (macOS clipboard info parsing)", () => { + it("should detect PNG format", () => { + expect(parseClipboardInfo("«class PNGf», 1234")).toEqual({ hasImage: true, format: "png" }) + }) + + it("should detect JPEG format", () => { + expect(parseClipboardInfo("«class JPEG», 5678")).toEqual({ hasImage: true, format: "jpeg" }) + }) + + it("should detect TIFF format", () => { + expect(parseClipboardInfo("TIFF picture, 9012")).toEqual({ hasImage: true, format: "tiff" }) + }) + + it("should detect GIF format", () => { + expect(parseClipboardInfo("«class GIFf», 3456")).toEqual({ hasImage: true, format: "gif" }) + }) + + it("should return no image for text-only clipboard", () => { + expect(parseClipboardInfo("«class utf8», 100")).toEqual({ hasImage: false, format: null }) + }) + + it("should return no image for empty string", () => { + expect(parseClipboardInfo("")).toEqual({ hasImage: false, format: null }) + }) + + it("should handle multiple types and pick first image", () => { + expect(parseClipboardInfo("«class PNGf», 1234, «class utf8», 100")).toEqual({ + hasImage: true, + format: "png", + }) + }) + }) + + describe("detectImageFormat (format detection from bytes)", () => { + it("should detect PNG from magic bytes", () => { + const pngBytes = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) + expect(detectImageFormat(pngBytes)).toBe("png") + }) + + it("should detect JPEG from magic bytes", () => { + const jpegBytes = Buffer.from([0xff, 0xd8, 0xff, 0xe0]) + expect(detectImageFormat(jpegBytes)).toBe("jpeg") + }) + + it("should detect GIF from magic bytes", () => { + const gifBytes = Buffer.from([0x47, 0x49, 0x46, 0x38, 0x39, 0x61]) // GIF89a + expect(detectImageFormat(gifBytes)).toBe("gif") + }) + + it("should detect WebP from magic bytes", () => { + const webpBytes = Buffer.from([0x52, 0x49, 0x46, 0x46, 0x00, 0x00, 0x00, 0x00, 0x57, 0x45, 0x42, 0x50]) + expect(detectImageFormat(webpBytes)).toBe("webp") + }) + + it("should return null for unknown format", () => { + const unknownBytes = Buffer.from([0x00, 0x01, 0x02, 0x03]) + expect(detectImageFormat(unknownBytes)).toBe(null) + }) + + it("should return null for empty buffer", () => { + expect(detectImageFormat(Buffer.from([]))).toBe(null) + }) + }) + + describe("buildDataUrl", () => { + it("should build PNG data URL", () => { + const data = Buffer.from([0x89, 0x50, 0x4e, 0x47]) + const result = buildDataUrl(data, "png") + expect(result).toBe(`data:image/png;base64,${data.toString("base64")}`) + }) + + it("should build JPEG data URL", () => { + const data = Buffer.from([0xff, 0xd8, 0xff]) + const result = buildDataUrl(data, "jpeg") + expect(result).toBe(`data:image/jpeg;base64,${data.toString("base64")}`) + }) + + it("should handle arbitrary binary data", () => { + const data = Buffer.from("Hello, World!") + const result = buildDataUrl(data, "png") + expect(result).toMatch(/^data:image\/png;base64,/) + expect(result).toContain(data.toString("base64")) + }) + }) + + describe("getUnsupportedClipboardPlatformMessage", () => { + it("should mention macOS", () => { + const msg = getUnsupportedClipboardPlatformMessage() + expect(msg).toContain("macOS") + }) + + it("should mention @path/to/image.png alternative", () => { + const msg = getUnsupportedClipboardPlatformMessage() + expect(msg).toContain("@") + expect(msg.toLowerCase()).toContain("image") + }) + }) + + describe("isClipboardSupported (platform detection)", () => { + const originalPlatform = process.platform + + afterEach(() => { + Object.defineProperty(process, "platform", { value: originalPlatform }) + }) + + it("should return true for darwin", async () => { + Object.defineProperty(process, "platform", { value: "darwin" }) + expect(await isClipboardSupported()).toBe(true) + }) + + it("should return false for win32", async () => { + Object.defineProperty(process, "platform", { value: "win32" }) + expect(await isClipboardSupported()).toBe(false) + }) + }) + + describe("getClipboardDir", () => { + it("should return clipboard directory in system temp", () => { + const result = getClipboardDir() + expect(result).toContain("kilocode-clipboard") + // Should be in temp directory, not a project directory + expect(result).not.toContain(".kilocode-clipboard") + }) + }) + + describe("generateClipboardFilename", () => { + it("should generate unique filenames", () => { + const filename1 = generateClipboardFilename("png") + const filename2 = generateClipboardFilename("png") + expect(filename1).not.toBe(filename2) + }) + + it("should include correct extension", () => { + const pngFilename = generateClipboardFilename("png") + const jpegFilename = generateClipboardFilename("jpeg") + expect(pngFilename).toMatch(/\.png$/) + expect(jpegFilename).toMatch(/\.jpeg$/) + }) + + it("should start with clipboard- prefix", () => { + const filename = generateClipboardFilename("png") + expect(filename).toMatch(/^clipboard-/) + }) + + it("should include timestamp", () => { + const before = Date.now() + const filename = generateClipboardFilename("png") + const after = Date.now() + + // Extract timestamp from filename (clipboard-TIMESTAMP-RANDOM.ext) + const match = filename.match(/^clipboard-(\d+)-/) + expect(match).toBeTruthy() + const timestamp = parseInt(match![1], 10) + expect(timestamp).toBeGreaterThanOrEqual(before) + expect(timestamp).toBeLessThanOrEqual(after) + }) + }) +}) diff --git a/cli/src/media/__tests__/images.test.ts b/cli/src/media/__tests__/images.test.ts new file mode 100644 index 00000000000..b453b375eb5 --- /dev/null +++ b/cli/src/media/__tests__/images.test.ts @@ -0,0 +1,251 @@ +import * as fs from "fs/promises" +import * as path from "path" +import * as os from "os" +import { + isImagePath, + getMimeType, + readImageAsDataUrl, + processImagePaths, + SUPPORTED_IMAGE_EXTENSIONS, + MAX_IMAGE_SIZE_BYTES, +} from "../images" + +describe("images utility", () => { + let tempDir: string + + beforeEach(async () => { + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "images-test-")) + }) + + afterEach(async () => { + await fs.rm(tempDir, { recursive: true, force: true }) + }) + + describe("isImagePath", () => { + it("should return true for supported image extensions", () => { + expect(isImagePath("image.png")).toBe(true) + expect(isImagePath("image.PNG")).toBe(true) + expect(isImagePath("image.jpg")).toBe(true) + expect(isImagePath("image.JPG")).toBe(true) + expect(isImagePath("image.jpeg")).toBe(true) + expect(isImagePath("image.JPEG")).toBe(true) + expect(isImagePath("image.webp")).toBe(true) + expect(isImagePath("image.WEBP")).toBe(true) + expect(isImagePath("image.gif")).toBe(true) + expect(isImagePath("image.GIF")).toBe(true) + expect(isImagePath("image.tiff")).toBe(true) + expect(isImagePath("image.TIFF")).toBe(true) + }) + + it("should return false for non-image extensions", () => { + expect(isImagePath("file.txt")).toBe(false) + expect(isImagePath("file.ts")).toBe(false) + expect(isImagePath("file.js")).toBe(false) + expect(isImagePath("file.pdf")).toBe(false) + expect(isImagePath("file.bmp")).toBe(false) // BMP not supported + expect(isImagePath("file")).toBe(false) + }) + + it("should handle paths with directories", () => { + expect(isImagePath("/path/to/image.png")).toBe(true) + expect(isImagePath("./relative/path/image.jpg")).toBe(true) + expect(isImagePath("../parent/image.webp")).toBe(true) + }) + + it("should handle paths with dots in filename", () => { + expect(isImagePath("my.file.name.png")).toBe(true) + expect(isImagePath("version.1.2.3.jpg")).toBe(true) + }) + }) + + describe("getMimeType", () => { + it("should return correct MIME type for PNG", () => { + expect(getMimeType("image.png")).toBe("image/png") + expect(getMimeType("image.PNG")).toBe("image/png") + }) + + it("should return correct MIME type for JPEG", () => { + expect(getMimeType("image.jpg")).toBe("image/jpeg") + expect(getMimeType("image.jpeg")).toBe("image/jpeg") + expect(getMimeType("image.JPG")).toBe("image/jpeg") + }) + + it("should return correct MIME type for WebP", () => { + expect(getMimeType("image.webp")).toBe("image/webp") + }) + + it("should return correct MIME type for GIF and TIFF", () => { + expect(getMimeType("image.gif")).toBe("image/gif") + expect(getMimeType("image.tiff")).toBe("image/tiff") + }) + + it("should throw for unsupported types", () => { + expect(() => getMimeType("image.bmp")).toThrow("Unsupported image type") + expect(() => getMimeType("image.svg")).toThrow("Unsupported image type") + }) + }) + + describe("readImageAsDataUrl", () => { + it("should read a PNG file and return data URL", async () => { + // Create a minimal valid PNG (1x1 red pixel) + const pngData = Buffer.from([ + 0x89, + 0x50, + 0x4e, + 0x47, + 0x0d, + 0x0a, + 0x1a, + 0x0a, // PNG signature + 0x00, + 0x00, + 0x00, + 0x0d, // IHDR length + 0x49, + 0x48, + 0x44, + 0x52, // IHDR type + 0x00, + 0x00, + 0x00, + 0x01, // width = 1 + 0x00, + 0x00, + 0x00, + 0x01, // height = 1 + 0x08, + 0x02, // bit depth 8, color type 2 (RGB) + 0x00, + 0x00, + 0x00, // compression, filter, interlace + 0x90, + 0x77, + 0x53, + 0xde, // CRC + 0x00, + 0x00, + 0x00, + 0x0c, // IDAT length + 0x49, + 0x44, + 0x41, + 0x54, // IDAT type + 0x08, + 0xd7, + 0x63, + 0xf8, + 0xcf, + 0xc0, + 0x00, + 0x00, + 0x01, + 0x01, + 0x01, + 0x00, // compressed data + 0x18, + 0xdd, + 0x8d, + 0xb5, // CRC + 0x00, + 0x00, + 0x00, + 0x00, // IEND length + 0x49, + 0x45, + 0x4e, + 0x44, // IEND type + 0xae, + 0x42, + 0x60, + 0x82, // CRC + ]) + + const imagePath = path.join(tempDir, "test.png") + await fs.writeFile(imagePath, pngData) + + const dataUrl = await readImageAsDataUrl(imagePath) + + expect(dataUrl).toMatch(/^data:image\/png;base64,/) + expect(dataUrl.length).toBeGreaterThan("data:image/png;base64,".length) + }) + + it("should resolve relative paths from basePath", async () => { + const pngData = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) // Minimal PNG header + const imagePath = path.join(tempDir, "relative.png") + await fs.writeFile(imagePath, pngData) + + const dataUrl = await readImageAsDataUrl("relative.png", tempDir) + + expect(dataUrl).toMatch(/^data:image\/png;base64,/) + }) + + it("should throw for non-existent files", async () => { + await expect(readImageAsDataUrl("/non/existent/path.png")).rejects.toThrow("Image file not found") + }) + + it("should throw for non-image files", async () => { + const textPath = path.join(tempDir, "test.txt") + await fs.writeFile(textPath, "Hello, world!") + + await expect(readImageAsDataUrl(textPath)).rejects.toThrow("Not a supported image type") + }) + + it("should throw for files larger than the maximum size", async () => { + const largeBuffer = Buffer.alloc(MAX_IMAGE_SIZE_BYTES + 1, 0xff) + const largePath = path.join(tempDir, "too-big.png") + await fs.writeFile(largePath, largeBuffer) + + await expect(readImageAsDataUrl(largePath)).rejects.toThrow("Image file is too large") + }) + }) + + describe("processImagePaths", () => { + it("should process multiple image paths", async () => { + // Create test images + const pngData = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) + const image1 = path.join(tempDir, "image1.png") + const image2 = path.join(tempDir, "image2.png") + await fs.writeFile(image1, pngData) + await fs.writeFile(image2, pngData) + + const result = await processImagePaths([image1, image2]) + + expect(result.images).toHaveLength(2) + expect(result.errors).toHaveLength(0) + expect(result.images[0]).toMatch(/^data:image\/png;base64,/) + expect(result.images[1]).toMatch(/^data:image\/png;base64,/) + }) + + it("should collect errors for failed paths", async () => { + const result = await processImagePaths(["/non/existent.png", "/another/missing.jpg"]) + + expect(result.images).toHaveLength(0) + expect(result.errors).toHaveLength(2) + expect(result.errors[0]).toMatchObject({ + path: "/non/existent.png", + }) + }) + + it("should partially succeed when some paths fail", async () => { + const pngData = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) + const validPath = path.join(tempDir, "valid.png") + await fs.writeFile(validPath, pngData) + + const result = await processImagePaths([validPath, "/non/existent.png"]) + + expect(result.images).toHaveLength(1) + expect(result.errors).toHaveLength(1) + }) + }) + + describe("SUPPORTED_IMAGE_EXTENSIONS", () => { + it("should contain expected extensions", () => { + expect(SUPPORTED_IMAGE_EXTENSIONS).toContain(".png") + expect(SUPPORTED_IMAGE_EXTENSIONS).toContain(".jpg") + expect(SUPPORTED_IMAGE_EXTENSIONS).toContain(".jpeg") + expect(SUPPORTED_IMAGE_EXTENSIONS).toContain(".webp") + expect(SUPPORTED_IMAGE_EXTENSIONS).toContain(".gif") + expect(SUPPORTED_IMAGE_EXTENSIONS).toContain(".tiff") + }) + }) +}) diff --git a/cli/src/media/__tests__/processMessageImages.test.ts b/cli/src/media/__tests__/processMessageImages.test.ts new file mode 100644 index 00000000000..e5c47cadd41 --- /dev/null +++ b/cli/src/media/__tests__/processMessageImages.test.ts @@ -0,0 +1,144 @@ +import { removeImageReferences, extractImageReferences, processMessageImages } from "../processMessageImages" +import * as images from "../images" + +// Mock the images module +vi.mock("../images", () => ({ + readImageAsDataUrl: vi.fn(), +})) + +// Mock the logs module +vi.mock("../../services/logs", () => ({ + logs: { + debug: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, +})) + +describe("processMessageImages helpers", () => { + describe("removeImageReferences", () => { + it("should remove image reference tokens without collapsing whitespace", () => { + const input = "Line1\n [Image #1]\nLine3" + const result = removeImageReferences(input) + + expect(result).toBe("Line1\n \nLine3") + }) + + it("should remove multiple image references", () => { + const input = "Hello [Image #1] world [Image #2] test" + const result = removeImageReferences(input) + + expect(result).toBe("Hello world test") + }) + + it("should handle text with no image references", () => { + const input = "Hello world" + const result = removeImageReferences(input) + + expect(result).toBe("Hello world") + }) + }) + + describe("extractImageReferences", () => { + it("should extract single image reference number", () => { + const input = "Hello [Image #1] world" + const result = extractImageReferences(input) + + expect(result).toEqual([1]) + }) + + it("should extract multiple image reference numbers", () => { + const input = "Hello [Image #1] world [Image #3] test [Image #2]" + const result = extractImageReferences(input) + + expect(result).toEqual([1, 3, 2]) + }) + + it("should return empty array when no references", () => { + const input = "Hello world" + const result = extractImageReferences(input) + + expect(result).toEqual([]) + }) + + it("should handle large reference numbers", () => { + const input = "[Image #999]" + const result = extractImageReferences(input) + + expect(result).toEqual([999]) + }) + }) + + describe("processMessageImages", () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it("should return original text when no images", async () => { + const result = await processMessageImages("Hello world") + + expect(result).toEqual({ + text: "Hello world", + images: [], + hasImages: false, + errors: [], + }) + }) + + it("should load images from [Image #N] references", async () => { + const mockDataUrl = "data:image/png;base64,abc123" + vi.mocked(images.readImageAsDataUrl).mockResolvedValue(mockDataUrl) + + const imageReferences = { 1: "/tmp/test.png" } + const result = await processMessageImages("Hello [Image #1] world", imageReferences) + + expect(images.readImageAsDataUrl).toHaveBeenCalledWith("/tmp/test.png") + expect(result.images).toEqual([mockDataUrl]) + expect(result.text).toBe("Hello world") + expect(result.hasImages).toBe(true) + expect(result.errors).toEqual([]) + }) + + it("should report error when image reference not found", async () => { + const imageReferences = { 2: "/tmp/other.png" } + const result = await processMessageImages("Hello [Image #1] world", imageReferences) + + expect(result.errors).toContain("Image #1 not found") + expect(result.images).toEqual([]) + }) + + it("should report error when image file fails to load", async () => { + vi.mocked(images.readImageAsDataUrl).mockRejectedValue(new Error("File not found")) + + const imageReferences = { 1: "/tmp/missing.png" } + const result = await processMessageImages("Hello [Image #1] world", imageReferences) + + expect(result.errors).toContain("Failed to load Image #1: File not found") + expect(result.images).toEqual([]) + }) + + it("should handle multiple image references", async () => { + const mockDataUrl1 = "data:image/png;base64,img1" + const mockDataUrl2 = "data:image/png;base64,img2" + vi.mocked(images.readImageAsDataUrl).mockResolvedValueOnce(mockDataUrl1).mockResolvedValueOnce(mockDataUrl2) + + const imageReferences = { + 1: "/tmp/test1.png", + 2: "/tmp/test2.png", + } + const result = await processMessageImages("[Image #1] and [Image #2]", imageReferences) + + expect(result.images).toEqual([mockDataUrl1, mockDataUrl2]) + expect(result.text).toBe(" and ") + expect(result.hasImages).toBe(true) + }) + + it("should process without imageReferences parameter", async () => { + const result = await processMessageImages("Hello world") + + expect(result.text).toBe("Hello world") + expect(result.images).toEqual([]) + expect(result.hasImages).toBe(false) + }) + }) +}) diff --git a/cli/src/media/atMentionParser.ts b/cli/src/media/atMentionParser.ts new file mode 100644 index 00000000000..f15b0cd6b0e --- /dev/null +++ b/cli/src/media/atMentionParser.ts @@ -0,0 +1,220 @@ +import { isImagePath } from "./images.js" +export interface ParsedSegment { + type: "text" | "atPath" + content: string + startIndex: number + endIndex: number +} + +export interface ParsedPrompt { + segments: ParsedSegment[] + paths: string[] + imagePaths: string[] + otherPaths: string[] +} + +const PATH_TERMINATORS = new Set([" ", "\t", "\n", "\r", ",", ";", ")", "]", "}", ">", "|", "&", "'", '"']) +const TRAILING_PUNCTUATION = new Set([".", ",", ":", ";", "!", "?"]) + +function isEscapedAt(input: string, index: number): boolean { + return index > 0 && input[index - 1] === "\\" +} + +function parseQuotedPath(input: string, startIndex: number): { path: string; endIndex: number } | null { + let i = startIndex + 2 // skip @ and opening quote + let path = "" + const quote = input[startIndex + 1] + + while (i < input.length) { + const char = input[i] + if (char === "\\" && i + 1 < input.length) { + const nextChar = input[i + 1] + if (nextChar === quote || nextChar === "\\") { + path += nextChar + i += 2 + continue + } + } + if (char === quote) { + return { path, endIndex: i + 1 } + } + path += char + i++ + } + + return path ? { path, endIndex: i } : null +} + +function stripTrailingPunctuation(path: string): { path: string; trimmed: boolean } { + let trimmed = path + let removed = false + while (trimmed.length > 0 && TRAILING_PUNCTUATION.has(trimmed[trimmed.length - 1]!)) { + trimmed = trimmed.slice(0, -1) + removed = true + } + return { path: trimmed, trimmed: removed } +} + +function parseUnquotedPath(input: string, startIndex: number): { path: string; endIndex: number } | null { + let i = startIndex + 1 + let path = "" + + while (i < input.length) { + const char = input[i]! + + if (char === "\\" && i + 1 < input.length) { + const nextChar = input[i + 1]! + if (nextChar === " " || nextChar === "\\" || PATH_TERMINATORS.has(nextChar)) { + path += nextChar + i += 2 + continue + } + } + + if (PATH_TERMINATORS.has(char)) { + break + } + + path += char + i++ + } + + if (!path) { + return null + } + + const { path: trimmedPath, trimmed } = stripTrailingPunctuation(path) + if (!trimmedPath) { + return null + } + + const endIndex = i - (trimmed ? path.length - trimmedPath.length : 0) + return { path: trimmedPath, endIndex } +} + +function extractPath(input: string, startIndex: number): { path: string; endIndex: number } | null { + if (startIndex + 1 >= input.length) { + return null + } + + const nextChar = input[startIndex + 1] + if (nextChar === '"' || nextChar === "'") { + return parseQuotedPath(input, startIndex) + } + + return parseUnquotedPath(input, startIndex) +} + +function pushTextSegment(segments: ParsedSegment[], input: string, textStart: number, currentIndex: number): void { + if (currentIndex > textStart) { + segments.push({ + type: "text", + content: input.slice(textStart, currentIndex), + startIndex: textStart, + endIndex: currentIndex, + }) + } +} + +function pushPathSegment( + segments: ParsedSegment[], + paths: string[], + imagePaths: string[], + otherPaths: string[], + currentIndex: number, + extracted: { path: string; endIndex: number }, +): void { + segments.push({ + type: "atPath", + content: extracted.path, + startIndex: currentIndex, + endIndex: extracted.endIndex, + }) + + paths.push(extracted.path) + + if (isImagePath(extracted.path)) { + imagePaths.push(extracted.path) + } else { + otherPaths.push(extracted.path) + } +} + +export function parseAtMentions(input: string): ParsedPrompt { + const segments: ParsedSegment[] = [] + const paths: string[] = [] + const imagePaths: string[] = [] + const otherPaths: string[] = [] + + let currentIndex = 0 + let textStart = 0 + + while (currentIndex < input.length) { + const char = input[currentIndex] + + if (char === "@" && !isEscapedAt(input, currentIndex)) { + const extracted = extractPath(input, currentIndex) + if (!extracted) { + currentIndex++ + continue + } + + pushTextSegment(segments, input, textStart, currentIndex) + pushPathSegment(segments, paths, imagePaths, otherPaths, currentIndex, extracted) + currentIndex = extracted.endIndex + textStart = currentIndex + continue + } + + currentIndex++ + } + + if (textStart < input.length) { + segments.push({ + type: "text", + content: input.slice(textStart), + startIndex: textStart, + endIndex: input.length, + }) + } + + return { segments, paths, imagePaths, otherPaths } +} + +export function extractImagePaths(input: string): string[] { + return parseAtMentions(input).imagePaths +} + +export function removeImageMentions(input: string, placeholder: string = ""): string { + const parsed = parseAtMentions(input) + + let result = "" + for (const segment of parsed.segments) { + if (segment.type === "text") { + result += segment.content + } else if (segment.type === "atPath") { + if (isImagePath(segment.content)) { + result += placeholder + } else { + result += `@${segment.content}` + } + } + } + + return result +} + +export function reconstructText(segments: ParsedSegment[], transform?: (segment: ParsedSegment) => string): string { + if (transform) { + return segments.map(transform).join("") + } + + return segments + .map((seg) => { + if (seg.type === "text") { + return seg.content + } + return `@${seg.content}` + }) + .join("") +} diff --git a/cli/src/media/clipboard-macos.ts b/cli/src/media/clipboard-macos.ts new file mode 100644 index 00000000000..3abdf143eec --- /dev/null +++ b/cli/src/media/clipboard-macos.ts @@ -0,0 +1,144 @@ +import * as fs from "fs" +import * as path from "path" +import { logs } from "../services/logs.js" +import { + buildDataUrl, + ensureClipboardDir, + execFileAsync, + generateClipboardFilename, + parseClipboardInfo, + type ClipboardImageResult, + type SaveClipboardResult, +} from "./clipboard-shared.js" + +export async function hasClipboardImageMacOS(): Promise { + const { stdout } = await execFileAsync("osascript", ["-e", "clipboard info"]) + return parseClipboardInfo(stdout).hasImage +} + +export async function readClipboardImageMacOS(): Promise { + const { stdout: info } = await execFileAsync("osascript", ["-e", "clipboard info"]) + const parsed = parseClipboardInfo(info) + + if (!parsed.hasImage || !parsed.format) { + return { + success: false, + error: "No image found in clipboard.", + } + } + + const formatToClass: Record = { + png: "PNGf", + jpeg: "JPEG", + tiff: "TIFF", + gif: "GIFf", + } + + const appleClass = formatToClass[parsed.format] + if (!appleClass) { + return { + success: false, + error: `Unsupported image format: ${parsed.format}`, + } + } + + const script = `set imageData to the clipboard as «class ${appleClass}» +return imageData` + + const { stdout } = await execFileAsync("osascript", ["-e", script], { + encoding: "buffer", + maxBuffer: 50 * 1024 * 1024, + }) + + const imageBuffer = Buffer.isBuffer(stdout) ? stdout : Buffer.from(stdout) + + if (imageBuffer.length === 0) { + return { + success: false, + error: "Failed to read image data from clipboard.", + } + } + + const mimeFormat = parsed.format === "tiff" ? "tiff" : parsed.format + + return { + success: true, + dataUrl: buildDataUrl(imageBuffer, mimeFormat), + } +} + +export async function saveClipboardImageMacOS(): Promise { + const { stdout: info } = await execFileAsync("osascript", ["-e", "clipboard info"]) + const parsed = parseClipboardInfo(info) + + if (!parsed.hasImage || !parsed.format) { + return { + success: false, + error: "No image found in clipboard.", + } + } + + const formatToClass: Record = { + png: "PNGf", + jpeg: "JPEG", + tiff: "TIFF", + gif: "GIFf", + } + + const appleClass = formatToClass[parsed.format] + if (!appleClass) { + return { + success: false, + error: `Unsupported image format: ${parsed.format}`, + } + } + + const clipboardDir = await ensureClipboardDir() + + const filename = generateClipboardFilename(parsed.format) + const filePath = path.join(clipboardDir, filename) + + // Escape backslashes and quotes for AppleScript string interpolation + const escapedPath = filePath.replace(/\\/g, "\\\\").replace(/"/g, '\\"') + + const script = ` +set imageData to the clipboard as «class ${appleClass}» +set filePath to POSIX file "${escapedPath}" +set fileRef to open for access filePath with write permission +write imageData to fileRef +close access fileRef +return "${escapedPath}" +` + + try { + await execFileAsync("osascript", ["-e", script], { + maxBuffer: 50 * 1024 * 1024, + }) + + const stats = await fs.promises.stat(filePath) + if (stats.size === 0) { + await fs.promises.unlink(filePath) + return { + success: false, + error: "Failed to write image data to file.", + } + } + + return { + success: true, + filePath, + } + } catch (error) { + try { + await fs.promises.unlink(filePath) + } catch (cleanupError) { + const err = cleanupError as NodeJS.ErrnoException + logs.debug("Failed to remove partial clipboard file after error", "clipboard", { + filePath, + error: err?.message ?? String(cleanupError), + code: err?.code, + }) + } + throw error + } +} diff --git a/cli/src/media/clipboard-shared.ts b/cli/src/media/clipboard-shared.ts new file mode 100644 index 00000000000..5592c9964cc --- /dev/null +++ b/cli/src/media/clipboard-shared.ts @@ -0,0 +1,109 @@ +import { execFile } from "child_process" +import * as fs from "fs" +import * as os from "os" +import * as path from "path" +import { promisify } from "util" + +export const execFileAsync = promisify(execFile) + +export const CLIPBOARD_DIR = "kilocode-clipboard" +export const MAX_CLIPBOARD_IMAGE_AGE_MS = 60 * 60 * 1000 + +export interface ClipboardImageResult { + success: boolean + dataUrl?: string + error?: string +} + +export interface ClipboardInfoResult { + hasImage: boolean + format: "png" | "jpeg" | "tiff" | "gif" | null +} + +export interface SaveClipboardResult { + success: boolean + filePath?: string + error?: string +} + +export function parseClipboardInfo(output: string): ClipboardInfoResult { + if (!output) { + return { hasImage: false, format: null } + } + + if (output.includes("PNGf") || output.includes("class PNGf")) { + return { hasImage: true, format: "png" } + } + if (output.includes("JPEG") || output.includes("class JPEG")) { + return { hasImage: true, format: "jpeg" } + } + if (output.includes("TIFF") || output.includes("TIFF picture")) { + return { hasImage: true, format: "tiff" } + } + if (output.includes("GIFf") || output.includes("class GIFf")) { + return { hasImage: true, format: "gif" } + } + + return { hasImage: false, format: null } +} + +export function detectImageFormat(buffer: Buffer): "png" | "jpeg" | "gif" | "webp" | null { + if (buffer.length < 4) { + return null + } + + if (buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4e && buffer[3] === 0x47) { + return "png" + } + + if (buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) { + return "jpeg" + } + + if (buffer[0] === 0x47 && buffer[1] === 0x49 && buffer[2] === 0x46 && buffer[3] === 0x38) { + return "gif" + } + + if ( + buffer.length >= 12 && + buffer[0] === 0x52 && + buffer[1] === 0x49 && + buffer[2] === 0x46 && + buffer[3] === 0x46 && + buffer[8] === 0x57 && + buffer[9] === 0x45 && + buffer[10] === 0x42 && + buffer[11] === 0x50 + ) { + return "webp" + } + + return null +} + +export function buildDataUrl(data: Buffer, format: string): string { + return `data:image/${format};base64,${data.toString("base64")}` +} + +export function getUnsupportedClipboardPlatformMessage(): string { + return `Clipboard image paste is only supported on macOS. + +Alternative: + - Use @path/to/image.png to attach images` +} + +export function getClipboardDir(): string { + return path.join(os.tmpdir(), CLIPBOARD_DIR) +} + +export async function ensureClipboardDir(): Promise { + const clipboardDir = getClipboardDir() + await fs.promises.mkdir(clipboardDir, { recursive: true }) + return clipboardDir +} + +export function generateClipboardFilename(format: string): string { + const timestamp = Date.now() + const random = Math.random().toString(36).substring(2, 8) + return `clipboard-${timestamp}-${random}.${format}` +} diff --git a/cli/src/media/clipboard.ts b/cli/src/media/clipboard.ts new file mode 100644 index 00000000000..57c88208c10 --- /dev/null +++ b/cli/src/media/clipboard.ts @@ -0,0 +1,119 @@ +import * as fs from "fs" +import * as path from "path" +import { logs } from "../services/logs.js" +import { + buildDataUrl, + detectImageFormat, + generateClipboardFilename, + getClipboardDir, + parseClipboardInfo, + MAX_CLIPBOARD_IMAGE_AGE_MS, + getUnsupportedClipboardPlatformMessage, + type ClipboardImageResult, + type ClipboardInfoResult, + type SaveClipboardResult, +} from "./clipboard-shared.js" +import { hasClipboardImageMacOS, readClipboardImageMacOS, saveClipboardImageMacOS } from "./clipboard-macos.js" + +export { + buildDataUrl, + detectImageFormat, + generateClipboardFilename, + getClipboardDir, + getUnsupportedClipboardPlatformMessage, + parseClipboardInfo, + type ClipboardImageResult, + type ClipboardInfoResult, + type SaveClipboardResult, +} + +export async function isClipboardSupported(): Promise { + return process.platform === "darwin" +} + +export async function clipboardHasImage(): Promise { + try { + if (process.platform === "darwin") { + return await hasClipboardImageMacOS() + } + return false + } catch (error) { + const err = error as NodeJS.ErrnoException + logs.debug("clipboardHasImage failed, treating as no image", "clipboard", { + error: err?.message ?? String(error), + code: err?.code, + }) + return false + } +} + +export async function readClipboardImage(): Promise { + if (process.platform !== "darwin") { + return { + success: false, + error: getUnsupportedClipboardPlatformMessage(), + } + } + + try { + return await readClipboardImageMacOS() + } catch (error) { + return { + success: false, + error: error instanceof Error ? error.message : String(error), + } + } +} + +export async function saveClipboardImage(): Promise { + if (process.platform !== "darwin") { + return { + success: false, + error: getUnsupportedClipboardPlatformMessage(), + } + } + + try { + return await saveClipboardImageMacOS() + } catch (error) { + return { + success: false, + error: error instanceof Error ? error.message : String(error), + } + } +} + +export async function cleanupOldClipboardImages(): Promise { + const clipboardDir = getClipboardDir() + + try { + const files = await fs.promises.readdir(clipboardDir) + const now = Date.now() + + for (const file of files) { + if (!file.startsWith("clipboard-")) continue + + const filePath = path.join(clipboardDir, file) + try { + const stats = await fs.promises.stat(filePath) + if (now - stats.mtimeMs > MAX_CLIPBOARD_IMAGE_AGE_MS) { + await fs.promises.unlink(filePath) + } + } catch (error) { + const err = error as NodeJS.ErrnoException + logs.debug("Failed to delete stale clipboard image", "clipboard", { + filePath, + error: err?.message ?? String(error), + code: err?.code, + }) + } + } + } catch (error) { + const err = error as NodeJS.ErrnoException + logs.debug("Skipping clipboard cleanup; directory not accessible", "clipboard", { + dir: clipboardDir, + error: err?.message ?? String(error), + code: err?.code, + }) + } +} diff --git a/cli/src/media/images.ts b/cli/src/media/images.ts new file mode 100644 index 00000000000..1cf4686571b --- /dev/null +++ b/cli/src/media/images.ts @@ -0,0 +1,99 @@ +import fs from "fs/promises" +import path from "path" +import { logs } from "../services/logs.js" + +export const MAX_IMAGE_SIZE_BYTES = 8 * 1024 * 1024 // 8MB + +export const SUPPORTED_IMAGE_EXTENSIONS = [".png", ".jpg", ".jpeg", ".webp", ".gif", ".tiff"] as const +export type SupportedImageExtension = (typeof SUPPORTED_IMAGE_EXTENSIONS)[number] + +export function isImagePath(filePath: string): boolean { + const ext = path.extname(filePath).toLowerCase() + return SUPPORTED_IMAGE_EXTENSIONS.includes(ext as SupportedImageExtension) +} + +export function getMimeType(filePath: string): string { + const ext = path.extname(filePath).toLowerCase() + switch (ext) { + case ".png": + return "image/png" + case ".jpeg": + case ".jpg": + return "image/jpeg" + case ".webp": + return "image/webp" + case ".gif": + return "image/gif" + case ".tiff": + return "image/tiff" + default: + throw new Error(`Unsupported image type: ${ext}`) + } +} + +export async function readImageAsDataUrl(imagePath: string, basePath?: string): Promise { + // Resolve the path + const resolvedPath = path.isAbsolute(imagePath) ? imagePath : path.resolve(basePath || process.cwd(), imagePath) + + // Verify it's a supported image type + if (!isImagePath(resolvedPath)) { + throw new Error(`Not a supported image type: ${imagePath}`) + } + + // Check if file exists + try { + await fs.access(resolvedPath) + } catch { + throw new Error(`Image file not found: ${resolvedPath}`) + } + + // Enforce size limit before reading + const stats = await fs.stat(resolvedPath) + if (stats.size > MAX_IMAGE_SIZE_BYTES) { + const maxMb = (MAX_IMAGE_SIZE_BYTES / (1024 * 1024)).toFixed(1) + const actualMb = (stats.size / (1024 * 1024)).toFixed(1) + throw new Error(`Image file is too large (${actualMb} MB). Max allowed is ${maxMb} MB.`) + } + + // Read file and convert to base64 + const buffer = await fs.readFile(resolvedPath) + const base64 = buffer.toString("base64") + const mimeType = getMimeType(resolvedPath) + const dataUrl = `data:${mimeType};base64,${base64}` + + logs.debug(`Read image as data URL: ${path.basename(imagePath)}`, "images", { + path: resolvedPath, + size: buffer.length, + mimeType, + }) + + return dataUrl +} + +export interface ProcessedImageMentions { + text: string + images: string[] + errors: Array<{ path: string; error: string }> +} + +export async function processImagePaths(imagePaths: string[], basePath?: string): Promise { + const images: string[] = [] + const errors: Array<{ path: string; error: string }> = [] + + for (const imagePath of imagePaths) { + try { + const dataUrl = await readImageAsDataUrl(imagePath, basePath) + images.push(dataUrl) + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error) + errors.push({ path: imagePath, error: errorMessage }) + logs.warn(`Failed to load image: ${imagePath}`, "images", { error: errorMessage }) + } + } + + return { + text: "", // Will be set by the caller + images, + errors, + } +} diff --git a/cli/src/media/processMessageImages.ts b/cli/src/media/processMessageImages.ts new file mode 100644 index 00000000000..21b22c59953 --- /dev/null +++ b/cli/src/media/processMessageImages.ts @@ -0,0 +1,140 @@ +import { logs } from "../services/logs.js" +import { parseAtMentions, removeImageMentions } from "./atMentionParser.js" +import { readImageAsDataUrl } from "./images.js" + +export interface ProcessedMessage { + text: string + images: string[] + hasImages: boolean + errors: string[] +} + +const IMAGE_REFERENCE_REGEX = /\[Image #(\d+)\]/g + +export function extractImageReferences(text: string): number[] { + const refs: number[] = [] + let match + IMAGE_REFERENCE_REGEX.lastIndex = 0 + while ((match = IMAGE_REFERENCE_REGEX.exec(text)) !== null) { + const ref = match[1] + if (ref !== undefined) { + refs.push(parseInt(ref, 10)) + } + } + return refs +} + +export function removeImageReferences(text: string): string { + return text.replace(IMAGE_REFERENCE_REGEX, "") +} + +async function loadImage( + imagePath: string, + onSuccess: (dataUrl: string) => void, + onError: (error: string) => void, + successLog: string, + errorLog: { message: string; meta?: Record }, +): Promise { + try { + const dataUrl = await readImageAsDataUrl(imagePath) + onSuccess(dataUrl) + logs.debug(successLog, "processMessageImages") + } catch (error) { + const errorMsg = error instanceof Error ? error.message : String(error) + onError(errorMsg) + logs.warn(errorLog.message, "processMessageImages", { ...errorLog.meta, error: errorMsg }) + } +} + +async function loadReferenceImages( + refs: number[], + imageReferences: Record, + images: string[], + errors: string[], +): Promise { + logs.debug(`Found ${refs.length} image reference(s)`, "processMessageImages", { refs }) + + for (const refNum of refs) { + const filePath = imageReferences[refNum] + if (!filePath) { + errors.push(`Image #${refNum} not found`) + logs.warn(`Image reference #${refNum} not found in references map`, "processMessageImages") + continue + } + + await loadImage( + filePath, + (dataUrl) => images.push(dataUrl), + (errorMsg) => errors.push(`Failed to load Image #${refNum}: ${errorMsg}`), + `Loaded image #${refNum}: ${filePath}`, + { message: `Failed to load image #${refNum}: ${filePath}` }, + ) + } +} + +async function loadPathImages(imagePaths: string[], images: string[], errors: string[]): Promise { + logs.debug(`Found ${imagePaths.length} @path image mention(s)`, "processMessageImages", { + paths: imagePaths, + }) + + for (const imagePath of imagePaths) { + await loadImage( + imagePath, + (dataUrl) => images.push(dataUrl), + (errorMsg) => errors.push(`Failed to load image "${imagePath}": ${errorMsg}`), + `Loaded image: ${imagePath}`, + { message: `Failed to load image: ${imagePath}` }, + ) + } +} + +async function handleReferenceImages( + text: string, + imageReferences: Record, + images: string[], + errors: string[], +): Promise { + const refs = extractImageReferences(text) + if (refs.length === 0) { + return text + } + + await loadReferenceImages(refs, imageReferences, images, errors) + return removeImageReferences(text) +} + +async function handlePathMentions( + text: string, + images: string[], + errors: string[], +): Promise<{ cleanedText: string; hasImages: boolean }> { + const parsed = parseAtMentions(text) + if (parsed.imagePaths.length === 0) { + return { cleanedText: text, hasImages: images.length > 0 } + } + + await loadPathImages(parsed.imagePaths, images, errors) + return { cleanedText: removeImageMentions(text), hasImages: images.length > 0 } +} + +export async function processMessageImages( + text: string, + imageReferences?: Record, +): Promise { + const images: string[] = [] + const errors: string[] = [] + + let cleanedText = text + if (imageReferences) { + cleanedText = await handleReferenceImages(cleanedText, imageReferences, images, errors) + } + + const { cleanedText: finalText, hasImages } = await handlePathMentions(cleanedText, images, errors) + + return { + text: finalText, + images, + hasImages, + errors, + } +} diff --git a/cli/src/state/atoms/__tests__/shell.test.ts b/cli/src/state/atoms/__tests__/shell.test.ts index b807ab78763..76f184131a7 100644 --- a/cli/src/state/atoms/__tests__/shell.test.ts +++ b/cli/src/state/atoms/__tests__/shell.test.ts @@ -12,10 +12,9 @@ import { } from "../shell.js" import { textBufferStringAtom, setTextAtom } from "../textBuffer.js" -// Mock child_process to avoid actual command execution +// Mock child_process to avoid actual command execution; provide exec and execFile for clipboard code vi.mock("child_process", () => ({ exec: vi.fn((command) => { - // Simulate successful command execution const stdout = `Mock output for: ${command}` const stderr = "" const process = { @@ -41,6 +40,9 @@ vi.mock("child_process", () => ({ } return process }), + execFile: vi.fn((..._args) => { + throw new Error("execFile mocked in shell tests") + }), })) describe("shell mode - comprehensive tests", () => { diff --git a/cli/src/state/atoms/keyboard.ts b/cli/src/state/atoms/keyboard.ts index 0b3c4419c71..821c6e333dc 100644 --- a/cli/src/state/atoms/keyboard.ts +++ b/cli/src/state/atoms/keyboard.ts @@ -58,6 +58,8 @@ import { navigateShellHistoryDownAtom, executeShellCommandAtom, } from "./shell.js" +import { saveClipboardImage, clipboardHasImage, cleanupOldClipboardImages } from "../../media/clipboard.js" +import { logs } from "../../services/logs.js" // Export shell atoms for backward compatibility export { @@ -68,6 +70,65 @@ export { executeShellCommandAtom, } +// ============================================================================ +// Clipboard Image Atoms +// ============================================================================ + +/** + * Map of image reference numbers to file paths for current message + * e.g., { 1: "/tmp/kilocode-clipboard/clipboard-xxx.png", 2: "/tmp/..." } + */ +export const imageReferencesAtom = atom>(new Map()) + +/** + * Current image reference counter (increments with each paste) + */ +export const imageReferenceCounterAtom = atom(0) + +/** + * Add a clipboard image and get its reference number + * Returns the reference number assigned to this image + */ +export const addImageReferenceAtom = atom(null, (get, set, filePath: string): number => { + const counter = get(imageReferenceCounterAtom) + 1 + set(imageReferenceCounterAtom, counter) + + const refs = new Map(get(imageReferencesAtom)) + refs.set(counter, filePath) + set(imageReferencesAtom, refs) + + return counter +}) + +/** + * Clear image references (after message is sent) + */ +export const clearImageReferencesAtom = atom(null, (_get, set) => { + set(imageReferencesAtom, new Map()) + set(imageReferenceCounterAtom, 0) +}) + +/** + * Get all image references as an object for easier consumption + */ +export const getImageReferencesAtom = atom((get) => { + return Object.fromEntries(get(imageReferencesAtom)) +}) + +/** + * Status message for clipboard operations + */ +export const clipboardStatusAtom = atom(null) +let clipboardStatusTimer: NodeJS.Timeout | null = null + +function setClipboardStatusWithTimeout(set: Setter, message: string, timeoutMs: number): void { + if (clipboardStatusTimer) { + clearTimeout(clipboardStatusTimer) + } + set(clipboardStatusAtom, message) + clipboardStatusTimer = setTimeout(() => set(clipboardStatusAtom, null), timeoutMs) +} + // ============================================================================ // Core State Atoms // ============================================================================ @@ -792,12 +853,41 @@ function handleTextInputKeys(get: Getter, set: Setter, key: Key) { } function handleGlobalHotkeys(get: Getter, set: Setter, key: Key): boolean { + // Debug logging for key detection + if (key.ctrl || key.sequence === "\x16") { + logs.debug( + `Key detected: name=${key.name}, ctrl=${key.ctrl}, meta=${key.meta}, sequence=${JSON.stringify(key.sequence)}`, + "clipboard", + ) + } + + // Check for Ctrl+V by sequence first (ASCII 0x16 = SYN character) + // This is how Ctrl+V appears in most terminals + if (key.sequence === "\x16") { + logs.debug("Detected Ctrl+V via sequence \\x16", "clipboard") + handleClipboardImagePaste(get, set).catch((err) => + logs.error("Unhandled clipboard paste error", "clipboard", { error: err }), + ) + return true + } + switch (key.name) { case "c": if (key.ctrl) { process.exit(0) } break + case "v": + // Ctrl+V - check for clipboard image + if (key.ctrl) { + logs.debug("Detected Ctrl+V via key.name", "clipboard") + // Handle clipboard image paste asynchronously + handleClipboardImagePaste(get, set).catch((err) => + logs.error("Unhandled clipboard paste error", "clipboard", { error: err }), + ) + return true + } + break case "x": if (key.ctrl) { const isStreaming = get(isStreamingAtom) @@ -831,6 +921,65 @@ function handleGlobalHotkeys(get: Getter, set: Setter, key: Key): boolean { return false } +/** + * Handle clipboard image paste (Ctrl+V) + * Saves clipboard image to a temp file and inserts @path reference into text buffer + */ +async function handleClipboardImagePaste(get: Getter, set: Setter): Promise { + logs.debug("handleClipboardImagePaste called", "clipboard") + try { + // Check if clipboard has an image + logs.debug("Checking clipboard for image...", "clipboard") + const hasImage = await clipboardHasImage() + logs.debug(`clipboardHasImage returned: ${hasImage}`, "clipboard") + if (!hasImage) { + setClipboardStatusWithTimeout(set, "No image in clipboard", 2000) + logs.debug("No image in clipboard", "clipboard") + return + } + + // Save the image to a file in temp directory + const result = await saveClipboardImage() + if (result.success && result.filePath) { + // Add image to references and get its number + const refNumber = set(addImageReferenceAtom, result.filePath) + + // Build the [Image #N] reference to insert + // Add space before and after if needed + const currentText = get(textBufferStringAtom) + let insertText = `[Image #${refNumber}]` + + // Check if we need spaces around the insertion + const charBefore = currentText.length > 0 ? currentText[currentText.length - 1] : "" + if (charBefore && charBefore !== " " && charBefore !== "\n") { + insertText = " " + insertText + } + insertText = insertText + " " + + // Insert at current cursor position + set(insertTextAtom, insertText) + + setClipboardStatusWithTimeout(set, `Image #${refNumber} attached`, 2000) + logs.debug(`Inserted clipboard image #${refNumber}: ${result.filePath}`, "clipboard") + + // Clean up old clipboard images in the background + cleanupOldClipboardImages().catch((cleanupError) => { + logs.debug("Clipboard cleanup failed", "clipboard", { + error: cleanupError instanceof Error ? cleanupError.message : String(cleanupError), + }) + }) + } else { + setClipboardStatusWithTimeout(set, result.error || "Failed to save clipboard image", 3000) + } + } catch (error) { + setClipboardStatusWithTimeout( + set, + `Clipboard error: ${error instanceof Error ? error.message : String(error)}`, + 3000, + ) + } +} + /** * Main keyboard handler that routes based on mode * This is the central keyboard handling atom that all key events go through diff --git a/cli/src/state/hooks/useMessageHandler.ts b/cli/src/state/hooks/useMessageHandler.ts index b885c777b63..b14518009d5 100644 --- a/cli/src/state/hooks/useMessageHandler.ts +++ b/cli/src/state/hooks/useMessageHandler.ts @@ -3,14 +3,16 @@ * Provides a clean interface for sending user messages to the extension */ -import { useSetAtom } from "jotai" +import { useSetAtom, useAtomValue } from "jotai" import { useCallback, useState } from "react" import { addMessageAtom } from "../atoms/ui.js" +import { imageReferencesAtom, clearImageReferencesAtom } from "../atoms/keyboard.js" import { useWebviewMessage } from "./useWebviewMessage.js" import { useTaskState } from "./useTaskState.js" import type { CliMessage } from "../../types/cli.js" import { logs } from "../../services/logs.js" import { getTelemetryService } from "../../services/telemetry/index.js" +import { processMessageImages } from "../../media/processMessageImages.js" /** * Options for useMessageHandler hook @@ -34,7 +36,7 @@ export interface UseMessageHandlerReturn { * Hook that provides message sending functionality * * This hook handles sending regular user messages (non-commands) to the extension, - * including adding the message to the UI and handling errors. + * including processing @path image mentions and handling errors. * * @example * ```tsx @@ -58,51 +60,72 @@ export function useMessageHandler(options: UseMessageHandlerOptions = {}): UseMe const { ciMode = false } = options const [isSending, setIsSending] = useState(false) const addMessage = useSetAtom(addMessageAtom) + const imageReferences = useAtomValue(imageReferencesAtom) + const clearImageReferences = useSetAtom(clearImageReferencesAtom) const { sendMessage, sendAskResponse } = useWebviewMessage() const { hasActiveTask } = useTaskState() const sendUserMessage = useCallback( async (text: string): Promise => { const trimmedText = text.trim() - if (!trimmedText) { return } - // Don't add user message to CLI state - the extension will handle it - // This prevents duplicate messages in the UI - - // Set sending state setIsSending(true) try { - // Track user message + // Convert image references Map to object for processMessageImages + const imageRefsObject = Object.fromEntries(imageReferences) + + // Process any @path image mentions and [Image #N] references in the message + const processed = await processMessageImages(trimmedText, imageRefsObject) + + // Show any image loading errors to the user + if (processed.errors.length > 0) { + for (const error of processed.errors) { + const errorMessage: CliMessage = { + id: `img-err-${Date.now()}-${Math.random()}`, + type: "error", + content: error, + ts: Date.now(), + } + addMessage(errorMessage) + } + } + + // Track telemetry getTelemetryService().trackUserMessageSent( - trimmedText.length, - false, // hasImages - CLI doesn't support images yet + processed.text.length, + processed.hasImages, hasActiveTask, - undefined, // taskId - will be added when we have task tracking + undefined, ) - // Check if there's an active task to determine message type - // This matches the webview behavior in ChatView.tsx (lines 650-683) + // Build message payload + const payload = { + text: processed.text, + ...(processed.hasImages && { images: processed.images }), + } + + // Clear image references after processing + if (imageReferences.size > 0) { + clearImageReferences() + } + + // Send to extension - either as response to active task or as new task if (hasActiveTask) { - // Send as response to existing task (like webview does) - logs.debug("Sending message as response to active task", "useMessageHandler") - await sendAskResponse({ - response: "messageResponse", - text: trimmedText, + logs.debug("Sending message as response to active task", "useMessageHandler", { + hasImages: processed.hasImages, }) + await sendAskResponse({ response: "messageResponse", ...payload }) } else { - // Start new task (no active conversation) - logs.debug("Starting new task", "useMessageHandler") - await sendMessage({ - type: "newTask", - text: trimmedText, + logs.debug("Starting new task", "useMessageHandler", { + hasImages: processed.hasImages, }) + await sendMessage({ type: "newTask", ...payload }) } } catch (error) { - // Add error message if sending failed const errorMessage: CliMessage = { id: Date.now().toString(), type: "error", @@ -111,11 +134,10 @@ export function useMessageHandler(options: UseMessageHandlerOptions = {}): UseMe } addMessage(errorMessage) } finally { - // Reset sending state setIsSending(false) } }, - [addMessage, ciMode, sendMessage, sendAskResponse, hasActiveTask], + [addMessage, ciMode, sendMessage, sendAskResponse, hasActiveTask, imageReferences, clearImageReferences], ) return {