TypeScript: Provide BackupsJsonExporter for takeout flow

Author: andrew-signal

RELEASE_NOTES.md

@@ -3,3 +3,5 @@ v0.84.0
 - keytrans: Verify signatures from all auditors
 
 - Java: Fixed `IdentityKeyPair(byte[])` to correctly declare that it throws InvalidKeyException.
+
+- Node: Add BackupsJsonExporter, to convert Backup proto objects to human-readable JSON for export.

acknowledgments/acknowledgments-android.md

@@ -5018,7 +5018,7 @@ DEALINGS IN THE SOFTWARE.
 
 ```
 
-## protobuf-codegen 3.7.2, protobuf-support 3.7.2, protobuf 3.7.2
+## protobuf-codegen 3.7.2, protobuf-json-mapping 3.7.2, protobuf-support 3.7.2, protobuf 3.7.2
 
 ```
 Copyright (c) 2019 Stepan Koltsov

node/Native.d.ts

@@ -229,6 +229,10 @@ export function BackupAuthCredential_GetBackupId(credentialBytes: Uint8Array): U
 export function BackupAuthCredential_GetBackupLevel(credentialBytes: Uint8Array): number;
 export function BackupAuthCredential_GetType(credentialBytes: Uint8Array): number;
 export function BackupAuthCredential_PresentDeterministic(credentialBytes: Uint8Array, serverParamsBytes: Uint8Array, randomness: Uint8Array): Uint8Array;
+export function BackupJsonExporter_ExportFrames(exporter: Wrapper<BackupJsonExporter>, frames: Uint8Array): string;
+export function BackupJsonExporter_Finish(exporter: Wrapper<BackupJsonExporter>): string;
+export function BackupJsonExporter_GetInitialChunk(exporter: Wrapper<BackupJsonExporter>): string;
+export function BackupJsonExporter_New(backupInfo: Uint8Array, shouldValidate: boolean): BackupJsonExporter;
 export function BackupKey_DeriveBackupId(backupKey: Uint8Array, aci: Uint8Array): Uint8Array;
 export function BackupKey_DeriveEcKey(backupKey: Uint8Array, aci: Uint8Array): PrivateKey;
 export function BackupKey_DeriveLocalBackupMetadataKey(backupKey: Uint8Array): Uint8Array;
@@ -734,6 +738,7 @@ export function initLogger(maxLevel: LogLevel, callback: (level: LogLevel, targe
 export function test_only_fn_returns_123(): number;
 interface Aes256GcmSiv { readonly __type: unique symbol; }
 interface AuthenticatedChatConnection { readonly __type: unique symbol; }
+interface BackupJsonExporter { readonly __type: unique symbol; }
 interface BackupRestoreResponse { readonly __type: unique symbol; }
 interface BackupStoreResponse { readonly __type: unique symbol; }
 interface BridgedStringMap { readonly __type: unique symbol; }

node/package-lock.json

@@ -64,6 +64,7 @@
     "mocha": "^11",
     "prettier": "^2.7.1",
     "prettier-plugin-packagejson": "^2.5.19",
+    "protobufjs": "^7.5.3",
     "rimraf": "^6.0.1",
     "sinon": "^21.0.0",
     "sinon-chai": "^4.0.1",

node/package.json

@@ -114,6 +114,7 @@ export class MessageBackupKey {
 export enum Purpose {
   DeviceTransfer = 0,
   RemoteBackup = 1,
+  TakeoutExport = 2,
 }
 
 /**
@@ -277,3 +278,52 @@ export class ComparableBackup {
     return Native.ComparableBackup_GetUnknownFields(this);
   }
 }
+
+/**
+ * Streaming exporter that produces a human-readable JSON representation of a backup.
+ */
+export class BackupJsonExporter {
+  readonly _nativeHandle: Native.BackupJsonExporter;
+
+  private constructor(handle: Native.BackupJsonExporter) {
+    this._nativeHandle = handle;
+  }
+
+  /**
+   * Initializes the streaming exporter and returns the first chunk of output.
+   * @param backupInfo The serialized BackupInfo protobuf without a varint header.
+   * @param [options] Additional configuration for the exporter.
+   * @param [options.validate=true] Whether to run semantic validation on the backup.
+   * @returns An object containing the exporter and the first chunk of output, containing the backup info.
+   * @throws Error if the input is invalid.
+   */
+  public static start(
+    backupInfo: Uint8Array,
+    options?: { validate?: boolean }
+  ): { exporter: BackupJsonExporter; chunk: string } {
+    const shouldValidate = options?.validate ?? true;
+    const handle = Native.BackupJsonExporter_New(backupInfo, shouldValidate);
+    const exporter = new BackupJsonExporter(handle);
+    const chunk = Native.BackupJsonExporter_GetInitialChunk(exporter);
+    return { exporter, chunk };
+  }
+
+  /**
+   * Validates and exports a human-readable JSON representation of backup frames.
+   * @param frames One or more varint delimited Frame serialized protobuf messages.
+   * @returns A string containing the exported frames.
+   * @throws Error if the input is invalid.
+   */
+  public exportFrames(frames: Uint8Array): string {
+    return Native.BackupJsonExporter_ExportFrames(this, frames);
+  }
+
+  /**
+   * Completes the validation and export of the previously exported frames.
+   * @returns A string containing the final chunk of the output.
+   * @throws Error if some previous input fails validation at the final stage.
+   */
+  public finish(): string {
+    return Native.BackupJsonExporter_Finish(this);
+  }
+}

node/ts/MessageBackup.ts

@@ -8,6 +8,8 @@ import { Buffer } from 'node:buffer';
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { Readable } from 'node:stream';
+import protobuf from 'protobufjs/minimal.js';
+const { Reader } = protobuf;
 
 import * as MessageBackup from '../MessageBackup.js';
 import * as util from './util.js';
@@ -190,6 +192,60 @@ const exampleBackup = fs.readFileSync(
   path.join(import.meta.dirname, '../../ts/test/canonical-backup.binproto')
 );
 
+function chunkLengthDelimited(binproto: Uint8Array): Uint8Array[] {
+  const r = Reader.create(binproto);
+  const chunks: Uint8Array[] = [];
+
+  while (r.pos < r.len) {
+    const headerStart = r.pos; // start of the varint length prefix
+    const length = r.uint32(); // implicitly advances to the start of the body
+    const bodyStart = r.pos; // now points to the start of the proto message
+    const end = bodyStart + length;
+
+    if (end > r.len) {
+      throw new Error('truncated length-delimited chunk');
+    }
+
+    // Include the varint header + body
+    chunks.push(binproto.subarray(headerStart, end));
+    r.pos = end;
+  }
+
+  return chunks;
+}
+
+function stripLengthPrefix(chunk: Uint8Array): Uint8Array {
+  const reader = Reader.create(chunk);
+  const length = reader.uint32();
+  const bodyStart = reader.pos;
+  const bodyEnd = bodyStart + length;
+  if (bodyEnd > reader.len) {
+    throw new Error('truncated length-delimited chunk');
+  }
+  if (bodyEnd !== reader.len) {
+    throw new Error('unexpected trailing data after chunk body');
+  }
+  return chunk.subarray(bodyStart, bodyEnd);
+}
+
+const exampleBackupChunks = chunkLengthDelimited(exampleBackup);
+if (exampleBackupChunks.length === 0) {
+  throw new Error('expected at least one length-delimited chunk');
+}
+const [exampleBackupInfoChunk, ...exampleFrameChunks] = exampleBackupChunks;
+const exampleBackupInfo = stripLengthPrefix(exampleBackupInfoChunk);
+const exampleFrames = exampleFrameChunks;
+
+function concatFrames(chunks: ReadonlyArray<Uint8Array>): Uint8Array {
+  if (chunks.length === 0) {
+    return new Uint8Array();
+  }
+  if (chunks.length === 1) {
+    return new Uint8Array(chunks[0]);
+  }
+  return Buffer.concat(chunks.map((chunk) => Buffer.from(chunk)));
+}
+
 describe('ComparableBackup', () => {
   describe('exampleBackup', () => {
     it('stringifies to the expected value', async () => {
@@ -211,6 +267,121 @@ describe('ComparableBackup', () => {
   });
 });
 
+describe('BackupJsonExporter', () => {
+  it('streams pretty JSON for a canonical backup', () => {
+    const backupInfo = exampleBackupInfo;
+    const frames = exampleFrames.slice();
+
+    const { exporter, chunk: initialChunk } =
+      MessageBackup.BackupJsonExporter.start(backupInfo);
+
+    // Stream the frames across multiple chunks to mirror the real exporter usage.
+    const chunkGroups = [frames.slice(0, 2), frames.slice(2)].filter(
+      (group) => group.length > 0
+    );
+    const exportedFrameChunks = chunkGroups.map((group) =>
+      exporter.exportFrames(concatFrames(group))
+    );
+
+    const jsonText = [
+      initialChunk,
+      ...exportedFrameChunks,
+      exporter.finish(),
+    ].join('');
+    assert.isTrue(jsonText.startsWith('[\n  {'));
+    assert.isTrue(jsonText.endsWith(']\n'));
+
+    const parsed = JSON.parse(jsonText) as unknown;
+    assert.isArray(parsed);
+
+    const parsedArray = parsed as Array<unknown>;
+    assert.lengthOf(parsedArray, frames.length + 1);
+
+    const [backupInfoJson, firstFrame] = parsedArray;
+    assert.isObject(backupInfoJson);
+    assert.containsAllKeys(backupInfoJson as Record<string, unknown>, [
+      'version',
+      'mediaRootBackupKey',
+    ]);
+    assert.isObject(firstFrame);
+    const firstFrameRecord = firstFrame as Record<string, unknown>;
+    assert.property(firstFrameRecord, 'account');
+    const accountValue = firstFrameRecord.account;
+    assert.isObject(accountValue);
+    const accountRecord = accountValue as Record<string, unknown>;
+    assert.containsAllKeys(accountRecord, [
+      'profileKey',
+      'username',
+      'accountSettings',
+    ]);
+  });
+
+  it('returns an empty chunk when no frames are provided', () => {
+    const backupInfo = exampleBackupInfo;
+    const { exporter } = MessageBackup.BackupJsonExporter.start(backupInfo);
+    assert.equal(exporter.exportFrames(new Uint8Array()), '');
+  });
+
+  it('validates frames when requested', () => {
+    const backupInfo = exampleBackupInfo;
+    const frames = exampleFrames.slice();
+    const { exporter } = MessageBackup.BackupJsonExporter.start(backupInfo, {
+      validate: true,
+    });
+
+    const groupedFrames = [frames.slice(0, 1), frames.slice(1)];
+    for (const group of groupedFrames) {
+      if (group.length === 0) {
+        continue;
+      }
+      exporter.exportFrames(concatFrames(group));
+    }
+
+    exporter.finish();
+  });
+
+  it('throws when validation fails', () => {
+    const backupInfo = exampleBackupInfo;
+    const frames = exampleFrames.slice();
+    const { exporter, chunk } = MessageBackup.BackupJsonExporter.start(
+      backupInfo,
+      {
+        validate: true,
+      }
+    );
+
+    // baseline chunk should still be produced
+    assert.isTrue(chunk.startsWith('[\n'));
+
+    const missingAccountChunk = concatFrames(frames.slice(1));
+    exporter.exportFrames(missingAccountChunk);
+
+    assert.throws(() => exporter.finish());
+  });
+
+  it('can skip validation when explicitly disabled', () => {
+    const backupInfo = exampleBackupInfo;
+    const frames = exampleFrames.slice();
+    const { exporter } = MessageBackup.BackupJsonExporter.start(backupInfo, {
+      validate: false,
+    });
+
+    const missingAccountChunk = concatFrames(frames.slice(1));
+    exporter.exportFrames(missingAccountChunk);
+
+    exporter.finish();
+  });
+
+  it('still rejects malformed data even when validation is disabled', () => {
+    const backupInfo = exampleBackupInfo;
+    const { exporter } = MessageBackup.BackupJsonExporter.start(backupInfo, {
+      validate: false,
+    });
+
+    assert.throws(() => exporter.exportFrames(Uint8Array.of(0x02, 0x01)));
+  });
+});
+
 describe('OnlineBackupValidator', () => {
   it('can read frames from a valid file', () => {
     // `Readable.read` normally returns `any`, because it supports settable encodings.

node/ts/test/MessageBackupTest.ts

@@ -22,7 +22,7 @@ libsignal-bridge-macros = { workspace = true }
 libsignal-bridge-types = { workspace = true }
 libsignal-core = { workspace = true }
 libsignal-keytrans = { workspace = true }
-libsignal-message-backup = { workspace = true }
+libsignal-message-backup = { workspace = true, features = ["json"] }
 libsignal-net = { workspace = true }
 libsignal-net-chat = { workspace = true }
 libsignal-protocol = { workspace = true }

rust/bridge/shared/Cargo.toml

@@ -117,6 +117,7 @@ async fn MessageBackupValidator_Validate(
 }
 
 bridge_handle_fns!(OnlineBackupValidator, clone = false);
+bridge_handle_fns!(BackupJsonExporter, clone = false, ffi = false, jni = false);
 
 #[bridge_fn]
 fn OnlineBackupValidator_New(
@@ -155,3 +156,32 @@ fn OnlineBackupValidator_AddFrame(
 fn OnlineBackupValidator_Finalize(backup: &mut OnlineBackupValidator) -> Result<(), ReadError> {
     backup.finalize().map_err(ReadError::with_error_only)
 }
+
+#[bridge_fn(ffi = false, jni = false)]
+fn BackupJsonExporter_New(
+    backup_info: &[u8],
+    should_validate: bool,
+) -> Result<BackupJsonExporter, ReadError> {
+    let (exporter, initial_chunk) =
+        libsignal_message_backup::json::exporter::JsonExporter::new(backup_info, should_validate)?;
+
+    Ok(BackupJsonExporter::new(exporter, initial_chunk))
+}
+
+#[bridge_fn(ffi = false, jni = false)]
+fn BackupJsonExporter_GetInitialChunk(exporter: &BackupJsonExporter) -> String {
+    exporter.initial_chunk()
+}
+
+#[bridge_fn(ffi = false, jni = false)]
+fn BackupJsonExporter_ExportFrames(
+    exporter: &mut BackupJsonExporter,
+    frames: &[u8],
+) -> Result<String, ReadError> {
+    exporter.inner_mut().export_frames(frames)
+}
+
+#[bridge_fn(ffi = false, jni = false)]
+fn BackupJsonExporter_Finish(exporter: &mut BackupJsonExporter) -> Result<String, ReadError> {
+    exporter.inner_mut().finish()
+}

rust/bridge/shared/src/message_backup.rs

@@ -8,11 +8,11 @@ use libsignal_bridge_types::io::{AsyncInput, InputStream};
 use libsignal_bridge_types::support::*;
 use libsignal_bridge_types::*;
 use libsignal_message_backup::backup::Purpose;
-use libsignal_message_backup::{BackupReader, ReadError, ReadResult};
+use libsignal_message_backup::{BackupReader, FoundUnknownField, ReadError, ReadResult};
 
 pub struct ComparableBackup {
-    pub backup: libsignal_message_backup::backup::serialize::Backup,
-    pub found_unknown_fields: Vec<libsignal_message_backup::FoundUnknownField>,
+    backup: libsignal_message_backup::backup::serialize::Backup,
+    found_unknown_fields: Vec<FoundUnknownField>,
 }
 
 bridge_as_handle!(ComparableBackup);