Change feed: Adds id and pk to ChangeFeedMetadata for delete operations

Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedItem.cs

@@ -57,21 +57,22 @@ namespace Microsoft.Azure.Cosmos
         class ChangeFeedItem<T>
     {
         /// <summary>
-        /// The full fidelity change feed current item.
+        /// The current version of the item for all versions and deletes change feed mode.
+        /// It is always null for delete change feed operations.
         /// </summary>
         [JsonProperty(PropertyName = "current")]
         [JsonPropertyName("current")]
         public T Current { get; set; }
 
         /// <summary>
-        /// The full fidelity change feed metadata.
+        /// The item metadata for all versions and deletes change feed mode.
         /// </summary>
         [JsonProperty(PropertyName = "metadata", NullValueHandling = NullValueHandling.Ignore)]
         [JsonPropertyName("metadata")]
         public ChangeFeedMetadata Metadata { get; set; }
 
         /// <summary>
-        /// For delete operations, previous image is always going to be provided. The previous image on replace operations is not going to be exposed by default and requires account-level or container-level opt-in.
+        /// The previous version of the item for all versions and deletes change feed mode. The previous version on delete and replace operations is not exposed by default and requires container-level opt-in. Refer to https://aka.ms/cosmosdb-change-feed-deletes for more information.
         /// </summary>
         [JsonProperty(PropertyName = "previous", NullValueHandling = NullValueHandling.Ignore)]
         [JsonPropertyName("previous")]

Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs

@@ -5,54 +5,125 @@
 namespace Microsoft.Azure.Cosmos
 {
     using System;
-    using System.Text.Json;
+    using System.Collections.Generic;
     using Microsoft.Azure.Cosmos.Resource.FullFidelity;
-    using Microsoft.Azure.Cosmos.Resource.FullFidelity.Converters;
     using Microsoft.Azure.Documents;
     using Newtonsoft.Json;
     using Newtonsoft.Json.Converters;
 
     /// <summary>
     /// The metadata of a change feed resource with <see cref="ChangeFeedMode"/> is initialized to <see cref="ChangeFeedMode.AllVersionsAndDeletes"/>.
     /// </summary>
-    [System.Text.Json.Serialization.JsonConverter(typeof(ChangeFeedMetadataConverter))]
 #if PREVIEW
     public
 #else
     internal
 #endif
-        class ChangeFeedMetadata
+    class ChangeFeedMetadata
     {
+        private readonly static DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
+
         /// <summary>
         /// The change's conflict resolution timestamp.
         /// </summary>
-        [JsonProperty(PropertyName = ChangeFeedMetadataFields.ConflictResolutionTimestamp, NullValueHandling = NullValueHandling.Ignore)]
-        [JsonConverter(typeof(UnixDateTimeConverter))]
-        public DateTime ConflictResolutionTimestamp { get; internal set; }
+        [System.Text.Json.Serialization.JsonIgnore]
+        [Newtonsoft.Json.JsonIgnore]
+        public DateTime? ConflictResolutionTimestamp => this.ConflictResolutionTimestampInSeconds.HasValue ? UnixEpoch.AddSeconds(this.ConflictResolutionTimestampInSeconds.Value) : null;
+
+        [System.Text.Json.Serialization.JsonInclude]
+        [System.Text.Json.Serialization.JsonPropertyName(ChangeFeedMetadataFields.ConflictResolutionTimestamp)]
+        [JsonProperty(PropertyName = ChangeFeedMetadataFields.ConflictResolutionTimestamp)]
+        internal double? ConflictResolutionTimestampInSeconds { get; set; }
 
         /// <summary>
         /// The current change's logical sequence number.
         /// </summary>
+        [System.Text.Json.Serialization.JsonInclude]
+        [System.Text.Json.Serialization.JsonPropertyName(ChangeFeedMetadataFields.Lsn)]
         [JsonProperty(PropertyName = ChangeFeedMetadataFields.Lsn, NullValueHandling = NullValueHandling.Ignore)]
         public long Lsn { get; internal set; }
 
         /// <summary>
         /// The change's feed operation type <see cref="ChangeFeedOperationType"/>.
         /// </summary>
+        [Newtonsoft.Json.JsonConverter(typeof(StringEnumConverter))]
+        [System.Text.Json.Serialization.JsonInclude]
+        [System.Text.Json.Serialization.JsonPropertyName(ChangeFeedMetadataFields.OperationType)]
+        [System.Text.Json.Serialization.JsonConverter(typeof(System.Text.Json.Serialization.JsonStringEnumConverter))]
         [JsonProperty(PropertyName = ChangeFeedMetadataFields.OperationType, NullValueHandling = NullValueHandling.Ignore)]
-        [JsonConverter(typeof(StringEnumConverter))]
         public ChangeFeedOperationType OperationType { get; internal set; }
 
         /// <summary>
         /// The previous change's logical sequence number.
         /// </summary>
+        [System.Text.Json.Serialization.JsonInclude]
+        [System.Text.Json.Serialization.JsonPropertyName(ChangeFeedMetadataFields.PreviousImageLSN)]
         [JsonProperty(PropertyName = ChangeFeedMetadataFields.PreviousImageLSN, NullValueHandling = NullValueHandling.Ignore)]
         public long PreviousLsn { get; internal set; }
 
         /// <summary>
-        /// Used to distinquish explicit deletes (e.g. via DeleteItem) from deletes caused by TTL expiration (a collection may define time-to-live policy for documents).
+        /// Used to distinguish explicit deletes (e.g. via DeleteItem) from deletes caused by TTL expiration (a collection may define time-to-live policy for documents).
         /// </summary>
+        [System.Text.Json.Serialization.JsonInclude]
+        [System.Text.Json.Serialization.JsonPropertyName(ChangeFeedMetadataFields.TimeToLiveExpired)]
         [JsonProperty(PropertyName = ChangeFeedMetadataFields.TimeToLiveExpired, NullValueHandling = NullValueHandling.Ignore)]
         public bool IsTimeToLiveExpired { get; internal set; }
+
+        /// <summary>
+        /// Applicable for delete operations only, otherwise null.
+        /// The id of the previous item version. 
+        /// </summary>
+        [System.Text.Json.Serialization.JsonInclude]
+        [System.Text.Json.Serialization.JsonPropertyName(ChangeFeedMetadataFields.Id)]
+        [JsonProperty(PropertyName = ChangeFeedMetadataFields.Id, NullValueHandling = NullValueHandling.Ignore)]
+        public string Id { get; internal set; }
+
+        /// <summary>
+        /// Applicable for delete operations only, otherwise null.
+        /// The partition key of the previous item version represented as a dictionary where the key is the partition key property name 
+        /// and the value is the partition key property value. All levels of hierarchy will be present if a hierarchical partition key (HPK) is used.
+        /// </summary>
+        /// <remarks>
+        /// <para>
+        /// For single partition key containers, the dictionary will contain one entry with the partition key path name (without the leading '/') 
+        /// as the key and the partition key value as the value.
+        /// </para>
+        /// <para>
+        /// For hierarchical partition key containers, the dictionary will contain multiple entries, one for each level of the hierarchy, 
+        /// as defined in the container's partition key definition.
+        /// </para>
+        /// <para>
+        /// Example for a single partition key container with partition key path "/tenantId":
+        /// <code>
+        /// {
+        ///     "tenantId": "tenant123"
+        /// }
+        /// </code>
+        /// </para>
+        /// <para>
+        /// Example for a hierarchical partition key container with partition key paths ["/tenantId", "/userId", "/sessionId"]:
+        /// <code>
+        /// {
+        ///     "tenantId": "tenant123",
+        ///     "userId": "user456",
+        ///     "sessionId": "session789"
+        /// }
+        /// </code>
+        /// </para>
+        /// <para>
+        /// The partition key values can be of different types (string, number, boolean, null) depending on the document's schema.
+        /// For example, with partition key paths ["/category", "/priority"]:
+        /// <code>
+        /// {
+        ///     "category": "electronics",
+        ///     "priority": 1
+        /// }
+        /// </code>
+        /// </para>
+        /// </remarks>
+        [System.Text.Json.Serialization.JsonInclude]
+        [System.Text.Json.Serialization.JsonPropertyName(ChangeFeedMetadataFields.PartitionKey)]
+        [JsonProperty(PropertyName = ChangeFeedMetadataFields.PartitionKey, NullValueHandling = NullValueHandling.Ignore)]
+        public Dictionary<string, object> PartitionKey { get; internal set; }
     }
 }

Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadataFields.cs

@@ -11,5 +11,7 @@ internal class ChangeFeedMetadataFields
         public const string OperationType = "operationType";
         public const string PreviousImageLSN = "previousImageLSN";
         public const string TimeToLiveExpired = "timeToLiveExpired";
+        public const string Id = "id";
+        public const string PartitionKey = "partitionKey";
     }
 }

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs

@@ -69,7 +69,8 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA
                             Assert.IsTrue(DateTime.TryParse(s: change.Metadata.ConflictResolutionTimestamp.ToString(), out _), message: "Invalid csrt must be a datetime value.");
                             Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value.");
                             Assert.IsFalse(change.Metadata.IsTimeToLiveExpired);
-
+                            Assert.IsNull(change.Metadata.Id);
+                            Assert.IsNull(change.Metadata.PartitionKey);
                             // previous
                             Assert.IsNull(change.Previous);
                         }
@@ -84,10 +85,9 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA
                             Assert.IsTrue(change.Metadata.IsTimeToLiveExpired);
 
                             // previous
-                            Assert.AreEqual(expected: "1", actual: change.Previous.id.ToString());
-                            Assert.AreEqual(expected: "1", actual: change.Previous.pk.ToString());
-                            Assert.AreEqual(expected: "Testing TTL on CFP.", actual: change.Previous.description.ToString());
-                            Assert.AreEqual(expected: ttlInSeconds, actual: change.Previous.ttl);
+                            Assert.AreEqual(expected: "1", actual: change.Metadata.Id.ToString());
+                            Assert.AreEqual(expected: "1", actual: change.Metadata.PartitionKey.Values.FirstOrDefault());
+                            Assert.IsNull(change.Previous);
 
                             // stop after reading delete since it is the last document in feed.
                             stopwatch.Stop();
@@ -145,7 +145,7 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA
         [TestMethod]
         [Owner("philipthomas-MSFT")]
         [Description("Scenario: When a document is created, then updated, and finally deleted, there should be 3 changes that will appear for that " +
-            "document when using ChangeFeedProcessor with AllVersionsAndDeletes set as the ChangeFeedMode.")]
+            "document when using ChangeFeedProcessor with AllVersionsAndDeletes set as the ChangeFeedMode and enablePreviousImageForDeleteInFFCF true")]
         public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync()
         {
             ContainerInternal monitoredContainer = await this.CreateMonitoredContainer(ChangeFeedMode.AllVersionsAndDeletes);
@@ -155,6 +155,8 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync()
             ChangeFeedProcessor processor = monitoredContainer
                 .GetChangeFeedProcessorBuilderWithAllVersionsAndDeletes(processorName: "processor", onChangesDelegate: (ChangeFeedProcessorContext context, IReadOnlyCollection<ChangeFeedItem<dynamic>> docs, CancellationToken token) =>
                 {
+                    string metadataId = default;
+                    string metadataPk = default;
                     string id = default;
                     string pk = default;
                     string description = default;
@@ -171,14 +173,13 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync()
                         }
                         else
                         {
-                            id = change.Previous.id.ToString();
-                            pk = change.Previous.pk.ToString();
-                            description = change.Previous.description.ToString();
+                            metadataId = change.Metadata.Id.ToString();
+                            metadataPk = change.Metadata.PartitionKey.Values.FirstOrDefault().ToString();
                         }
 
                         ChangeFeedOperationType operationType = change.Metadata.OperationType;
                         long previousLsn = change.Metadata.PreviousLsn;
-                        DateTime m = change.Metadata.ConflictResolutionTimestamp;
+                        DateTime? m = change.Metadata.ConflictResolutionTimestamp;
                         long lsn = change.Metadata.Lsn;
                         bool isTimeToLiveExpired = change.Metadata.IsTimeToLiveExpired;
                     }
@@ -211,8 +212,9 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync()
 
                     ChangeFeedItem<dynamic> deleteChange = docs.ElementAt(2);
                     Assert.IsNull(deleteChange.Current.id);
+                    Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.Id.ToString());
+                    Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.PartitionKey.Values.FirstOrDefault());
                     Assert.AreEqual(expected: deleteChange.Metadata.OperationType, actual: ChangeFeedOperationType.Delete);
-                    Assert.AreEqual(expected: replaceChange.Metadata.Lsn, actual: deleteChange.Metadata.PreviousLsn);
                     Assert.IsNotNull(deleteChange.Previous);
                     Assert.AreEqual(expected: "1", actual: deleteChange.Previous.id.ToString());
                     Assert.AreEqual(expected: "1", actual: deleteChange.Previous.pk.ToString());