Fix deduplication of limited subset requests (#914)

* Fix: Correctly handle limited queries in isPredicateSubset

Co-authored-by: sam.willis <[email protected]>

* Fix: Adjust live query deduplication logic and test expectations

Co-authored-by: sam.willis <[email protected]>

* Refactor deduplication tests for limited queries

Co-authored-by: sam.willis <[email protected]>

* Fix: Deduplicate limited queries based on where clause

Co-authored-by: sam.willis <[email protected]>

---------

Co-authored-by: Cursor Agent <[email protected]>

Author: samwillis

.changeset/fix-limited-query-deduplication.md

@@ -0,0 +1,5 @@
+---
+"@tanstack/db": patch
+---
+
+Fixed incorrect deduplication of limited queries with different where clauses. Previously, a query like `{where: searchFilter, limit: 10}` could be incorrectly deduplicated against a prior query `{where: undefined, limit: 10}`, causing search/filter results to only show cached data. Now, limited queries are only deduplicated when their where clauses are structurally equal.

packages/db/src/query/predicate-utils.ts

@@ -802,13 +802,57 @@ export function isPredicateSubset(
   subset: LoadSubsetOptions,
   superset: LoadSubsetOptions
 ): boolean {
+  // When the superset has a limit, we can only determine subset relationship
+  // if the where clauses are equal (not just subset relationship).
+  //
+  // This is because a limited query only loads a portion of the matching rows.
+  // A more restrictive where clause might require rows outside that portion.
+  //
+  // Example: superset = {where: undefined, limit: 10, orderBy: desc}
+  //          subset = {where: LIKE 'search%', limit: 10, orderBy: desc}
+  // The top 10 items matching 'search%' might include items outside the overall top 10.
+  //
+  // However, if the where clauses are equal, then the subset relationship can
+  // be determined by orderBy and limit alone:
+  // Example: superset = {where: status='active', limit: 10, orderBy: desc}
+  //          subset = {where: status='active', limit: 5, orderBy: desc}
+  // The top 5 active items ARE contained in the top 10 active items.
+  if (superset.limit !== undefined) {
+    // For limited supersets, where clauses must be equal
+    if (!areWhereClausesEqual(subset.where, superset.where)) {
+      return false
+    }
+    return (
+      isOrderBySubset(subset.orderBy, superset.orderBy) &&
+      isLimitSubset(subset.limit, superset.limit)
+    )
+  }
+
+  // For unlimited supersets, use the normal subset logic
   return (
     isWhereSubset(subset.where, superset.where) &&
     isOrderBySubset(subset.orderBy, superset.orderBy) &&
     isLimitSubset(subset.limit, superset.limit)
   )
 }
 
+/**
+ * Check if two where clauses are structurally equal.
+ * Used for limited query subset checks where subset relationship isn't sufficient.
+ */
+function areWhereClausesEqual(
+  a: BasicExpression<boolean> | undefined,
+  b: BasicExpression<boolean> | undefined
+): boolean {
+  if (a === undefined && b === undefined) {
+    return true
+  }
+  if (a === undefined || b === undefined) {
+    return false
+  }
+  return areExpressionsEqual(a, b)
+}
+
 // ============================================================================
 // Helper functions
 // ============================================================================

packages/db/tests/query/predicate-utils.test.ts

@@ -648,7 +648,8 @@ describe(`isLimitSubset`, () => {
 })
 
 describe(`isPredicateSubset`, () => {
-  it(`should check all components`, () => {
+  it(`should check all components for unlimited superset`, () => {
+    // For unlimited supersets, where-subset logic applies
     const subset: LoadSubsetOptions = {
       where: gt(ref(`age`), val(20)),
       orderBy: [orderByClause(ref(`age`), `asc`)],
@@ -660,11 +661,66 @@ describe(`isPredicateSubset`, () => {
         orderByClause(ref(`age`), `asc`),
         orderByClause(ref(`name`), `desc`),
       ],
+      // No limit - unlimited superset
+    }
+    expect(isPredicateSubset(subset, superset)).toBe(true)
+  })
+
+  it(`should require equal where clauses for limited supersets`, () => {
+    // For limited supersets, where clauses must be EQUAL
+    const sameWhere = gt(ref(`age`), val(10))
+
+    const subset: LoadSubsetOptions = {
+      where: sameWhere,
+      orderBy: [orderByClause(ref(`age`), `asc`)],
+      limit: 5,
+    }
+    const superset: LoadSubsetOptions = {
+      where: sameWhere, // Same where clause
+      orderBy: [
+        orderByClause(ref(`age`), `asc`),
+        orderByClause(ref(`name`), `desc`),
+      ],
       limit: 20,
     }
     expect(isPredicateSubset(subset, superset)).toBe(true)
   })
 
+  it(`should return false for limited superset with different where clause`, () => {
+    // Even if subset's where is more restrictive, it can't be a subset
+    // of a limited superset with a different where clause.
+    // The top N items of "age > 20" may not be in the top M items of "age > 10"
+    const subset: LoadSubsetOptions = {
+      where: gt(ref(`age`), val(20)), // More restrictive
+      orderBy: [orderByClause(ref(`age`), `asc`)],
+      limit: 5,
+    }
+    const superset: LoadSubsetOptions = {
+      where: gt(ref(`age`), val(10)), // Less restrictive but LIMITED
+      orderBy: [orderByClause(ref(`age`), `asc`)],
+      limit: 20,
+    }
+    // This should be FALSE because the top 5 of "age > 20"
+    // might include items outside the top 20 of "age > 10"
+    expect(isPredicateSubset(subset, superset)).toBe(false)
+  })
+
+  it(`should return false for limited superset with no where vs subset with where`, () => {
+    // This is the reported bug case: pagination with search filter
+    const subset: LoadSubsetOptions = {
+      where: gt(ref(`age`), val(20)), // Has a filter
+      orderBy: [orderByClause(ref(`age`), `asc`)],
+      limit: 10,
+    }
+    const superset: LoadSubsetOptions = {
+      where: undefined, // No filter but LIMITED
+      orderBy: [orderByClause(ref(`age`), `asc`)],
+      limit: 10,
+    }
+    // The filtered results might include items outside the unfiltered top 10
+    expect(isPredicateSubset(subset, superset)).toBe(false)
+  })
+
   it(`should return false if where is not subset`, () => {
     const subset: LoadSubsetOptions = {
       where: gt(ref(`age`), val(5)),

packages/db/tests/query/subset-dedupe.test.ts

@@ -153,24 +153,68 @@ describe(`createDeduplicatedLoadSubset`, () => {
       },
     ]
 
+    const whereClause = gt(ref(`age`), val(10))
+
     // First call: age > 10, orderBy age asc, limit 10
     await deduplicated.loadSubset({
-      where: gt(ref(`age`), val(10)),
+      where: whereClause,
       orderBy: orderBy1,
       limit: 10,
     })
     expect(callCount).toBe(1)
 
-    // Second call: age > 20, orderBy age asc, limit 5 (subset)
+    // Second call: SAME where clause, same orderBy, smaller limit (subset)
+    // For limited queries, where clauses must be EQUAL for subset relationship
     const result = await deduplicated.loadSubset({
-      where: gt(ref(`age`), val(20)),
+      where: whereClause, // Same where clause
       orderBy: orderBy1,
       limit: 5,
     })
     expect(result).toBe(true)
     expect(callCount).toBe(1) // Should not call - subset of first
   })
 
+  it(`should NOT dedupe limited calls with different where clauses`, async () => {
+    let callCount = 0
+    const mockLoadSubset = () => {
+      callCount++
+      return Promise.resolve()
+    }
+
+    const deduplicated = new DeduplicatedLoadSubset({
+      loadSubset: mockLoadSubset,
+    })
+
+    const orderBy1: OrderBy = [
+      {
+        expression: ref(`age`),
+        compareOptions: {
+          direction: `asc`,
+          nulls: `last`,
+          stringSort: `lexical`,
+        },
+      },
+    ]
+
+    // First call: age > 10, orderBy age asc, limit 10
+    await deduplicated.loadSubset({
+      where: gt(ref(`age`), val(10)),
+      orderBy: orderBy1,
+      limit: 10,
+    })
+    expect(callCount).toBe(1)
+
+    // Second call: DIFFERENT where clause (age > 20) - should NOT be deduped
+    // even though age > 20 is "more restrictive" than age > 10,
+    // the top 5 of age > 20 might not be in the top 10 of age > 10
+    await deduplicated.loadSubset({
+      where: gt(ref(`age`), val(20)),
+      orderBy: orderBy1,
+      limit: 5,
+    })
+    expect(callCount).toBe(2) // Should call - different where clause
+  })
+
   it(`should call underlying for non-subset limited calls`, async () => {
     let callCount = 0
     const mockLoadSubset = () => {
@@ -671,17 +715,20 @@ describe(`createDeduplicatedLoadSubset`, () => {
         },
       ]
 
+      const whereClause = gt(ref(`age`), val(10))
+
       // First limited call
       await deduplicated.loadSubset({
-        where: gt(ref(`age`), val(10)),
+        where: whereClause,
         orderBy: orderBy1,
         limit: 10,
       })
       expect(callCount).toBe(1)
 
-      // Second limited call is a subset (stricter where and smaller limit)
+      // Second limited call is a subset (SAME where clause and smaller limit)
+      // For limited queries, where clauses must be EQUAL for subset relationship
       const subsetOptions = {
-        where: gt(ref(`age`), val(20)),
+        where: whereClause, // Same where clause
         orderBy: orderBy1,
         limit: 5,
       }
@@ -741,4 +788,98 @@ describe(`createDeduplicatedLoadSubset`, () => {
       expect(onDeduplicate).toHaveBeenCalledWith(subsetOptions)
     })
   })
+
+  describe(`limited queries with different where clauses`, () => {
+    // When a query has a limit, only the top N rows (by orderBy) are loaded.
+    // A subsequent query with a different where clause cannot reuse that data,
+    // even if the new where clause is "more restrictive", because the filtered
+    // top N might include rows outside the original unfiltered top N.
+
+    it(`should NOT dedupe when where clause differs on limited queries`, async () => {
+      let callCount = 0
+      const calls: Array<LoadSubsetOptions> = []
+      const mockLoadSubset = (options: LoadSubsetOptions) => {
+        callCount++
+        calls.push(options)
+        return Promise.resolve()
+      }
+
+      const deduplicated = new DeduplicatedLoadSubset({
+        loadSubset: mockLoadSubset,
+      })
+
+      const orderByCreatedAt: OrderBy = [
+        {
+          expression: ref(`created_at`),
+          compareOptions: {
+            direction: `desc`,
+            nulls: `last`,
+            stringSort: `lexical`,
+          },
+        },
+      ]
+
+      // First query: top 10 items with no filter
+      await deduplicated.loadSubset({
+        where: undefined,
+        orderBy: orderByCreatedAt,
+        limit: 10,
+      })
+      expect(callCount).toBe(1)
+
+      // Second query: top 10 items WITH a filter
+      // This requires a separate request because the filtered top 10
+      // might include items outside the unfiltered top 10
+      const searchWhere = and(eq(ref(`title`), val(`test`)))
+      await deduplicated.loadSubset({
+        where: searchWhere,
+        orderBy: orderByCreatedAt,
+        limit: 10,
+      })
+
+      expect(callCount).toBe(2)
+      expect(calls[1]?.where).toEqual(searchWhere)
+    })
+
+    it(`should dedupe when where clause is identical on limited queries`, async () => {
+      let callCount = 0
+      const mockLoadSubset = () => {
+        callCount++
+        return Promise.resolve()
+      }
+
+      const deduplicated = new DeduplicatedLoadSubset({
+        loadSubset: mockLoadSubset,
+      })
+
+      const orderByCreatedAt: OrderBy = [
+        {
+          expression: ref(`created_at`),
+          compareOptions: {
+            direction: `desc`,
+            nulls: `last`,
+            stringSort: `lexical`,
+          },
+        },
+      ]
+
+      // First query: top 10 items with no filter
+      await deduplicated.loadSubset({
+        where: undefined,
+        orderBy: orderByCreatedAt,
+        limit: 10,
+      })
+      expect(callCount).toBe(1)
+
+      // Second query: same where clause (undefined), smaller limit
+      // The top 5 are contained within the already-loaded top 10
+      const result = await deduplicated.loadSubset({
+        where: undefined,
+        orderBy: orderByCreatedAt,
+        limit: 5,
+      })
+      expect(result).toBe(true)
+      expect(callCount).toBe(1)
+    })
+  })
 })