Improve search filtering with server-side type aggregations and post_filter (#2305)

* Improve search filtering with server-side type aggregations and post_filter

- Implement server-side type filtering using Elasticsearch `post_filter` for accurate aggregation counts
- Add terms aggregation on `type` field to return doc/api counts from the server
- Refactor search API types: rename `SearchRequest`/`SearchResponse` to `SearchApiRequest`/`SearchApiResponse`
- Replace tuple return type in `ISearchGateway` with new `SearchResult` record
- Return aggregations nested under `aggregations.type` for future extensibility
- Update frontend to fetch filtered results from server instead of client-side filtering
- Add `typeFilter` state to search store for consistent filter state management
- Split `ElasticsearchIngestChannel` and `ElasticsearchMarkdownExporter` into partial files for better code organization

- `ISearchGateway.SearchAsync` now returns `SearchResult` with `TotalHits`, `Results`, and `Aggregations`
- Search endpoint accepts `type` query parameter (values: `doc`, `api`)
- `post_filter` ensures aggregation counts reflect total matches regardless of active filter
- New `SearchAggregations` record wraps type counts for clean JSON structure

- Filter selection triggers new API request with `?type=` parameter
- Aggregation counts from server displayed in filter buttons
- Removed client-side result filtering (server handles it)
- Query cache keys include `typeFilter` for proper result caching

* fix formatting

* remove log line

Author: Mpdreamz

src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/Search/SearchResults/SearchResults.tsx

@@ -1,6 +1,6 @@
 import { SearchOrAskAiErrorCallout } from '../../SearchOrAskAiErrorCallout'
 import { useSearchActions, useSearchTerm } from '../search.store'
-import { useSearchFilters, type FilterType } from '../useSearchFilters'
+import { useSearchFilters, type TypeFilter } from '../useSearchFilters'
 import { useSearchQuery } from '../useSearchQuery'
 import { SearchResultListItem } from './SearchResultsListItem'
 import {
@@ -18,7 +18,6 @@ import {
 import { css } from '@emotion/react'
 import { useDebounce } from '@uidotdev/usehooks'
 import { useEffect, useRef, useCallback } from 'react'
-import type { MouseEvent } from 'react'
 
 interface SearchResultsProps {
     onKeyDown?: (
@@ -61,9 +60,10 @@ export const SearchResults = ({
 
     const { data, error, isLoading } = useSearchQuery()
 
-    const { selectedFilters, handleFilterClick, filteredResults, counts } =
+    const { selectedFilter, handleFilterClick, filteredResults, counts } =
         useSearchFilters({
             results: data?.results ?? [],
+            aggregations: data?.aggregations,
         })
 
     const isInitialLoading = isLoading && !data
@@ -84,7 +84,7 @@ export const SearchResults = ({
             {!error && (
                 <>
                     <Filter
-                        selectedFilters={selectedFilters}
+                        selectedFilter={selectedFilter}
                         onFilterClick={handleFilterClick}
                         counts={counts}
                         isLoading={isInitialLoading}
@@ -201,13 +201,13 @@ export const SearchResults = ({
 }
 
 const Filter = ({
-    selectedFilters,
+    selectedFilter,
     onFilterClick,
     counts,
     isLoading,
 }: {
-    selectedFilters: Set<FilterType>
-    onFilterClick: (filter: FilterType, event?: MouseEvent) => void
+    selectedFilter: TypeFilter
+    onFilterClick: (filter: TypeFilter) => void
     counts: {
         apiResultsCount: number
         docsResultsCount: number
@@ -245,12 +245,12 @@ const Filter = ({
                     color="text"
                     // @ts-expect-error: xs is valid size according to EuiButton docs
                     size="xs"
-                    fill={selectedFilters.has('all')}
+                    fill={selectedFilter === 'all'}
                     isLoading={isLoading}
-                    onClick={(e: MouseEvent) => onFilterClick('all', e)}
+                    onClick={() => onFilterClick('all')}
                     css={buttonStyle}
                     aria-label={`Show all results, ${totalCount} total`}
-                    aria-pressed={selectedFilters.has('all')}
+                    aria-pressed={selectedFilter === 'all'}
                 >
                     {isLoading ? 'ALL' : `ALL (${totalCount})`}
                 </EuiButton>
@@ -264,12 +264,12 @@ const Filter = ({
                     color="text"
                     // @ts-expect-error: xs is valid size according to EuiButton docs
                     size="xs"
-                    fill={selectedFilters.has('doc')}
+                    fill={selectedFilter === 'doc'}
                     isLoading={isLoading}
-                    onClick={(e: MouseEvent) => onFilterClick('doc', e)}
+                    onClick={() => onFilterClick('doc')}
                     css={buttonStyle}
                     aria-label={`Filter to documentation results, ${docsResultsCount} available`}
-                    aria-pressed={selectedFilters.has('doc')}
+                    aria-pressed={selectedFilter === 'doc'}
                 >
                     {isLoading ? 'DOCS' : `DOCS (${docsResultsCount})`}
                 </EuiButton>
@@ -283,12 +283,12 @@ const Filter = ({
                     color="text"
                     // @ts-expect-error: xs is valid size according to EuiButton docs
                     size="xs"
-                    fill={selectedFilters.has('api')}
+                    fill={selectedFilter === 'api'}
                     isLoading={isLoading}
-                    onClick={(e: MouseEvent) => onFilterClick('api', e)}
+                    onClick={() => onFilterClick('api')}
                     css={buttonStyle}
                     aria-label={`Filter to API results, ${apiResultsCount} available`}
-                    aria-pressed={selectedFilters.has('api')}
+                    aria-pressed={selectedFilter === 'api'}
                 >
                     {isLoading ? 'API' : `API (${apiResultsCount})`}
                 </EuiButton>

src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/Search/search.store.ts

@@ -1,25 +1,33 @@
 import { create } from 'zustand/react'
 
+export type TypeFilter = 'all' | 'doc' | 'api'
+
 interface SearchState {
     searchTerm: string
     page: number
+    typeFilter: TypeFilter
     actions: {
         setSearchTerm: (term: string) => void
         setPageNumber: (page: number) => void
+        setTypeFilter: (filter: TypeFilter) => void
         clearSearchTerm: () => void
     }
 }
 
 export const searchStore = create<SearchState>((set) => ({
     searchTerm: '',
     page: 1,
+    typeFilter: 'all',
     actions: {
         setSearchTerm: (term: string) => set({ searchTerm: term }),
         setPageNumber: (page: number) => set({ page }),
-        clearSearchTerm: () => set({ searchTerm: '' }),
+        setTypeFilter: (filter: TypeFilter) =>
+            set({ typeFilter: filter, page: 0 }),
+        clearSearchTerm: () => set({ searchTerm: '', typeFilter: 'all' }),
     },
 }))
 
 export const useSearchTerm = () => searchStore((state) => state.searchTerm)
 export const usePageNumber = () => searchStore((state) => state.page)
+export const useTypeFilter = () => searchStore((state) => state.typeFilter)
 export const useSearchActions = () => searchStore((state) => state.actions)

src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/Search/useSearchFilters.ts

@@ -1,67 +1,42 @@
+import {
+    useTypeFilter,
+    useSearchActions,
+    type TypeFilter,
+} from './search.store'
 import { SearchResponse } from './useSearchQuery'
-import { useState, useMemo } from 'react'
-import type { MouseEvent } from 'react'
-
-export type FilterType = 'all' | 'doc' | 'api'
 
 interface UseSearchFiltersOptions {
     results: SearchResponse['results']
+    aggregations?: SearchResponse['aggregations']
 }
 
-export const useSearchFilters = ({ results }: UseSearchFiltersOptions) => {
-    const [selectedFilters, setSelectedFilters] = useState<Set<FilterType>>(
-        new Set(['all'])
-    )
-
-    const isMultiSelectModifierPressed = (event?: MouseEvent): boolean => {
-        return !!(event && (event.metaKey || event.altKey || event.ctrlKey))
-    }
+export const useSearchFilters = ({
+    results,
+    aggregations,
+}: UseSearchFiltersOptions) => {
+    const typeFilter = useTypeFilter()
+    const { setTypeFilter } = useSearchActions()
 
-    const toggleFilter = (
-        currentFilters: Set<FilterType>,
-        filter: FilterType
-    ): Set<FilterType> => {
-        const newFilters = new Set(currentFilters)
-        newFilters.delete('all')
-        if (newFilters.has(filter)) {
-            newFilters.delete(filter)
-        } else {
-            newFilters.add(filter)
-        }
-        return newFilters.size === 0 ? new Set(['all']) : newFilters
+    const handleFilterClick = (filter: TypeFilter) => {
+        setTypeFilter(filter)
     }
 
-    const handleFilterClick = (filter: FilterType, event?: MouseEvent) => {
-        if (filter === 'all') {
-            setSelectedFilters(new Set(['all']))
-            return
-        }
+    // Results come pre-filtered from the server, so we just return them directly
+    const filteredResults = results
 
-        if (isMultiSelectModifierPressed(event)) {
-            setSelectedFilters((prev) => toggleFilter(prev, filter))
-        } else {
-            setSelectedFilters(new Set([filter]))
-        }
-    }
-
-    const filteredResults = useMemo(() => {
-        if (selectedFilters.has('all')) {
-            return results
-        }
-        return results.filter((result) => selectedFilters.has(result.type))
-    }, [results, selectedFilters])
-
-    const counts = useMemo(() => {
-        const apiResultsCount = results.filter((r) => r.type === 'api').length
-        const docsResultsCount = results.filter((r) => r.type === 'doc').length
-        const totalCount = docsResultsCount + apiResultsCount
-        return { apiResultsCount, docsResultsCount, totalCount }
-    }, [results])
+    const typeAggregations = aggregations?.type
+    const apiResultsCount = typeAggregations?.['api'] ?? 0
+    const docsResultsCount = typeAggregations?.['doc'] ?? 0
+    const totalCount = docsResultsCount + apiResultsCount
+    const counts = { apiResultsCount, docsResultsCount, totalCount }
 
     return {
-        selectedFilters,
+        selectedFilter: typeFilter,
         handleFilterClick,
         filteredResults,
         counts,
     }
 }
+
+// Re-export TypeFilter for convenience
+export type { TypeFilter }

src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/Search/useSearchQuery.ts

@@ -8,7 +8,7 @@ import {
 import { traceSpan } from '../../../telemetry/tracing'
 import { createApiErrorFromResponse, shouldRetry } from '../errorHandling'
 import { ApiError } from '../errorHandling'
-import { usePageNumber, useSearchTerm } from './search.store'
+import { usePageNumber, useSearchTerm, useTypeFilter } from './search.store'
 import {
     useIsSearchAwaitingNewInput,
     useSearchCooldownActions,
@@ -41,19 +41,25 @@ const SearchResultItem = z.object({
 
 export type SearchResultItem = z.infer<typeof SearchResultItem>
 
+const SearchAggregations = z.object({
+    type: z.record(z.string(), z.number()).optional(),
+})
+
 const SearchResponse = z.object({
     results: z.array(SearchResultItem),
     totalResults: z.number(),
     pageCount: z.number(),
     pageNumber: z.number(),
     pageSize: z.number(),
+    aggregations: SearchAggregations.optional(),
 })
 
 export type SearchResponse = z.infer<typeof SearchResponse>
 
 export const useSearchQuery = () => {
     const searchTerm = useSearchTerm()
     const pageNumber = usePageNumber() + 1
+    const typeFilter = useTypeFilter()
     const trimmedSearchTerm = searchTerm.trim()
     const debouncedSearchTerm = useDebounce(trimmedSearchTerm, 300)
     const isCooldownActive = useIsSearchCooldownActive()
@@ -80,7 +86,11 @@ export const useSearchQuery = () => {
     const query = useQuery<SearchResponse, ApiError>({
         queryKey: [
             'search',
-            { searchTerm: debouncedSearchTerm.toLowerCase(), pageNumber },
+            {
+                searchTerm: debouncedSearchTerm.toLowerCase(),
+                pageNumber,
+                typeFilter,
+            },
         ],
         queryFn: async ({ signal }) => {
             // Don't create span for empty searches
@@ -101,6 +111,11 @@ export const useSearchQuery = () => {
                     page: pageNumber.toString(),
                 })
 
+                // Only add type filter if not 'all'
+                if (typeFilter !== 'all') {
+                    params.set('type', typeFilter)
+                }
+
                 const response = await fetch(
                     '/docs/_api/v1/search?' + params.toString(),
                     { signal }
@@ -140,10 +155,14 @@ export const useSearchQuery = () => {
         queryClient.cancelQueries({
             queryKey: [
                 'search',
-                { searchTerm: debouncedSearchTerm.toLowerCase(), pageNumber },
+                {
+                    searchTerm: debouncedSearchTerm.toLowerCase(),
+                    pageNumber,
+                    typeFilter,
+                },
             ],
         })
-    }, [queryClient, debouncedSearchTerm, pageNumber])
+    }, [queryClient, debouncedSearchTerm, pageNumber, typeFilter])
 
     return {
         ...query,

src/Elastic.Markdown/Exporters/Elasticsearch/ElasticsearchIngestChannel.Mapping.cs

@@ -113,7 +113,8 @@ protected static string CreateMapping(string? inferenceId) =>
 		$$"""
 		  {
 		    "properties": {
-		      "url" : {
+		      "type": { "type" : "keyword", "normalizer": "keyword_normalizer" },
+		      "url": {
 		        "type": "keyword",
 		        "fields": {
 		          "match": { "type": "text" },

src/api/Elastic.Documentation.Api.Core/Search/ISearchGateway.cs

@@ -6,10 +6,18 @@ namespace Elastic.Documentation.Api.Core.Search;
 
 public interface ISearchGateway
 {
-	Task<(int TotalHits, List<SearchResultItem> Results)> SearchAsync(
+	Task<SearchResult> SearchAsync(
 		string query,
 		int pageNumber,
 		int pageSize,
+		string? filter = null,
 		Cancel ctx = default
 	);
 }
+
+public record SearchResult
+{
+	public required int TotalHits { get; init; }
+	public required List<SearchResultItem> Results { get; init; }
+	public IReadOnlyDictionary<string, long> Aggregations { get; init; } = new Dictionary<string, long>();
+}