add logic

Author: jaffrepaul

app/api/source-map/route.ts

@@ -0,0 +1,24 @@
+import {NextResponse} from 'next/server';
+
+import {isDeveloperDocs} from 'sentry-docs/isDeveloperDocs';
+import {getDevDocsFrontMatter, getDocsFrontMatter} from 'sentry-docs/mdx';
+
+/**
+ * API endpoint that returns a mapping of slugs to their source file paths.
+ * This is used by the 404 link checker to deduplicate pages that share the same source.
+ */
+export async function GET() {
+  const docs = await (isDeveloperDocs ? getDevDocsFrontMatter() : getDocsFrontMatter());
+
+  const sourceMap: Record<string, string | null> = {};
+
+  for (const doc of docs) {
+    // Normalize slug (remove trailing slash if present)
+    const slug = doc.slug.replace(/\/$/, '');
+    // sourcePath will be null for API-generated pages, which we want to keep
+    sourceMap[slug] = doc.sourcePath ?? null;
+  }
+
+  return NextResponse.json(sourceMap);
+}
+

scripts/lint-404s/README.md

@@ -0,0 +1,65 @@
+# 404 Link Checker
+
+This script checks all documentation pages for broken internal links (404s).
+
+## Usage
+
+```bash
+# Basic usage (with deduplication - recommended)
+bun ./scripts/lint-404s/main.ts
+
+# Show progress for each page
+bun ./scripts/lint-404s/main.ts --progress
+
+# Skip deduplication and check all pages (for debugging)
+bun ./scripts/lint-404s/main.ts --skip-deduplication
+
+# Filter to a specific path
+bun ./scripts/lint-404s/main.ts --path platforms/javascript
+```
+
+## Deduplication
+
+By default, the checker **deduplicates common files** to improve performance.
+
+### Why?
+
+The Sentry docs use a "common" file system where documentation is shared across multiple platforms. For example:
+
+- `/platforms/apple/common/configuration/index.mdx` is rendered as:
+  - `/platforms/apple/guides/ios/configuration/`
+  - `/platforms/apple/guides/macos/configuration/`
+  - `/platforms/apple/guides/watchos/configuration/`
+  - ... and many more
+
+Without deduplication, the checker would fetch and test the same content dozens of times, which:
+
+- Takes much longer to run
+- Wastes CI resources
+- Provides no additional value (the content is identical)
+
+### How it works
+
+1. The checker fetches a source map from `/api/source-map` that maps each slug to its source file
+2. It tracks which source files have been checked
+3. For common files, it only checks the first instance
+4. **API-generated pages** are always checked (they have no source file)
+
+This typically reduces the number of pages checked from **~9,000 to ~2,500**, a **72% reduction**.
+
+### When to use `--skip-deduplication`
+
+Use this flag to skip deduplication and verify that all rendered pages work correctly, even if they share the same source. This is rarely necessary but can help debug issues with:
+
+- Path routing
+- Platform-specific rendering bugs
+- Edge cases in the build system
+
+## Ignore List
+
+The `ignore-list.txt` file contains paths that should be skipped during checking. Add paths here (one per line) if they're known to be inaccessible or are special cases.
+
+## Exit Codes
+
+- `0` - No 404s found
+- `1` - 404s were detected

scripts/lint-404s/main.ts

@@ -13,6 +13,7 @@ const trimSlashes = (s: string) => s.replace(/(^\/|\/$)/g, '');
 const ignoreListFile = path.join(dirname(import.meta.url), './ignore-list.txt');
 
 const showProgress = process.argv.includes('--progress');
+const deduplicatePages = !process.argv.includes('--skip-deduplication');
 
 // Get the path filter if specified
 const pathFilterIndex = process.argv.indexOf('--path');
@@ -35,22 +36,73 @@ async function fetchWithFollow(url: URL | string): Promise<Response> {
   return r;
 }
 
+async function deduplicateSlugs(
+  allSlugs: string[]
+): Promise<{skippedCount: number; slugsToCheck: string[]}> {
+  try {
+    const sourceMap: Record<string, string | null> = await fetch(
+      `${baseURL}api/source-map`
+    ).then(r => r.json());
+
+    const checkedSources = new Set<string>();
+    const slugsToCheck: string[] = [];
+    let skippedCount = 0;
+
+    for (const slug of allSlugs) {
+      const normalizedSlug = slug.replace(/\/$/, '');
+      const sourcePath = sourceMap[normalizedSlug];
+
+      // Always check API-generated pages (no source file)
+      if (!sourcePath) {
+        slugsToCheck.push(slug);
+        continue;
+      }
+
+      // Skip if we've already checked this source file
+      if (checkedSources.has(sourcePath)) {
+        skippedCount++;
+        continue;
+      }
+
+      // First time seeing this source file
+      checkedSources.add(sourcePath);
+      slugsToCheck.push(slug);
+    }
+
+    return {skippedCount, slugsToCheck};
+  } catch (error) {
+    console.warn('⚠️  Failed to fetch source map:', error.message);
+    console.warn('Falling back to checking all pages...\n');
+    return {skippedCount: 0, slugsToCheck: allSlugs};
+  }
+}
+
 async function main() {
   const sitemap = await fetch(`${baseURL}sitemap.xml`).then(r => r.text());
 
-  const slugs = [...sitemap.matchAll(/<loc>([^<]*)<\/loc>/g)]
+  const allSlugs = [...sitemap.matchAll(/<loc>([^<]*)<\/loc>/g)]
     .map(l => l[1])
     .map(url => trimSlashes(new URL(url).pathname))
     .filter(Boolean)
     .filter(slug => (pathFilter ? slug.startsWith(pathFilter) : true));
-  const allSlugsSet = new Set(slugs);
-
-  if (pathFilter) {
-    console.log('Checking 404s on %d pages in /%s', slugs.length, pathFilter);
-  } else {
-    console.log('Checking 404s on %d pages', slugs.length);
+  const allSlugsSet = new Set(allSlugs);
+
+  // Deduplicate pages with same source file (default behavior)
+  const {skippedCount, slugsToCheck} = deduplicatePages
+    ? await deduplicateSlugs(allSlugs)
+    : {skippedCount: 0, slugsToCheck: allSlugs};
+
+  if (skippedCount > 0) {
+    console.log(
+      'Deduplication: checking %d unique pages (skipped %d duplicates)\n',
+      slugsToCheck.length,
+      skippedCount
+    );
   }
 
+  const pathInfo = pathFilter ? ` in /${pathFilter}` : '';
+  console.log('Checking 404s on %d pages%s', slugsToCheck.length, pathInfo);
+
   const all404s: {page404s: Link[]; slug: string}[] = [];
 
   // check if the slug equivalent of the href is in the sitemap
@@ -100,7 +152,7 @@ async function main() {
     return false;
   }
 
-  for (const slug of slugs) {
+  for (const slug of slugsToCheck) {
     const pageUrl = new URL(slug, baseURL);
     const now = performance.now();
     const html = await fetchWithFollow(pageUrl.href).then(r => r.text());
@@ -134,7 +186,7 @@ async function main() {
   }
 
   if (all404s.length === 0) {
-    console.log('\n\n🎉 No 404s found');
+    console.log('\n🎉 No 404s found');
     return false;
   }
   const numberOf404s = all404s.map(x => x.page404s.length).reduce((a, b) => a + b, 0);