Merge pull request #122 from cipherCOM/main

feat: add exclude pattern for links in config

Author: marcelovicentegc

src/config.ts

@@ -20,7 +20,12 @@ export const configSchema = z.object({
    * @default ""
    */
   match: z.string().or(z.array(z.string())),
-
+  /**
+   * Pattern to match against for links on a page to exclude from crawling
+   * @example "https://www.builder.io/c/docs/**"
+   * @default ""
+   */
+  exclude: z.string().or(z.array(z.string())).optional(),
   /**
    * Selector to grab the inner text from
    * @example ".docs-builder-container"

src/core.ts

@@ -92,6 +92,8 @@ export async function crawl(config: Config) {
         await enqueueLinks({
           globs:
             typeof config.match === "string" ? [config.match] : config.match,
+          exclude:
+            typeof config.exclude === "string" ? [config.exclude] : config.exclude ?? [],
         });
       },
       // Comment this option to scrape the full website.