fix(firecrawl): complete v2 response and depth compatibility

apps/docs/content/docs/integrations/firecrawl.mdx

@@ -184,7 +184,7 @@ Crawl entire websites and extract structured content from all accessible pages
 | --------- | ---- | -------- | ----------- |
 | `url` | string | Yes | The website URL to crawl \(e.g., "https://example.com" or "https://docs.example.com/guide") |
 | `limit` | number | No | Maximum number of pages to crawl \(e.g., 50, 100, 500\). Default: 100 |
-| `maxDepth` | number | No | Maximum depth to crawl from the starting URL \(e.g., 1, 2, 3\). Controls how many levels deep to follow links |
+| `maxDepth` | number | No | Maximum URL path depth \(e.g., 1 allows /docs and /docs/index.html\). Independent of link discovery depth |
 | `formats` | json | No | Output formats for scraped content \(e.g., \["markdown"\], \["markdown", "html"\], \["markdown", "links"\]\) |
 | `prompt` | string | No | Natural-language crawl guidance supplied by existing configurations |
 | `excludePaths` | json | No | URL paths to exclude from crawling \(e.g., \["/blog/*", "/admin/*", "/*.pdf"\]\) |

apps/sim/tools/firecrawl/crawl.test.ts

@@ -0,0 +1,75 @@
+import { isPlainRecord } from '@sim/utils/object'
+import { describe, expect, it } from 'vitest'
+import { crawlTool } from '@/tools/firecrawl/crawl'
+
+function crawlBody(options: {
+  maxDepth: number
+  maxDiscoveryDepth?: number
+  includePaths?: string[]
+  excludePaths?: string[]
+}) {
+  if (typeof crawlTool.request.body !== 'function') throw new Error('Missing request formatter')
+  const body = crawlTool.request.body({ apiKey: 'test', url: 'https://example.com', ...options })
+  if (!isPlainRecord(body)) throw new Error('Expected a JSON request body')
+  return body
+}
+
+describe('Firecrawl legacy crawl depth', () => {
+  it('bounds URL path segments without limiting discovery hops', () => {
+    const body = crawlBody({ maxDepth: 1, maxDiscoveryDepth: 4 })
+    const patterns = (body.includePaths as string[]).map((pattern) => new RegExp(pattern))
+    const accepts = (path: string) => patterns.some((pattern) => pattern.test(path))
+
+    expect(body.maxDiscoveryDepth).toBe(4)
+    expect(body.regexOnFullURL).toBe(false)
+    expect(accepts('/')).toBe(true)
+    expect(accepts('/docs')).toBe(true)
+    expect(accepts('/docs/index.html')).toBe(true)
+    expect(accepts('/docs/guide')).toBe(false)
+    expect(accepts('/docs/guide/page')).toBe(false)
+  })
+
+  it('keeps existing path filters while excluding paths beyond the legacy depth', () => {
+    const body = crawlBody({ maxDepth: 1, includePaths: ['^/docs'], excludePaths: ['private'] })
+    const patterns = (body.excludePaths as string[]).map((pattern) => new RegExp(pattern))
+    const excludes = (path: string) => patterns.some((pattern) => pattern.test(path))
+
+    expect(body.includePaths).toEqual(['^/docs'])
+    expect(excludes('/docs')).toBe(false)
+    expect(excludes('/docs/guide')).toBe(true)
+    expect(excludes('/private')).toBe(true)
+    expect(body.maxDiscoveryDepth).toBeUndefined()
+  })
+
+  it('honors a zero depth instead of dropping the limit', () => {
+    const body = crawlBody({ maxDepth: 0 })
+    const patterns = (body.includePaths as string[]).map((pattern) => new RegExp(pattern))
+    expect(patterns.some((pattern) => pattern.test('/'))).toBe(true)
+    expect(patterns.some((pattern) => pattern.test('/docs'))).toBe(false)
+  })
+
+  it.each<[number, string, boolean]>([
+    [0, '/index.html', false],
+    [0, '/index.php/', false],
+    [1, '/docs/index.html', false],
+    [1, '/docs/index.php', false],
+    [1, '/docs//index.html/', false],
+    [1, '/docs/guide/index.html', true],
+    [1, '/docs/index.html/child', true],
+    [1, '/docs/index.html.bak', true],
+    [1, '/docs/index.phpx', true],
+    [1, '/docs/inde', true],
+    [1, '/docs/index.htm', true],
+  ])('applies depth %s with existing includes to %s', (maxDepth, path, excluded) => {
+    const body = crawlBody({ maxDepth, includePaths: ['^/'] })
+    const patterns = (body.excludePaths as string[]).map((pattern) => new RegExp(pattern))
+    expect(patterns.some((pattern) => pattern.test(path))).toBe(excluded)
+  })
+
+  it.each([-1, 1.5, Number.NaN, Number.POSITIVE_INFINITY])(
+    'rejects an invalid legacy depth: %s',
+    (maxDepth) => {
+      expect(() => crawlBody({ maxDepth })).toThrow(/maxDepth/)
+    }
+  )
+})

apps/sim/tools/firecrawl/crawl.ts

@@ -1,5 +1,6 @@
 import { createLogger } from '@sim/logger'
 import { sleep } from '@sim/utils/helpers'
+import { escapeRegExp } from '@sim/utils/string'
 import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits'
 import { firecrawlHosting } from '@/tools/firecrawl/hosting'
 import {
@@ -41,7 +42,7 @@ export const crawlTool: ToolConfig<FirecrawlCrawlParams, FirecrawlCrawlResponse>
       required: false,
       visibility: 'user-or-llm',
       description:
-        'Maximum depth to crawl from the starting URL (e.g., 1, 2, 3). Controls how many levels deep to follow links',
+        'Maximum URL path depth (e.g., 1 allows /docs and /docs/index.html). Independent of link discovery depth',
     },
     formats: {
       type: 'json',
@@ -130,7 +131,7 @@ export const crawlTool: ToolConfig<FirecrawlCrawlParams, FirecrawlCrawlResponse>
       Authorization: `Bearer ${params.apiKey}`,
     }),
     body: (params) => {
-      const body: Record<string, any> = {
+      const body: Record<string, unknown> = {
         url: params.url,
         limit: Number(params.limit) || 100,
         scrapeOptions: params.scrapeOptions || {
@@ -140,8 +141,8 @@ export const crawlTool: ToolConfig<FirecrawlCrawlParams, FirecrawlCrawlResponse>
       }
 
       if (params.prompt) body.prompt = params.prompt
-      if (params.maxDepth) body.maxDiscoveryDepth = Number(params.maxDepth)
-      if (params.maxDiscoveryDepth) body.maxDiscoveryDepth = Number(params.maxDiscoveryDepth)
+      if (params.maxDiscoveryDepth != null)
+        body.maxDiscoveryDepth = Number(params.maxDiscoveryDepth)
       if (params.sitemap) body.sitemap = params.sitemap
       if (typeof params.crawlEntireDomain === 'boolean')
         body.crawlEntireDomain = params.crawlEntireDomain
@@ -154,6 +155,44 @@ export const crawlTool: ToolConfig<FirecrawlCrawlParams, FirecrawlCrawlResponse>
       if (params.maxConcurrency) body.maxConcurrency = Number(params.maxConcurrency)
       if (params.excludePaths) body.excludePaths = params.excludePaths
       if (params.includePaths) body.includePaths = params.includePaths
+      if (params.maxDepth != null) {
+        const maxDepth = Number(params.maxDepth)
+        if (!Number.isSafeInteger(maxDepth) || maxDepth < 0) {
+          throw new Error('maxDepth must be a non-negative integer')
+        }
+
+        // v1 depth counted URL path segments; v2 discovery depth counts link hops.
+        body.regexOnFullURL = false
+        if (params.includePaths?.length) {
+          const indexFiles = ['index.html', 'index.php']
+          const nonIndexSegmentPatterns = new Set<string>()
+          // Rust regex has no lookarounds, so enumerate incomplete prefixes and first mismatches.
+          for (const file of indexFiles) {
+            for (let length = 0; length <= file.length; length++) {
+              const prefix = file.slice(0, length)
+              const nextCharacters = indexFiles
+                .filter((indexFile) => indexFile.startsWith(prefix))
+                .map((indexFile) => indexFile.charAt(length))
+                .join('')
+              const escapedPrefix = escapeRegExp(prefix)
+              if (prefix && !indexFiles.includes(prefix)) {
+                nonIndexSegmentPatterns.add(escapedPrefix)
+              }
+              nonIndexSegmentPatterns.add(`${escapedPrefix}[^/${nextCharacters}][^/]*`)
+            }
+          }
+          const nonIndexBoundary = `^(/+[^/]+){${maxDepth}}/+(${[...nonIndexSegmentPatterns].join('|')})/*$`
+          body.excludePaths = [
+            ...(params.excludePaths ?? []),
+            `^(/+[^/]+){${maxDepth + 2}}|${nonIndexBoundary}`,
+          ]
+        } else {
+          body.includePaths = [
+            `^(/+[^/]+){0,${maxDepth}}/*$`,
+            `^(/+[^/]+){0,${maxDepth}}/+index\\.(html|php)/*$`,
+          ]
+        }
+      }
       if (params.webhook) body.webhook = params.webhook
       if (typeof params.zeroDataRetention === 'boolean')
         body.zeroDataRetention = params.zeroDataRetention

apps/sim/tools/firecrawl/map.test.ts

@@ -0,0 +1,20 @@
+import { jsonResponse } from '@sim/testing'
+import { describe, expect, it } from 'vitest'
+import { mapTool } from '@/tools/firecrawl/map'
+
+describe('Firecrawl v2 map results', () => {
+  it('preserves URL strings for workflows when the provider returns link objects', async () => {
+    if (!mapTool.transformResponse) throw new Error('Missing response transform')
+    const result = await mapTool.transformResponse(
+      jsonResponse({
+        success: true,
+        links: [
+          { url: 'https://example.com', title: 'Home', description: 'Home page' },
+          { url: 'https://example.com/docs', title: 'Docs' },
+        ],
+      })
+    )
+
+    expect(result.output.links).toEqual(['https://example.com', 'https://example.com/docs'])
+  })
+})

apps/sim/tools/firecrawl/map.ts

@@ -79,7 +79,7 @@ export const mapTool: ToolConfig<MapParams, MapResponse> = {
       Authorization: `Bearer ${params.apiKey}`,
     }),
     body: (params) => {
-      const body: Record<string, any> = {
+      const body: Record<string, unknown> = {
         url: params.url,
       }
 
@@ -98,13 +98,13 @@ export const mapTool: ToolConfig<MapParams, MapResponse> = {
   },
 
   transformResponse: async (response: Response) => {
-    const data = await response.json()
+    const data: { success: boolean; links?: { url: string }[] } = await response.json()
 
     return {
       success: data.success,
       output: {
         success: data.success,
-        links: data.links || [],
+        links: data.links?.map((link) => link.url) ?? [],
         creditsUsed: 1,
       },
     }

apps/sim/tools/firecrawl/search.test.ts

@@ -0,0 +1,32 @@
+import { jsonResponse } from '@sim/testing'
+import { describe, expect, it } from 'vitest'
+import { searchTool } from '@/tools/firecrawl/search'
+
+describe('Firecrawl v2 search results', () => {
+  it('preserves the workflow result array when the provider groups results by source', async () => {
+    if (!searchTool.transformResponse) throw new Error('Missing response transform')
+    const result = await searchTool.transformResponse(
+      jsonResponse({
+        success: true,
+        data: {
+          web: [{ url: 'https://example.com', title: 'Example', description: 'A web result' }],
+        },
+        creditsUsed: 2,
+      })
+    )
+
+    expect(result.output.data).toEqual([
+      { url: 'https://example.com', title: 'Example', description: 'A web result' },
+    ])
+    expect(result.output.creditsUsed).toBe(2)
+  })
+
+  it('returns an empty result array when the provider finds no web results', async () => {
+    if (!searchTool.transformResponse) throw new Error('Missing response transform')
+    const result = await searchTool.transformResponse(
+      jsonResponse({ success: true, data: {}, creditsUsed: 0 })
+    )
+
+    expect(result.output.data).toEqual([])
+  })
+})

apps/sim/tools/firecrawl/search.ts

@@ -56,12 +56,11 @@ export const searchTool: ToolConfig<SearchParams, SearchResponse> = {
       Authorization: `Bearer ${params.apiKey}`,
     }),
     body: (params) => {
-      const body: Record<string, any> = {
+      const body: Record<string, unknown> = {
         query: params.query,
+        limit: params.limit ? Number(params.limit) : 5,
       }
 
-      // Add optional parameters if provided (truthy check filters empty strings, null, undefined)
-      if (params.limit) body.limit = Number(params.limit)
       if (params.sources) body.sources = params.sources
       if (params.categories) body.categories = params.categories
       if (params.tbs) body.tbs = params.tbs
@@ -77,12 +76,15 @@ export const searchTool: ToolConfig<SearchParams, SearchResponse> = {
   },
 
   transformResponse: async (response: Response) => {
-    const data = await response.json()
+    const data: {
+      data?: { web?: SearchResponse['output']['data'] }
+      creditsUsed?: number
+    } = await response.json()
 
     return {
       success: true,
       output: {
-        data: data.data,
+        data: data.data?.web ?? [],
         creditsUsed: data.creditsUsed,
       },
     }

scripts/check-explicit-any.baseline.json

@@ -444,11 +444,9 @@
     "apps/sim/tools/exa/utils.ts": 5,
     "apps/sim/tools/firecrawl/agent.ts": 2,
     "apps/sim/tools/firecrawl/batch-scrape.ts": 3,
-    "apps/sim/tools/firecrawl/crawl.ts": 2,
+    "apps/sim/tools/firecrawl/crawl.ts": 1,
     "apps/sim/tools/firecrawl/extract.ts": 3,
-    "apps/sim/tools/firecrawl/map.ts": 1,
     "apps/sim/tools/firecrawl/scrape.ts": 1,
-    "apps/sim/tools/firecrawl/search.ts": 1,
     "apps/sim/tools/firecrawl/types.ts": 8,
     "apps/sim/tools/function/types.ts": 1,
     "apps/sim/tools/github/add_assignees.ts": 3,