{"openapi":"3.0.3","info":{"title":"JS Crawler API","version":"0.1.0","description":"JS-enabled crawler API powered by Playwright. Render a URL, return HTML + metadata, optional selector extraction, and an optional screenshot (configured via required screenshot options)."},"components":{"securitySchemes":{"ApiKeyAuth":{"type":"apiKey","in":"header","name":"X-Api-Key"}},"schemas":{"CrawlRequest":{"type":"object","additionalProperties":false,"required":["url","waitUntil","extraWaitMs","maxHtmlBytes","screenshot"],"properties":{"url":{"type":"string","description":"Target URL (http/https)."},"renderJs":{"type":"boolean","default":true,"description":"When false, JS execution is disabled (no-JS crawl). When no-JS is requested and the request does not require browser-only features (screenshot, waitForSelector, selector extraction, blockUrlPatterns), the server may use a fast direct-HTTP fetch path instead of launching Chromium."},"waitUntil":{"type":"array","minItems":1,"items":{"type":"string","enum":["domcontentloaded","load","networkidle"]},"description":"Required. Multi-select load states to wait for (best-effort)."},"timeoutMs":{"type":"number","minimum":1000,"maximum":180000},"viewport":{"type":"object","additionalProperties":false,"required":["width","height"],"properties":{"width":{"type":"number"},"height":{"type":"number"}}},"userAgent":{"type":"string"},"headers":{"type":"object","additionalProperties":{"type":"string"},"description":"Additional request headers (some hop-by-hop headers are ignored)."},"locale":{"type":"string"},"timezoneId":{"type":"string"},"geolocation":{"type":"object","additionalProperties":false,"required":["latitude","longitude"],"properties":{"latitude":{"type":"number"},"longitude":{"type":"number"},"accuracy":{"type":"number"}}},"deviceScaleFactor":{"type":"number"},"isMobile":{"type":"boolean"},"hasTouch":{"type":"boolean"},"colorScheme":{"type":"string","enum":["light","dark","no-preference"]},"blockUrlPatterns":{"type":"array","items":{"type":"string"},"description":"Optional. URL patterns to block. Supports '*' wildcards; patterns without '*' are treated as substring matches."},"extraWaitMs":{"type":"number","minimum":0,"maximum":180000,"description":"Required."},"waitForSelector":{"type":"string"},"extract":{"type":"object","additionalProperties":false,"properties":{"includeHtml":{"type":"boolean","default":true},"selectors":{"type":"object","additionalProperties":{"oneOf":[{"type":"string","description":"Selector shorthand (defaults to text)."},{"type":"object","additionalProperties":false,"required":["selector"],"properties":{"selector":{"type":"string"},"how":{"type":"string","enum":["text","html","attr"],"default":"text"},"attr":{"type":"string","description":"Required when how=attr."}}}]}}}},"formats":{"type":"array","minItems":1,"items":{"type":"string","enum":["html","markdown","text","metadata","links"]},"description":"Optional. LLM-friendly output formats to return. When present this is authoritative for which outputs are returned (e.g. [\"markdown\",\"metadata\"]). When omitted, HTML is returned per extract.includeHtml. Derived formats are computed server-side from the rendered HTML."},"respectRobots":{"type":"boolean","description":"Optional. When true, the crawl is refused (403 ROBOTS_DISALLOWED) if the host's robots.txt disallows the URL for the effective user agent. Defaults to server ROBOTS_RESPECT_DEFAULT."},"maxRetries":{"type":"integer","minimum":0,"maximum":5,"description":"Optional. Max extra attempts on a detected anti-bot block (rotating fingerprint + proxy). Capped by the server's BLOCK_RETRY_MAX."},"includeDebug":{"type":"boolean"},"maxHtmlBytes":{"type":"number","minimum":1,"maximum":25,"description":"Required. Maximum rendered HTML size, specified in MB."},"screenshot":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"type":{"type":"string","enum":["png","jpeg"]},"fullPage":{"type":"boolean","description":"Defaults to true."},"quality":{"type":"number","minimum":0,"maximum":100,"description":"JPEG only."}},"description":"Required. Provide screenshot config (enabled true/false)."},"fingerprint":{"type":"string","description":"Optional anti-blocking fingerprint profile. One of \"win-chrome\", \"mac-chrome\", \"android-chrome\", or \"random\" (default, server-configurable). The profile supplies a coherent userAgent / viewport / locale / timezone / client-hint set; any explicit fields above override it."},"proxy":{"description":"Optional per-request proxy override. Either a string (\"scheme://user:pass@host:port\") or an object. When omitted, a proxy may be chosen from the server-side pool (sticky per host).","oneOf":[{"type":"string"},{"type":"object","additionalProperties":false,"required":["server"],"properties":{"server":{"type":"string","description":"e.g. http://host:port or socks5://host:port"},"username":{"type":"string"},"password":{"type":"string"}}}]}}},"CrawlSuccess":{"type":"object","additionalProperties":false,"required":["finalUrl","timings"],"properties":{"html":{"type":"string"},"finalUrl":{"type":"string"},"status":{"type":"number"},"timings":{"type":"object","additionalProperties":false,"required":["startedAtMs","finishedAtMs","totalMs"],"properties":{"startedAtMs":{"type":"number"},"finishedAtMs":{"type":"number"},"totalMs":{"type":"number"},"gotoMs":{"type":"number"}}},"extracted":{"type":"object","additionalProperties":true},"markdown":{"type":"string","description":"Clean Markdown of the main content (formats: markdown)."},"text":{"type":"string","description":"Boilerplate-free main-content text (formats: text)."},"metadata":{"type":"object","description":"Page metadata (formats: metadata).","additionalProperties":true,"properties":{"title":{"type":"string"},"description":{"type":"string"},"lang":{"type":"string"},"canonical":{"type":"string"},"favicon":{"type":"string"},"author":{"type":"string"},"siteName":{"type":"string"},"openGraph":{"type":"object","additionalProperties":{"type":"string"}},"twitter":{"type":"object","additionalProperties":{"type":"string"}},"jsonLd":{"type":"array","items":{}},"meta":{"type":"object","additionalProperties":{"type":"string"}}}},"links":{"type":"array","description":"Absolute, de-duplicated links found on the page (formats: links).","items":{"type":"object","additionalProperties":false,"required":["url"],"properties":{"url":{"type":"string"},"text":{"type":"string"},"rel":{"type":"string"}}}},"debug":{"type":"object","additionalProperties":false,"required":["blockedRequests","requestFailed","consoleErrors"],"properties":{"blockedRequests":{"type":"number"},"requestFailed":{"type":"number"},"consoleErrors":{"type":"number"},"mainDocumentStatus":{"type":"number"},"attempts":{"type":"number","description":"Total attempts made (including retries)."},"blocked":{"type":"boolean","description":"Whether the final result still looked blocked."},"blockReason":{"type":"string"}}},"screenshot":{"type":"object","additionalProperties":false,"required":["url","expiresAtMs","contentType"],"properties":{"url":{"type":"string","description":"Relative URL to download the asset (e.g. /assets/<id>)."},"expiresAtMs":{"type":"number"},"contentType":{"type":"string"}}}}},"ErrorResponse":{"type":"object","additionalProperties":false,"required":["error"],"properties":{"error":{"type":"object","required":["code","message"],"properties":{"code":{"type":"string","description":"Machine-readable error code.","enum":["BAD_REQUEST","UNSUPPORTED_PROTOCOL","SSRF_BLOCKED","TIMEOUT","PAYLOAD_TOO_LARGE","SERVICE_UNAVAILABLE","ROBOTS_DISALLOWED","CRAWL_FAILED","UNAUTHORIZED","NOT_FOUND"]},"message":{"type":"string"},"httpStatus":{"type":"integer","format":"int32"},"detail":{}},"additionalProperties":true}}}}},"tags":[{"name":"meta","description":"Health and misc endpoints."},{"name":"crawl","description":"Crawl endpoints."},{"name":"assets","description":"Temporary assets."}],"paths":{"/":{"get":{"tags":["meta"],"summary":"Homepage","description":"Links to docs and UI.","security":[],"responses":{"200":{"description":"HTML page"}}}},"/healthz":{"get":{"tags":["meta"],"summary":"Liveness check","security":[],"responses":{"200":{"description":"Process is up","content":{"application/json":{"schema":{"type":"object","additionalProperties":false,"required":["ok"],"properties":{"ok":{"type":"boolean"}}}}}}}}},"/readyz":{"get":{"tags":["meta"],"summary":"Readiness check","description":"Verifies Chromium can launch and is connected. Returns 503 when not ready.","security":[],"responses":{"200":{"description":"Ready (browser connected)","content":{"application/json":{"schema":{"type":"object","additionalProperties":true,"required":["ok"],"properties":{"ok":{"type":"boolean"},"browser":{"type":"boolean"}}}}}},"503":{"description":"Not ready (browser unavailable)"}}}},"/v1/crawl":{"post":{"tags":["crawl"],"summary":"Crawl a URL","description":"Renders a URL in headless Chromium and returns rendered HTML (optional), metadata, optional extracted selectors, and an optional screenshot asset URL.","security":[{"ApiKeyAuth":[]}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CrawlRequest"}}}},"responses":{"200":{"description":"Crawl result (success)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CrawlSuccess"}}}},"400":{"description":"Bad request","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"}}}},"401":{"description":"Unauthorized","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"}}}},"403":{"description":"Blocked by SSRF policy or robots.txt","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"}}}},"413":{"description":"Rendered HTML too large","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"}}}},"500":{"description":"Unexpected crawl failure","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"}}}},"503":{"description":"Service unavailable","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"}}}},"504":{"description":"Timeout","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"}}}}}}},"/assets/{id}":{"get":{"tags":["assets"],"summary":"Download a temporary asset","description":"Downloads a screenshot (or other temporary asset) created by a crawl. Assets expire ~10 minutes.","security":[{"ApiKeyAuth":[]}],"parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string"},"description":"Asset ID."}],"responses":{"200":{"description":"Binary asset"},"401":{"description":"Unauthorized","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"}}}},"404":{"description":"Not found / expired","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"}}}}}}}}}