CheerioCrawler

RequestQueue

requestHandler

preNavigationHooks

ts
preNavigationHooks: [
  async ({ request }) => {
    if (shouldAbortRequest(request)) {
      request.noRetry = true;
      throw new CustomAbortError();
    }
  },
]

errorHandler

failedRequestHandler

Request failed and reached maximum retries

request.skipNavigation = true

ts
preNavigationHooks: [
  async ({ request }) => {
    if (shouldAbortRequest(request)) {
      request.skipNavigation = true;
    }
  },
]

requestHandler

maxRequestsPerCrawl

maxRequestsPerCrawl: 20000

const configuration = {
  purgeOnStart: false,
  availableMemoryRatio: 0.75,
  maxMemoryOverloadedRatio: 0.75,
  maxEventLoopOverloadedRatio: 0.75,
  maxCpuOverloadedRatio: 0.75,
  maxClientOverloadedRatio: 0.7,
  desiredConcurrencyRatio: 0.5,
  persistStorage: true,
  persistStateIntervalMillis: 10000,
}

const crawler = new PlaywrightCrawler(
  {
    keepAlive: true,
    minConcurrency: 126,
    maxConcurrency: 252,
    maxRequestRetries: 5,
    requestHandlerTimeoutSecs: 10,
    proxyConfiguration,
    useSessionPool: false, // for performance and less detection must be false
    sessionPoolOptions: { maxPoolSize: 426 },
    persistCookiesPerSession: false,

yaml
jobs:
  - name: "amazon.de"
    enabled: true

    crawler:
      id: "test"
      enabled: true
      config:
        urls:
          - "https://example.com"

  - name: "amazon.fr"
    enabled: true

    crawler:
      id: "test"
      enabled: true
      config:
        urls:
          - "https://example.com"

typescript
export const testCrawler = createCrawler({
  id: "test",

  configSchema: z.object({
    urls: z.array(z.string()),
  }),

  handler: async ({ urls }) => {
    if (!crawler) {
      crawler = new PlaywrightCrawler({
        async requestHandler({ request, log }) {
          log.info(`Processing: ${request.url}`);
        },
      });
    }
    await crawler.run(urls);
  },
});

requestHandler

crawler.addRequests()

testCrawler

PlaywrightCrawler

CheerioCrawler

class SiteXCrawlerDefinition {
  getCrawler() { return new PlaywrightCrawler() }
}

getCrawler

BasicCrawler

crawlee@3.12.1

apify@^3.0.0

typescript
const proxyConfiguration = await Actor.createProxyConfiguration();
const crawler = new CheerioCrawler({
  proxyConfiguration: proxyConfiguration, // ❌ Type error
  requestHandler: router,
});

as any

Error: Detected incompatible Crawlee version used by the SDK.
User installed 3.12.1 but the SDK uses 3.15.3.
    at checkCrawleeVersion (node_modules/apify/utils.js:68)
    at Actor.init (node_modules/apify/main.js:123)

json
{
  "dependencies": {
    "apify": "^3.0.0",
    "crawlee": "3.12.1"
  },
  "overrides": {
    "@crawlee/core": "3.12.1",
    "@crawlee/types": "3.12.1",
    "@crawlee/utils": "3.12.1"
  }
}

sendRequest

sendRequest

session

proxyInfo.url

typescript
// CURRENT EXPENSIVE APPROACH:
// Enqueues a whole new request, costing 1 Write + 1 Handled per item
/*
await crawler.addRequests([{
    url: detailUrl,
    label: 'EXTRA_DATA',
    userData: { ... }
}]);
*/

// PROPOSED COST-SAVING APPROACH:
// Fetches inline to avoid touching the Request Queue DB
router.addHandler('ITEM_PAGE', async ({ request, session, proxyInfo, sendRequest }) => {
    
    // 1. Scrape the main item data
    const mainData = extractMainData();
    const detailUrl = extractDetailUrl();

    // 2. Fetch extra details inline without queueing
    if (detailUrl) {  
      // My concern: Does this perfectly mimic the parent request?
      const { body } = await sendRequest({
          url: detailUrl,
          session: session,          // Passing current session
          proxyUrl: proxyInfo?.url,  // Passing current proxy IP
      });
      
      const extraData = extractExtraData(body);
    }

    await pushData(mainData);
});

Reclaiming failed request back to the list or queue. Expected `RequestOptions.url` to be of type `string` but received type `undefined`

import { v4 as uuidv4 } from 'uuid';

const itemRequests: Partial<Source>[] = dataResults.map((result) => {
    const itemKey = result.item.key; // generic key
    return {
        uniqueKey: request.userData.saveOnlyUniqueItems ? itemKey : uuidv4(),
        label: labels.SPECIFIC_ITEM, // generic label
        userData: {
            ...request.userData,
            itemData: result.item, // pass the data generically
        },
    };
});

userData

ts
import { Router } from "crawlee";

export const router = Router.create();

// ... other handlers

// imagine the category handler is called twice
router.addHandler("CATEGORY", async ({ enqueueLinks }) => {
  // ... parsing

  // result of first call
  await enqueueLinks({
    urls: ["https://example.com/my-product"],
    userData: { category: "foo" },
    label: "DETAIL",
  });

  // result of second call
  await enqueueLinks({
    urls: ["https://example.com/my-product"],
    userData: { category: "bar" },
    label: "DETAIL",
  });
});

router.addHandler("DETAIL", async ({ request }) => {
  // todo: how to get both "foo" and "bar" here?
  const category = request.userData.category;

  // ... saving
});

<html><head></head><body></body></html>

await waitForSelector('#myID')