louisdck
03/27/2025, 9:43 AMTypeScript
import { PlaywrightCrawler, Dataset } from 'crawlee';
const crawler = new PlaywrightCrawler({
preNavigationHooks: [
async ({ page, request, log }) => {
if (playwrightCookies.length > 0) {
log.info(`Setting ${playwrightCookies.length} cookies for ${request.url}...`);
await page.context().addCookies(playwrightCookies);
}
},
],
launchContext: {
launchOptions: {
headless: true,
},
},
async requestHandler({ page, request, log }) {
log.info(`Processing ${request.url}...`);
const startTime = Date.now();
if (waitForSelector) {
try {
await page.waitForSelector(waitForSelector, { timeout: 60000 });
} catch (error) {
log.info(`Selector "${waitForSelector}" not detected after 1 minute. Continuing...`);
}
}
https://cdn.discordapp.com/attachments/1354752554881519727/1354752555241963571/message.txt?ex=67e66f23&is=67e51da3&hm=47295fd81b6a49cb047ebddb3371f5bc38e991adcfd70418f534ebeebbd2cdad&Hall
03/27/2025, 9:43 AMlouisdck
03/27/2025, 9:50 AM