crawlee_script.py

import { PlaywrightCrawler, DefaultFingerprintGenerator } from 'crawlee';
import fs from 'fs';

// Load URLs from CSV
const inputCsvPath = 'collected_thumbnails.csv';
const urls = fs.readFileSync(inputCsvPath, 'utf8')
    .split('\n')
    .filter((line, idx) => idx > 0 && line.trim() !== '');

// Setup fingerprint generator
const fingerprintGenerator = new DefaultFingerprintGenerator({
    browsers: [{ name: 'chrome', minVersion: 96 }],
    devices: ['desktop'],
    locales: ['en-US'],
});

const crawler = new PlaywrightCrawler({
    launchContext: {
        launchOptions: {
            headless: true,
        },
    },
    preNavigationHooks: [
        async ({ launchContext }) => {
            const fingerprint = fingerprintGenerator.getFingerprint({
                browserName: 'chrome',
            });
            launchContext.useFingerprint(fingerprint);
        },
    ],
    requestHandler: async ({ page, request, log }) => {
        log.info(`Scraping: ${request.url}`);

        try {
            const rows = await page.$$eval('tr.tablescraper-selected-row, tr.tablescraper-selected-row2, tr.tablescraper-selected-row3, tr.tablescraper-selected-row4', trs => {
                return trs.map(tr => {
                    const cells = Array.from(tr.querySelectorAll('th, td'));
                    return cells.map(td => td.innerText.trim());
                });
            });

            // Build a CSV-style output
            const headers = ['Label'];
            for (let i = 1; i < rows[0]?.length; i++) {
                headers.push(`tablescraper-selected-row ${i}`);
            }

            const formattedRows = rows.map(row => {
                while (row.length < headers.length) row.push('');
                return row;
            });

            console.log(`\n🧾 [Extracted Table from ${request.url}]`);
            console.log(headers.join(','));
            formattedRows.forEach(row => {
                console.log(row.join(','));
            });

        } catch (err) {
            log.warning(`Failed to extract from: ${request.url}`);
        }
    },
    maxRequestsPerCrawl: urls.length,
});

await crawler.run(urls.map(url => ({ url })));