Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/upgrading/upgrading_v4.md
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,10 @@ await crawler.run([{ url: 'https://example.com', sessionId: 'basic' }]);

More complex routing (more tiers, weighted draws, sticky assignment, cooldowns) can be expressed with additional named sessions and custom `errorHandler` logic.

## `maxSessionRotations` and `request.sessionRotationCount` are removed

Session errors no longer have their own retry budget. The `maxSessionRotations` crawler option, the `Request.sessionRotationCount` property, and the special-case retry logic for `SessionError` are all gone. A `SessionError` now retires the session and counts toward `maxRequestRetries` like any other failure, so configure a single retry limit via `maxRequestRetries` (default `3`). `SessionError` also no longer extends `RetryRequestError` - if you were catching `RetryRequestError` to detect a session-triggered retry, branch on `SessionError` directly instead.

## Remove `experimentalContainers` option

This experimental option relied on an outdated manifest version for browser extensions, it is not possible to achieve this with the currently supported versions.
Expand Down
55 changes: 16 additions & 39 deletions packages/basic-crawler/src/internals/basic-crawler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -238,11 +238,8 @@ export interface BasicCrawlerOptions<

/**
* Specifies the maximum number of retries allowed for a request if its processing fails.
* This includes retries due to navigation errors or errors thrown from user-supplied functions
* (`requestHandler`, `preNavigationHooks`, `postNavigationHooks`).
*
* This limit does not apply to retries triggered by session rotation
* (see {@apilink BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`}).
* This includes retries due to navigation errors, session/proxy errors, or errors thrown from user-supplied
* functions (`requestHandler`, `preNavigationHooks`, `postNavigationHooks`).
* @default 3
*/
maxRequestRetries?: number;
Expand All @@ -253,15 +250,6 @@ export interface BasicCrawlerOptions<
*/
sameDomainDelaySecs?: number;

/**
* Maximum number of session rotations per request.
* The crawler will automatically rotate the session in case of a proxy error or if it gets blocked by the website.
*
* The session rotations are not counted towards the {@apilink BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} limit.
* @default 10
*/
maxSessionRotations?: number;

/**
* Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
* This value should always be set in order to prevent infinite loops in misconfigured crawlers.
Expand Down Expand Up @@ -645,7 +633,6 @@ export class BasicCrawler<
protected maxCrawlDepth?: number;
protected sameDomainDelayMillis: number;
protected domainAccessedTime: Map<string, number>;
protected maxSessionRotations: number;
protected maxRequestsPerCrawl?: number;
protected handledRequestsCount = 0;
protected statusMessageLoggingInterval: number;
Expand Down Expand Up @@ -685,7 +672,6 @@ export class BasicCrawler<
failedRequestHandler: ow.optional.function,
maxRequestRetries: ow.optional.number,
sameDomainDelaySecs: ow.optional.number,
maxSessionRotations: ow.optional.number,
maxRequestsPerCrawl: ow.optional.number,
maxCrawlDepth: ow.optional.number,
autoscaledPoolOptions: ow.optional.object,
Expand Down Expand Up @@ -738,7 +724,6 @@ export class BasicCrawler<
requestManager,
maxRequestRetries = 3,
sameDomainDelaySecs = 0,
maxSessionRotations = 10,
maxRequestsPerCrawl,
maxCrawlDepth,
autoscaledPoolOptions = {},
Expand Down Expand Up @@ -865,7 +850,6 @@ export class BasicCrawler<
this.maxRequestRetries = maxRequestRetries;
this.maxCrawlDepth = maxCrawlDepth;
this.sameDomainDelayMillis = sameDomainDelaySecs * 1000;
this.maxSessionRotations = maxSessionRotations;
this.stats = new Statistics({
logMessage: `${this.constructor.name} request statistics:`,
log: this.log,
Expand Down Expand Up @@ -969,7 +953,11 @@ export class BasicCrawler<
request,
this.requestManager!,
);
crawlingContext.session?.markBad();
// SessionError already retired the session in `_requestFunctionErrorHandler`;
// skip `markBad` to avoid double-counting usage/error score.
if (!(unwrappedError instanceof SessionError)) {
crawlingContext.session?.markBad();
}
return;
}
throw this.unwrapError(error);
Expand Down Expand Up @@ -1939,8 +1927,11 @@ export class BasicCrawler<
request.state = RequestState.ERROR;
throw unwrappedSecondaryError;
}
// decrease the session score if the request fails (but the error handler did not throw)
crawlingContext.session.markBad();
// decrease the session score if the request fails (but the error handler did not throw);
// skip when the error is a SessionError, which already retired the session
if (!(err instanceof SessionError)) {
crawlingContext.session.markBad();
}
} finally {
// Safety net - release the lock if nobody managed to do it before
if (isRequestLocked && requestSource instanceof RequestProvider) {
Expand Down Expand Up @@ -2065,14 +2056,6 @@ export class BasicCrawler<
return !this.requestManager || (await this.requestManager.isFinished());
}

private async _rotateSession(crawlingContext: CrawlingContext) {
const { request } = crawlingContext;

request.sessionRotationCount ??= 0;
request.sessionRotationCount++;
crawlingContext.session.retire();
}

/**
* Unwraps errors thrown by the context pipeline to get the actual user error.
* RequestHandlerError and ContextPipelineInitializationError wrap the actual error.
Expand Down Expand Up @@ -2115,13 +2098,11 @@ export class BasicCrawler<
);

if (error instanceof SessionError) {
await this._rotateSession(crawlingContext);
crawlingContext.session?.retire();
}

if (!request.noRetry) {
if (!(error instanceof SessionError)) {
request.retryCount++;
}
request.retryCount++;

Comment thread
barjin marked this conversation as resolved.
const { url, retryCount, id } = request;

Expand Down Expand Up @@ -2212,12 +2193,8 @@ export class BasicCrawler<
}

protected _canRequestBeRetried(request: Request, error: Error) {
// Request should never be retried, or the error encountered makes it not able to be retried, or the session rotation limit has been reached
if (
request.noRetry ||
error instanceof NonRetryableError ||
(error instanceof SessionError && this.maxSessionRotations <= (request.sessionRotationCount ?? 0))
) {
// Request should never be retried, or the error encountered makes it not able to be retried.
if (request.noRetry || error instanceof NonRetryableError) {
return false;
}

Expand Down
8 changes: 4 additions & 4 deletions packages/core/src/errors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,13 @@ export class RetryRequestError extends Error {
}

/**
* Errors of `SessionError` type will trigger a session rotation.
* Errors of `SessionError` type retire the session associated with the request and trigger a regular retry.
*
* This error doesn't respect the `maxRequestRetries` option and has a separate limit of `maxSessionRotations`.
* The retry counts towards the `maxRequestRetries` limit, just like any other error.
*/
export class SessionError extends RetryRequestError {
export class SessionError extends Error {
constructor(message?: string) {
super(`Detected a session error, rotating session... ${message ? `\n${message}` : ''}`);
super(`Detected a session error, retiring session... ${message ? `\n${message}` : ''}`);
}
}

Expand Down
18 changes: 0 additions & 18 deletions packages/core/src/request.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ const requestOptionalPredicates = {
payload: ow.optional.any(ow.string, ow.uint8Array),
noRetry: ow.optional.boolean,
retryCount: ow.optional.number,
sessionRotationCount: ow.optional.number,
sessionId: ow.optional.string,
maxRetries: ow.optional.number,
errorMessages: ow.optional.array.ofType(ow.string),
Expand Down Expand Up @@ -170,7 +169,6 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {
payload,
noRetry = false,
retryCount = 0,
sessionRotationCount = 0,
sessionId,
maxRetries,
errorMessages = [],
Expand All @@ -186,7 +184,6 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {
} = options as RequestOptions & {
loadedUrl?: string;
retryCount?: number;
sessionRotationCount?: number;
sessionId?: string;
errorMessages?: string[];
handledAt?: string | Date;
Expand All @@ -208,7 +205,6 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {
this.payload = payload;
this.noRetry = noRetry;
this.retryCount = retryCount;
this.sessionRotationCount = sessionRotationCount;
this.errorMessages = [...errorMessages];
this.headers = { ...headers };
this.handledAt = (handledAt as unknown) instanceof Date ? (handledAt as Date).toISOString() : handledAt!;
Expand Down Expand Up @@ -322,20 +318,6 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {
this.userData.__crawlee.crawlDepth = value;
}

/** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
get sessionRotationCount(): number {
return this.userData.__crawlee?.sessionRotationCount ?? 0;
}

/** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
set sessionRotationCount(value: number) {
if (!this.userData.__crawlee) {
(this.userData as Dictionary).__crawlee = { sessionRotationCount: value };
} else {
this.userData.__crawlee.sessionRotationCount = value;
}
}

/** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
get sessionId(): string | undefined {
return this.userData.__crawlee?.sessionId;
Expand Down
29 changes: 14 additions & 15 deletions test/core/crawlers/browser_crawler.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -617,7 +617,7 @@ describe('BrowserCrawler', () => {

try {
const urls = [new URL('/special/cloudflareBlocking', serverAddress).href];
const maxSessionRotations = 1;
const maxRequestRetries = 1;

let processed = false;
const errorMessages: string[] = [];
Expand All @@ -627,7 +627,7 @@ describe('BrowserCrawler', () => {
browserPlugins: [puppeteerPlugin],
},
retryOnBlocked: true,
maxSessionRotations,
maxRequestRetries,
requestHandler: async ({ page, response }) => {
processed = true;
},
Expand All @@ -638,8 +638,8 @@ describe('BrowserCrawler', () => {

await crawler.run(urls);

expect(errorMessages).toHaveLength(urls.length * (maxSessionRotations + 1));
expect(errorMessages.every((x) => x.includes('Detected a session error, rotating session...'))).toBe(true);
expect(errorMessages).toHaveLength(urls.length * (maxRequestRetries + 1));
expect(errorMessages.every((x) => x.includes('Detected a session error, retiring session...'))).toBe(true);
expect(processed).toBe(false);
} finally {
await localStorageEmulator.destroy();
Expand All @@ -660,7 +660,7 @@ describe('BrowserCrawler', () => {
};
});
const requestList = await RequestList.open(null, sources);
const maxSessionRotations = 1;
const maxRequestRetries = 1;
const errorMessages: string[] = [];

let processed = false;
Expand All @@ -670,7 +670,7 @@ describe('BrowserCrawler', () => {
},
requestList,
retryOnBlocked: true,
maxSessionRotations,
maxRequestRetries,
requestHandler: async () => {
processed = true;
},
Expand All @@ -686,8 +686,8 @@ describe('BrowserCrawler', () => {

await crawler.run();

expect(errorMessages.length).toBe(sources.length * (maxSessionRotations + 1));
expect(errorMessages.every((x) => x.includes('Detected a session error, rotating session...'))).toBe(true);
expect(errorMessages.length).toBe(sources.length * (maxRequestRetries + 1));
expect(errorMessages.every((x) => x.includes('Detected a session error, retiring session...'))).toBe(true);
expect(processed).toBe(false);
} finally {
await localStorageEmulator.destroy();
Expand Down Expand Up @@ -930,9 +930,8 @@ describe('BrowserCrawler', () => {
});

const goodProxyUrl = 'http://good.proxy';
const proxyConfiguration = new ProxyConfiguration({
proxyUrls: ['http://localhost', 'http://localhost:1234', goodProxyUrl],
});
const proxyUrls = ['http://localhost', 'http://localhost:1234', goodProxyUrl];
const proxyConfiguration = new ProxyConfiguration({ proxyUrls });
const requestHandler = vitest.fn();

const browserCrawler = new (class extends BrowserCrawlerTest {
Expand All @@ -952,7 +951,7 @@ describe('BrowserCrawler', () => {
browserPlugins: [puppeteerPlugin],
},
requestList,
maxRequestRetries: 0,
maxRequestRetries: proxyUrls.length - 1,
maxConcurrency: 1,

Comment on lines 951 to 956
proxyConfiguration,
Expand All @@ -966,7 +965,7 @@ describe('BrowserCrawler', () => {
}
});

test.concurrent('proxy rotation on error respects maxSessionRotations, calls failedRequestHandler', async () => {
test.concurrent('proxy rotation on error respects maxRequestRetries, calls failedRequestHandler', async () => {
const localStorageEmulator = new MemoryStorageEmulator();
await localStorageEmulator.init();
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
Expand Down Expand Up @@ -1009,7 +1008,7 @@ describe('BrowserCrawler', () => {
browserPlugins: [puppeteerPlugin],
},
requestList,
maxSessionRotations: 5,
maxRequestRetries: 5,
maxConcurrency: 1,
proxyConfiguration,
requestHandler: async () => {},
Expand Down Expand Up @@ -1061,7 +1060,7 @@ describe('BrowserCrawler', () => {
browserPlugins: [puppeteerPlugin],
},
requestList,
maxSessionRotations: 1,
maxRequestRetries: 1,
maxConcurrency: 1,
proxyConfiguration,
requestHandler: async () => {},
Expand Down
19 changes: 9 additions & 10 deletions test/core/crawlers/cheerio_crawler.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -794,7 +794,7 @@ describe('CheerioCrawler', () => {
throw new Error('Proxy responded with 400 - Bad request');
}
})({
maxSessionRotations: 2,
maxRequestRetries: 2,
maxConcurrency: 1,

proxyConfiguration,
Expand All @@ -805,7 +805,7 @@ describe('CheerioCrawler', () => {
expect(check).toBeCalledWith(expect.objectContaining({ proxyUrl: goodProxyUrl }));
});

test('proxy rotation on error respects maxSessionRotations, calls failedRequestHandler', async () => {
test('proxy rotation on error respects maxRequestRetries, calls failedRequestHandler', async () => {
const proxyConfiguration = new ProxyConfiguration({
proxyUrls: ['http://localhost', 'http://localhost:1234'],
});
Expand All @@ -818,7 +818,7 @@ describe('CheerioCrawler', () => {
const impit = new ImpitHttpClient();
const crawler = new CheerioCrawler({
proxyConfiguration,
maxSessionRotations: 5,
maxRequestRetries: 5,
requestHandler: async () => {},
failedRequestHandler,
httpClient: {
Expand Down Expand Up @@ -848,7 +848,7 @@ describe('CheerioCrawler', () => {

const crawler = new CheerioCrawler({
proxyConfiguration,
maxSessionRotations: 1,
maxRequestRetries: 1,
requestHandler: async () => {},
httpClient: {
sendRequest: async (request, opts) => {
Expand Down Expand Up @@ -948,7 +948,7 @@ describe('CheerioCrawler', () => {
for (const code of [401, 403, 429]) {
const failed: Request[] = [];
const sessions: Session[] = [];
const maxSessionRotations = 5;
const maxRequestRetries = 5;
const crawler = new CheerioCrawler({
requestList: await getRequestListForMock({
statusCode: code,
Expand All @@ -957,8 +957,7 @@ describe('CheerioCrawler', () => {
}),

saveResponseCookies: false,
maxRequestRetries: 10,
maxSessionRotations,
maxRequestRetries,
requestHandler: ({ session }) => {
sessions.push(session!);
},
Expand All @@ -970,9 +969,9 @@ describe('CheerioCrawler', () => {

// @ts-expect-error private symbol
const poolSessions: Session[] = crawler.sessionPool.sessions;
// each request is retried with session rotation (maxSessionRotations times), so we get
// (maxSessionRotations + 1) sessions per request (rotated ones + the final one)
expect(poolSessions.length).toBe(4 * (maxSessionRotations + 1));
// each request retires its session on every retry, so we get
// (maxRequestRetries + 1) sessions per request (retired ones + the final one)
expect(poolSessions.length).toBe(4 * (maxRequestRetries + 1));

poolSessions.forEach((session) => {
expect(session.errorScore).toBeGreaterThanOrEqual(session.maxErrorScore);
Expand Down
Loading
Loading