diff --git a/src/pages/workspace/utils/web_scraping/consts.ts b/src/pages/workspace/utils/web_scraping/consts.ts index eab9f1f..a5bc63f 100644 --- a/src/pages/workspace/utils/web_scraping/consts.ts +++ b/src/pages/workspace/utils/web_scraping/consts.ts @@ -1,63 +1,89 @@ import type { SchedulerJobRetryStrategy } from './web_page_tracker'; +export const WEB_PAGE_TRACKER_MANUAL_SCHEDULE = '@'; +export const WEB_PAGE_TRACKER_CUSTOM_SCHEDULE = '@@'; export const WEB_PAGE_TRACKER_SCHEDULES = [ - { value: '@', text: 'Manually' }, + { value: WEB_PAGE_TRACKER_MANUAL_SCHEDULE, text: 'Manually' }, { value: '@hourly', text: 'Hourly' }, { value: '@daily', text: 'Daily' }, { value: '@weekly', text: 'Weekly' }, { value: '@monthly', text: 'Monthly' }, + { value: WEB_PAGE_TRACKER_CUSTOM_SCHEDULE, text: 'Custom' }, ]; -export const WEB_PAGE_TRACKER_RETRY_STRATEGIES = [ - { value: 'none', text: 'None' }, - { value: 'constant', text: 'Constant backoff' }, -]; +export function getScheduleMinInterval(schedule: string) { + switch (schedule) { + case '@hourly': + return 3600000; + case '@daily': + return 86400000; + case '@weekly': + return 604800000; + case '@monthly': + return 2592000000; + default: + return 0; + } +} -export const WEB_PAGE_TRACKER_RETRY_INTERVALS = new Map([ - [ - '@hourly', - [ - { label: '1m', value: 60000 }, - { label: '3m', value: 180000 }, - { label: '5m', value: 300000 }, - { label: '10m', value: 600000 }, - ], - ], - [ - '@daily', - [ - { label: '30m', value: 1800000 }, - { label: '1h', value: 3600000 }, - { label: '2h', value: 7200000 }, +export function getRetryStrategies(retryIntervals: RetryInterval[]) { + return [ + { value: 'none', text: 'None' }, + ...(retryIntervals.length > 0 ? [{ value: 'constant', text: 'Constant backoff' }] : []), + ]; +} + +export type RetryInterval = { label: string; value: number }; +export function getRetryIntervals(minInterval: number): RetryInterval[] { + if (minInterval > 1209600000 /** 14 days **/) { + return [ { label: '3h', value: 10800000 }, - ], - ], - [ - '@weekly', - [ + { label: '12h', value: 43200000 }, + { label: '1d', value: 86400000 }, + { label: '2d', value: 172800000 }, + { label: '3d', value: 259200000 }, + ]; + } + + if (minInterval > 172800000 /** 48 hours **/) { + return [ { label: '1h', value: 3600000 }, { label: '3h', value: 10800000 }, { label: '6h', value: 21600000 }, + { label: '9h', value: 32400000 }, { label: '12h', value: 43200000 }, - ], - ], - [ - '@monthly', - [ + ]; + } + + if (minInterval > 3600000 /** 1 hour **/) { + return [ + { label: '10m', value: 600000 }, + { label: '30m', value: 1800000 }, + { label: '1h', value: 3600000 }, + { label: '2h', value: 7200000 }, { label: '3h', value: 10800000 }, - { label: '12h', value: 43200000 }, - { label: '1d', value: 86400000 }, - { label: '3d', value: 259200000 }, - ], - ], -]); + ]; + } + + if (minInterval > 600000 /** 10 minutes **/) { + return [ + { label: '1m', value: 60000 }, + { label: '3m', value: 180000 }, + { label: '5m', value: 300000 }, + { label: '7m', value: 420000 }, + { label: '10m', value: 600000 }, + ]; + } + + // For intervals less than 10 minutes, it doesn't make sense to retry more than once. + return []; +} -export function getDefaultRetryStrategy(schedule: string): SchedulerJobRetryStrategy { - return { type: 'constant', maxAttempts: 3, interval: getDefaultRetryInterval(schedule) }; +export function getDefaultRetryStrategy(retryIntervals: RetryInterval[]): SchedulerJobRetryStrategy { + return { type: 'constant', maxAttempts: 3, interval: getDefaultRetryInterval(retryIntervals) }; } // By default, use the middle interval, e.g. 5 minutes for hourly schedule. -export function getDefaultRetryInterval(schedule: string) { - const intervals = WEB_PAGE_TRACKER_RETRY_INTERVALS.get(schedule)!; +export function getDefaultRetryInterval(intervals: RetryInterval[]) { return intervals[Math.floor(intervals.length / 2)].value; } diff --git a/src/pages/workspace/utils/web_scraping/web_page_content_tracker_edit_flyout.tsx b/src/pages/workspace/utils/web_scraping/web_page_content_tracker_edit_flyout.tsx index 7728b3a..32095b7 100644 --- a/src/pages/workspace/utils/web_scraping/web_page_content_tracker_edit_flyout.tsx +++ b/src/pages/workspace/utils/web_scraping/web_page_content_tracker_edit_flyout.tsx @@ -10,13 +10,19 @@ import { EuiFormRow, EuiLink, EuiRange, - EuiSelect, EuiSwitch, } from '@elastic/eui'; import axios from 'axios'; -import { getDefaultRetryStrategy, WEB_PAGE_TRACKER_SCHEDULES } from './consts'; +import type { RetryInterval } from './consts'; +import { + getDefaultRetryStrategy, + getRetryIntervals, + getScheduleMinInterval, + WEB_PAGE_TRACKER_CUSTOM_SCHEDULE, +} from './consts'; import type { SchedulerJobConfig, WebPageContentTracker } from './web_page_tracker'; +import { WebPageTrackerJobSchedule } from './web_page_tracker_job_schedule'; import { WebPageTrackerRetryStrategy } from './web_page_tracker_retry_strategy'; import { useRangeTicks } from '../../../../hooks'; import { type AsyncData, getApiRequestConfig, getApiUrl, getErrorMessage, isClientError } from '../../../../model'; @@ -49,6 +55,9 @@ export function WebPageContentTrackerEditFlyout({ onClose, tracker }: Props) { }, []); const [jobConfig, setJobConfig] = useState(tracker?.jobConfig ?? null); + const [retryIntervals, setRetryIntervals] = useState( + jobConfig?.schedule ? getRetryIntervals(getScheduleMinInterval(jobConfig.schedule)) : [], + ); const [delay, setDelay] = useState(tracker?.settings.delay ?? 5000); const onDelayChange = useCallback((e: ChangeEvent) => { @@ -154,6 +163,26 @@ export function WebPageContentTrackerEditFlyout({ onClose, tracker }: Props) { ) : null; + // Link to the cron expression documentation only if it's allowed by the subscription. + const supportsCustomSchedule = + !uiState.subscription?.features?.webScraping.trackerSchedules || + uiState.subscription.features.webScraping.trackerSchedules.includes(WEB_PAGE_TRACKER_CUSTOM_SCHEDULE); + const scheduleHelpText = supportsCustomSchedule ? ( + + How often web page should be checked for changes. By default, automatic checks are disabled and can be initiated + manually. Custom schedules can be set using a cron expression. Refer to the{' '} + + documentation + {' '} + for supported cron expression formats and examples + + ) : ( + <> + How often web page should be checked for changes. By default, automatic checks are disabled and can be initiated + manually + + ); + const maxTrackerRevisions = uiState.subscription?.features?.webScraping.trackerRevisions ?? 0; const tickInterval = Math.ceil(maxTrackerRevisions / maxTicks); return ( @@ -161,7 +190,9 @@ export function WebPageContentTrackerEditFlyout({ onClose, tracker }: Props) { title={`${tracker ? 'Edit' : 'Add'} tracker`} onClose={() => onClose()} onSave={onSave} - canSave={name.trim().length > 0 && isValidURL(url.trim()) && !headers.invalid} + canSave={ + name.trim().length > 0 && isValidURL(url.trim()) && !headers.invalid && (!jobConfig || !!jobConfig.schedule) + } saveInProgress={updatingStatus?.status === 'pending'} > @@ -225,26 +256,28 @@ export function WebPageContentTrackerEditFlyout({ onClose, tracker }: Props) { 'Properties defining how frequently web page should be checked for changes and how those changes should be reported' } > - - - setJobConfig( - e.target.value === '@' - ? null - : { - ...(jobConfig ?? { - retryStrategy: getDefaultRetryStrategy(e.target.value), - notifications: true, - }), - schedule: e.target.value, - }, - ) - } + + { + // If schedule is invalid, update only schedule. + if (schedule === '' && jobConfig) { + setJobConfig({ ...jobConfig, schedule }); + return; + } + + if (schedule === null) { + setJobConfig(null); + } else if (schedule !== jobConfig?.schedule) { + setJobConfig({ + ...(jobConfig ?? { notifications: true }), + retryStrategy: retryIntervals.length > 0 ? getDefaultRetryStrategy(retryIntervals) : undefined, + schedule, + }); + } + + setRetryIntervals(retryIntervals); + }} /> {notifications} @@ -255,7 +288,8 @@ export function WebPageContentTrackerEditFlyout({ onClose, tracker }: Props) { description={'Properties defining how failed automatic checks should be retried'} > { if (jobConfig) { setJobConfig({ ...jobConfig, retryStrategy: newStrategy ?? undefined }); diff --git a/src/pages/workspace/utils/web_scraping/web_page_resources_tracker_edit_flyout.tsx b/src/pages/workspace/utils/web_scraping/web_page_resources_tracker_edit_flyout.tsx index a08e89b..ac332ae 100644 --- a/src/pages/workspace/utils/web_scraping/web_page_resources_tracker_edit_flyout.tsx +++ b/src/pages/workspace/utils/web_scraping/web_page_resources_tracker_edit_flyout.tsx @@ -10,13 +10,19 @@ import { EuiFormRow, EuiLink, EuiRange, - EuiSelect, EuiSwitch, } from '@elastic/eui'; import axios from 'axios'; -import { getDefaultRetryStrategy, WEB_PAGE_TRACKER_SCHEDULES } from './consts'; +import { + getDefaultRetryStrategy, + getRetryIntervals, + getScheduleMinInterval, + type RetryInterval, + WEB_PAGE_TRACKER_CUSTOM_SCHEDULE, +} from './consts'; import type { SchedulerJobConfig, WebPageResourcesTracker } from './web_page_tracker'; +import { WebPageTrackerJobSchedule } from './web_page_tracker_job_schedule'; import { WebPageTrackerRetryStrategy } from './web_page_tracker_retry_strategy'; import { useRangeTicks } from '../../../../hooks'; import { type AsyncData, getApiRequestConfig, getApiUrl, getErrorMessage, isClientError } from '../../../../model'; @@ -49,6 +55,9 @@ export function WebPageResourcesTrackerEditFlyout({ onClose, tracker }: Props) { }, []); const [jobConfig, setJobConfig] = useState(tracker?.jobConfig ?? null); + const [retryIntervals, setRetryIntervals] = useState( + jobConfig?.schedule ? getRetryIntervals(getScheduleMinInterval(jobConfig.schedule)) : [], + ); const [delay, setDelay] = useState(tracker?.settings.delay ?? 5000); const onDelayChange = useCallback((e: ChangeEvent) => { @@ -158,6 +167,26 @@ export function WebPageResourcesTrackerEditFlyout({ onClose, tracker }: Props) { ) : null; + // Link to the cron expression documentation only if it's allowed by the subscription. + const supportsCustomSchedule = + !uiState.subscription?.features?.webScraping.trackerSchedules || + uiState.subscription.features.webScraping.trackerSchedules.includes(WEB_PAGE_TRACKER_CUSTOM_SCHEDULE); + const scheduleHelpText = supportsCustomSchedule ? ( + + How often web page should be checked for changes. By default, automatic checks are disabled and can be initiated + manually. Custom schedules can be set using a cron expression. Refer to the{' '} + + documentation + {' '} + for supported cron expression formats and examples + + ) : ( + <> + How often web page should be checked for changes. By default, automatic checks are disabled and can be initiated + manually + + ); + const maxTrackerRevisions = uiState.subscription?.features?.webScraping.trackerRevisions ?? 0; const tickInterval = Math.ceil(maxTrackerRevisions / maxTicks); return ( @@ -165,7 +194,9 @@ export function WebPageResourcesTrackerEditFlyout({ onClose, tracker }: Props) { title={`${tracker ? 'Edit' : 'Add'} tracker`} onClose={() => onClose()} onSave={onSave} - canSave={name.trim().length > 0 && isValidURL(url.trim()) && !headers.invalid} + canSave={ + name.trim().length > 0 && isValidURL(url.trim()) && !headers.invalid && (!jobConfig || !!jobConfig.schedule) + } saveInProgress={updatingStatus?.status === 'pending'} > @@ -229,26 +260,28 @@ export function WebPageResourcesTrackerEditFlyout({ onClose, tracker }: Props) { 'Properties defining how frequently web page should be checked for changes and how those changes should be reported' } > - - - setJobConfig( - e.target.value === '@' - ? null - : { - ...(jobConfig ?? { - retryStrategy: getDefaultRetryStrategy(e.target.value), - notifications: true, - }), - schedule: e.target.value, - }, - ) - } + + { + // If schedule is invalid, update only schedule. + if (schedule === '' && jobConfig) { + setJobConfig({ ...jobConfig, schedule }); + return; + } + + if (schedule === null) { + setJobConfig(null); + } else if (schedule !== jobConfig?.schedule) { + setJobConfig({ + ...(jobConfig ?? { notifications: true }), + retryStrategy: retryIntervals.length > 0 ? getDefaultRetryStrategy(retryIntervals) : undefined, + schedule, + }); + } + + setRetryIntervals(retryIntervals); + }} /> {notifications} @@ -259,7 +292,8 @@ export function WebPageResourcesTrackerEditFlyout({ onClose, tracker }: Props) { description={'Properties defining how failed automatic checks should be retried'} > { if (jobConfig) { setJobConfig({ ...jobConfig, retryStrategy: newStrategy ?? undefined }); diff --git a/src/pages/workspace/utils/web_scraping/web_page_tracker_job_schedule.tsx b/src/pages/workspace/utils/web_scraping/web_page_tracker_job_schedule.tsx new file mode 100644 index 0000000..0c6689a --- /dev/null +++ b/src/pages/workspace/utils/web_scraping/web_page_tracker_job_schedule.tsx @@ -0,0 +1,127 @@ +import { useEffect, useState } from 'react'; + +import { EuiButtonIcon, EuiFieldText, EuiSelect, EuiSpacer, EuiToolTip } from '@elastic/eui'; + +import type { RetryInterval } from './consts'; +import { + getRetryIntervals, + getScheduleMinInterval, + WEB_PAGE_TRACKER_CUSTOM_SCHEDULE, + WEB_PAGE_TRACKER_MANUAL_SCHEDULE, + WEB_PAGE_TRACKER_SCHEDULES, +} from './consts'; +import type { AsyncData } from '../../../../model'; +import { getErrorMessage } from '../../../../model'; +import { useWorkspaceContext } from '../../hooks'; + +export interface WebPageTrackerJobScheduleProps { + schedule?: string; + onChange: (schedule: string | null, retryIntervals: RetryInterval[]) => void; +} + +interface CustomScheduleCheck { + minInterval: number; + nextOccurrences: number[]; +} + +export function WebPageTrackerJobSchedule({ schedule, onChange }: WebPageTrackerJobScheduleProps) { + const { uiState } = useWorkspaceContext(); + + // Filter schedules based on subscription. + const subscriptionSchedules = uiState.subscription?.features?.webScraping.trackerSchedules; + const schedules = subscriptionSchedules + ? WEB_PAGE_TRACKER_SCHEDULES.filter((knownSchedule) => subscriptionSchedules.includes(knownSchedule.value)) + : WEB_PAGE_TRACKER_SCHEDULES; + const [scheduleType, setScheduleType] = useState(() => { + if (!schedule) { + return WEB_PAGE_TRACKER_MANUAL_SCHEDULE; + } + + return schedules.some((s) => s.value === schedule) ? schedule : WEB_PAGE_TRACKER_CUSTOM_SCHEDULE; + }); + + const [customSchedule, setCustomSchedule] = useState( + scheduleType === WEB_PAGE_TRACKER_CUSTOM_SCHEDULE ? schedule ?? '' : '', + ); + const [customScheduleCheck, setCustomScheduleCheck] = useState | null>(null); + useEffect(() => { + if (!customSchedule) { + setCustomScheduleCheck(null); + return; + } + + setCustomScheduleCheck({ status: 'pending' }); + + fetch(`/api/scheduler/parse_schedule`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ schedule: customSchedule }), + }) + .then((response) => response.json()) + .then((data: CustomScheduleCheck) => { + setCustomScheduleCheck({ status: 'succeeded', data }); + onChange(customSchedule, getRetryIntervals(data.minInterval)); + }) + .catch((e) => { + setCustomScheduleCheck({ status: 'failed', error: getErrorMessage(e) }); + onChange('', []); + }); + }, [customSchedule]); + + const typePicker = ( + { + setScheduleType(e.target.value); + setCustomSchedule(''); + + if ( + e.target.value === WEB_PAGE_TRACKER_MANUAL_SCHEDULE || + e.target.value === WEB_PAGE_TRACKER_CUSTOM_SCHEDULE + ) { + onChange(null, []); + } else { + onChange(e.target.value, getRetryIntervals(getScheduleMinInterval(e.target.value))); + } + }} + /> + ); + + if (scheduleType !== WEB_PAGE_TRACKER_CUSTOM_SCHEDULE) { + return typePicker; + } + + return ( +
+ {typePicker} + + + {customScheduleCheck.data.nextOccurrences.map((occurrence) => ( +

{new Date(occurrence * 1000).toUTCString()}

+ ))} + + } + > + + + ) : ( + + ) + } + onChange={(e) => setCustomSchedule(e.target.value)} + /> +
+ ); +} diff --git a/src/pages/workspace/utils/web_scraping/web_page_tracker_retry_strategy.tsx b/src/pages/workspace/utils/web_scraping/web_page_tracker_retry_strategy.tsx index 33e324e..aea9430 100644 --- a/src/pages/workspace/utils/web_scraping/web_page_tracker_retry_strategy.tsx +++ b/src/pages/workspace/utils/web_scraping/web_page_tracker_retry_strategy.tsx @@ -1,60 +1,32 @@ -import { useEffect, useState } from 'react'; - import { EuiFormRow, EuiRange, EuiSelect } from '@elastic/eui'; -import { - getDefaultRetryInterval, - getDefaultRetryStrategy, - WEB_PAGE_TRACKER_RETRY_INTERVALS, - WEB_PAGE_TRACKER_RETRY_STRATEGIES, -} from './consts'; -import type { SchedulerJobConfig, SchedulerJobRetryStrategy } from './web_page_tracker'; +import type { RetryInterval } from './consts'; +import { getDefaultRetryStrategy, getRetryStrategies } from './consts'; +import type { SchedulerJobRetryStrategy } from './web_page_tracker'; export interface WebPageTrackerRetryStrategyProps { - jobConfig: SchedulerJobConfig; + intervals: RetryInterval[]; + strategy?: SchedulerJobRetryStrategy; onChange: (strategy: SchedulerJobRetryStrategy | null) => void; } -export function WebPageTrackerRetryStrategy({ jobConfig, onChange }: WebPageTrackerRetryStrategyProps) { - const [currentJobConfig, setCurrentJobConfig] = useState(jobConfig); - - useEffect(() => { - const changedSchedule = currentJobConfig.schedule !== jobConfig.schedule; - if (currentJobConfig.retryStrategy === jobConfig.retryStrategy && !changedSchedule) { - return; - } - - const newRetryStrategy = - currentJobConfig.retryStrategy && changedSchedule - ? { ...currentJobConfig.retryStrategy, interval: getDefaultRetryInterval(jobConfig.schedule) } - : currentJobConfig.retryStrategy; - setCurrentJobConfig({ ...jobConfig, retryStrategy: newRetryStrategy }); - onChange(newRetryStrategy ?? null); - }, [jobConfig, currentJobConfig, onChange]); - - const retryStrategy = currentJobConfig.retryStrategy; +export function WebPageTrackerRetryStrategy({ intervals, strategy, onChange }: WebPageTrackerRetryStrategyProps) { let maxAttempts = null; let interval = null; - if (retryStrategy) { + if (strategy && intervals.length > 0) { maxAttempts = ( - setCurrentJobConfig({ - ...currentJobConfig, - retryStrategy: { ...retryStrategy, maxAttempts: +e.currentTarget.value }, - }) - } + value={strategy.maxAttempts} + onChange={(e) => onChange({ ...strategy, maxAttempts: +e.currentTarget.value })} showTicks /> ); - const intervals = WEB_PAGE_TRACKER_RETRY_INTERVALS.get(currentJobConfig.schedule)!; const minInterval = intervals[0].value; const maxInterval = intervals[intervals.length - 1].value; interval = ( @@ -63,32 +35,29 @@ export function WebPageTrackerRetryStrategy({ jobConfig, onChange }: WebPageTrac min={minInterval} max={maxInterval} step={minInterval} - value={retryStrategy.interval} - disabled={retryStrategy.maxAttempts === 0} + value={strategy.interval} + disabled={strategy.maxAttempts === 0} ticks={intervals} - onChange={(e) => - setCurrentJobConfig({ - ...currentJobConfig, - retryStrategy: { ...retryStrategy, interval: +e.currentTarget.value }, - }) - } + onChange={(e) => onChange({ ...strategy, interval: +e.currentTarget.value })} showTicks />
); } + const strategies = getRetryStrategies(intervals); + const canChangeStrategy = strategies.length > 1; return ( <> - setCurrentJobConfig({ - ...currentJobConfig, - retryStrategy: e.target.value === 'none' ? undefined : getDefaultRetryStrategy(currentJobConfig.schedule), - }) + options={strategies} + disabled={!canChangeStrategy} + value={strategy?.type ?? strategies[0].value} + onChange={ + canChangeStrategy + ? (e) => onChange(e.target.value === 'none' ? null : getDefaultRetryStrategy(intervals)) + : undefined } />