From 667040fbecdb084581dec8b6cfe99bd992cd48db Mon Sep 17 00:00:00 2001 From: Charlotte Alexandra Wilson Date: Fri, 24 Jan 2025 21:59:44 +0000 Subject: [PATCH] Entity Store Config - Lookback period (#206421) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary This PR enables configurability of the lookbackPeriod for Security’s Entity Store through the enable API. ### Testing 1. Open the latest instance of Kibana. 2. Ensure both the Entity Store and Risk Score are enabled. 3. Navigate to Dev Tools. 4. Use the enable endpoint with lookbackPeriod, where the lookbackPeriod must be a number followed by one of the supported units (m, s, or h). ``` POST kbn:api/entity_store/enable { "lookbackPeriod": "72h" // example value } ``` 5. After enabling, use the status endpoint to confirm the configuration: ``` GET kbn:api/entity_store/status {} ``` 6. Post some documents, to check the entities available within the lookbackPeriod - confirm you can only see those within this period, and not outside of this period e.g. "5h" period, post with a timestamp of today (within 5 hours) and from yesterday and view results. ``` POST lookback-period-test/_doc { "entity_id": "LookbackPeriod Test Today", "entity_name": "LookbackPeriod Test Today", "timestamp": "2025-01-17T14:31:00Z" } ``` ``` POST lookback-period-test/_doc { "entity_id": "LookbackPeriod Test Yesterday", "entity_name": "LookbackPeriod Test Yesterday", "timestamp": "2025-01-16T14:31:00Z" } ``` 7. Can also go to entity store, check one of the transforms and check the JSON gte ranges from here too, they should show the new lookbackPeriod. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- oas_docs/output/kibana.serverless.yaml | 9 +++ oas_docs/output/kibana.yaml | 9 +++ .../kbn-entities-schema/src/schema/common.ts | 4 +- .../entity_store/common.gen.ts | 5 ++ .../entity_store/common.schema.yaml | 4 ++ .../entity_store/enable.gen.ts | 8 +++ .../entity_store/enable.schema.yaml | 5 ++ ...alytics_api_2023_10_31.bundled.schema.yaml | 9 +++ ...alytics_api_2023_10_31.bundled.schema.yaml | 9 +++ .../entity_store/entity_store_data_client.ts | 64 ++++++++++++++----- .../installation/engine_description.ts | 6 +- .../saved_object/engine_descriptor.ts | 5 +- .../entity_store.ts | 4 ++ .../entity_store_nondefault_spaces.ts | 4 ++ 14 files changed, 127 insertions(+), 18 deletions(-) diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml index 1abb5e6471079..ba0690d06b502 100644 --- a/oas_docs/output/kibana.serverless.yaml +++ b/oas_docs/output/kibana.serverless.yaml @@ -9735,6 +9735,11 @@ paths: type: string indexPattern: $ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern' + lookbackPeriod: + default: 24h + description: The lookback period for the entity store + pattern: '[smdh]$' + type: string description: Schema for the entity store initialization required: true responses: @@ -50354,6 +50359,10 @@ components: type: string indexPattern: $ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern' + lookbackPeriod: + default: 24h + pattern: '[smdh]$' + type: string status: $ref: '#/components/schemas/Security_Entity_Analytics_API_EngineStatus' type: diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml index 911e48be46adf..8039bcd8d4397 100644 --- a/oas_docs/output/kibana.yaml +++ b/oas_docs/output/kibana.yaml @@ -11883,6 +11883,11 @@ paths: type: string indexPattern: $ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern' + lookbackPeriod: + default: 24h + description: The lookback period for the entity store + pattern: '[smdh]$' + type: string description: Schema for the entity store initialization required: true responses: @@ -57206,6 +57211,10 @@ components: type: string indexPattern: $ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern' + lookbackPeriod: + default: 24h + pattern: '[smdh]$' + type: string status: $ref: '#/components/schemas/Security_Entity_Analytics_API_EngineStatus' type: diff --git a/x-pack/platform/packages/shared/kbn-entities-schema/src/schema/common.ts b/x-pack/platform/packages/shared/kbn-entities-schema/src/schema/common.ts index 3383cfaf7debb..b7342f3e6698f 100644 --- a/x-pack/platform/packages/shared/kbn-entities-schema/src/schema/common.ts +++ b/x-pack/platform/packages/shared/kbn-entities-schema/src/schema/common.ts @@ -44,7 +44,9 @@ export const docCountMetricSchema = z.object({ filter: filterSchema, }); -export const durationSchema = z.string().regex(/^\d+[m|d|s|h]$/); +export const durationSchema = z.string().regex(/^\d+[m|s|d|h]$/, { + message: 'Must be a number followed by one of the units: m, s, d or h', +}); export const durationSchemaWithMinimum = (minimumMinutes: number) => durationSchema.refine( diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts index 8fa1a680a2396..3c809aaac73ee 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts @@ -36,6 +36,11 @@ export const EngineDescriptor = z.object({ status: EngineStatus, filter: z.string().optional(), fieldHistoryLength: z.number().int(), + lookbackPeriod: z + .string() + .regex(/[smdh]$/) + .optional() + .default('24h'), error: z.object({}).optional(), }); diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml index f4c174e1213bb..1bc6e6b941fe1 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml @@ -32,6 +32,10 @@ components: type: string fieldHistoryLength: type: integer + lookbackPeriod: + type: string + default: 24h + pattern: '[smdh]$' error: type: object diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.gen.ts index 2ef7e2e76cd4d..35d7d9d136283 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.gen.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.gen.ts @@ -24,6 +24,14 @@ export const InitEntityStoreRequestBody = z.object({ * The number of historical values to keep for each field. */ fieldHistoryLength: z.number().int().optional().default(10), + /** + * The lookback period for the entity store + */ + lookbackPeriod: z + .string() + .regex(/[smdh]$/) + .optional() + .default('24h'), indexPattern: IndexPattern.optional(), filter: z.string().optional(), entityTypes: z.array(EntityType).optional(), diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.schema.yaml index 7074f938fb839..ceb160120cf69 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.schema.yaml @@ -23,6 +23,11 @@ paths: type: integer description: The number of historical values to keep for each field. default: 10 + lookbackPeriod: + type: string + description: The lookback period for the entity store + default: 24h + pattern: '[smdh]$' indexPattern: $ref: './common.schema.yaml#/components/schemas/IndexPattern' filter: diff --git a/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml b/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml index 543c1541dc792..9187185aef5db 100644 --- a/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml @@ -321,6 +321,11 @@ paths: type: string indexPattern: $ref: '#/components/schemas/IndexPattern' + lookbackPeriod: + default: 24h + description: The lookback period for the entity store + pattern: '[smdh]$' + type: string description: Schema for the entity store initialization required: true responses: @@ -1010,6 +1015,10 @@ components: type: string indexPattern: $ref: '#/components/schemas/IndexPattern' + lookbackPeriod: + default: 24h + pattern: '[smdh]$' + type: string status: $ref: '#/components/schemas/EngineStatus' type: diff --git a/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml b/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml index b103393a4f6ed..6b4b9276e256a 100644 --- a/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml @@ -321,6 +321,11 @@ paths: type: string indexPattern: $ref: '#/components/schemas/IndexPattern' + lookbackPeriod: + default: 24h + description: The lookback period for the entity store + pattern: '[smdh]$' + type: string description: Schema for the entity store initialization required: true responses: @@ -1010,6 +1015,10 @@ components: type: string indexPattern: $ref: '#/components/schemas/IndexPattern' + lookbackPeriod: + default: 24h + pattern: '[smdh]$' + type: string status: $ref: '#/components/schemas/EngineStatus' type: diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts index 39fabfd19176d..e3e0e21bf1f5a 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts @@ -130,6 +130,18 @@ interface SearchEntitiesParams { sortOrder: SortOrder; } +export const DEFAULT_INIT_ENTITY_STORE: InitEntityStoreRequestBody = { + indexPattern: '', + lookbackPeriod: '24h', + filter: '', + fieldHistoryLength: 10, + enrichPolicyExecutionInterval: DEFAULT_INTERVAL, +}; + +const DEFAULT_ENTITY_ENGINE: InitEntityEngineRequestBody & { lookbackPeriod?: string } = { + ...DEFAULT_INIT_ENTITY_STORE, +}; + export class EntityStoreDataClient { private engineClient: EngineDescriptorClient; private assetCriticalityMigrationClient: AssetCriticalityMigrationClient; @@ -204,19 +216,25 @@ export class EntityStoreDataClient { } public async enable( - { - indexPattern = '', - filter = '', - fieldHistoryLength = 10, - entityTypes, - enrichPolicyExecutionInterval, - }: InitEntityStoreRequestBody, + requestBodyOverrides: Partial = {}, { pipelineDebugMode = false }: { pipelineDebugMode?: boolean } = {} ): Promise { if (!this.options.taskManager) { throw new Error('Task Manager is not available'); } + const { + indexPattern, + lookbackPeriod, + filter, + fieldHistoryLength, + entityTypes, + enrichPolicyExecutionInterval, + } = { + ...DEFAULT_INIT_ENTITY_STORE, + ...requestBodyOverrides, + }; + // Immediately defer the initialization to the next tick. This way we don't block on the init preflight checks const run = (fn: () => Promise) => new Promise((resolve) => setTimeout(() => fn().then(resolve), 0)); @@ -233,7 +251,13 @@ export class EntityStoreDataClient { run(() => this.init( entity, - { indexPattern, filter, fieldHistoryLength, enrichPolicyExecutionInterval }, + { + indexPattern, + lookbackPeriod, + filter, + fieldHistoryLength, + enrichPolicyExecutionInterval, + }, { pipelineDebugMode } ) ) @@ -293,14 +317,21 @@ export class EntityStoreDataClient { public async init( entityType: EntityType, - { - indexPattern = '', - filter = '', - fieldHistoryLength = 10, - enrichPolicyExecutionInterval = DEFAULT_INTERVAL, - }: InitEntityEngineRequestBody, + InitEntityEngineRequestBodyOverrides: Partial = {}, { pipelineDebugMode = false }: { pipelineDebugMode?: boolean } = {} ): Promise { + const mergedRequest = { + ...DEFAULT_ENTITY_ENGINE, + ...InitEntityEngineRequestBodyOverrides, + } as Required; + + const { + indexPattern, + filter, + fieldHistoryLength, + lookbackPeriod, + enrichPolicyExecutionInterval, + } = mergedRequest; const { experimentalFeatures } = this.options; if (entityType === EntityType.universal && !experimentalFeatures.assetInventoryStoreEnabled) { @@ -348,6 +379,7 @@ export class EntityStoreDataClient { const descriptor = await this.engineClient.init(entityType, { filter, fieldHistoryLength, + lookbackPeriod, indexPattern, }); this.log('debug', entityType, `Initialized engine saved object`); @@ -355,6 +387,7 @@ export class EntityStoreDataClient { this.asyncSetup( entityType, fieldHistoryLength, + lookbackPeriod, enrichPolicyExecutionInterval, this.options.taskManager, indexPattern, @@ -371,6 +404,7 @@ export class EntityStoreDataClient { private async asyncSetup( entityType: EntityType, fieldHistoryLength: number, + lookbackPeriod: string, enrichPolicyExecutionInterval: string, taskManager: TaskManagerStartContract, indexPattern: string, @@ -386,7 +420,7 @@ export class EntityStoreDataClient { const description = createEngineDescription({ entityType, namespace, - requestParams: { indexPattern, fieldHistoryLength }, + requestParams: { indexPattern, fieldHistoryLength, lookbackPeriod }, defaultIndexPatterns, config, }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.ts index d60e09935bda9..83c738f2c8895 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.ts @@ -42,6 +42,7 @@ interface EngineDescriptionParams { requestParams?: { indexPattern?: string; fieldHistoryLength?: number; + lookbackPeriod?: string; }; defaultIndexPatterns: string[]; } @@ -59,7 +60,10 @@ export const createEngineDescription = (options: EngineDescriptionParams) => { const settings: EntityEngineInstallationDescriptor['settings'] = { syncDelay: `${config.syncDelay.asSeconds()}s`, frequency: `${config.frequency.asSeconds()}s`, - lookbackPeriod: description.settings?.lookbackPeriod || DEFAULT_LOOKBACK_PERIOD, + lookbackPeriod: + requestParams?.lookbackPeriod || + description.settings?.lookbackPeriod || + DEFAULT_LOOKBACK_PERIOD, timestampField: description.settings?.timestampField || DEFAULT_TIMESTAMP_FIELD, }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/saved_object/engine_descriptor.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/saved_object/engine_descriptor.ts index a3c7a7d0266a1..ccf25e7cede99 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/saved_object/engine_descriptor.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/saved_object/engine_descriptor.ts @@ -37,7 +37,8 @@ export class EngineDescriptorClient { filter, fieldHistoryLength, indexPattern, - }: { filter: string; fieldHistoryLength: number; indexPattern: string } + lookbackPeriod, + }: { filter: string; fieldHistoryLength: number; indexPattern: string; lookbackPeriod: string } ) { const engineDescriptor = await this.find(entityType); @@ -54,6 +55,7 @@ export class EngineDescriptorClient { filter, fieldHistoryLength, indexPattern, + lookbackPeriod, }; await this.deps.soClient.update( entityEngineDescriptorTypeName, @@ -73,6 +75,7 @@ export class EngineDescriptorClient { indexPattern, filter, fieldHistoryLength, + lookbackPeriod, }, { id: this.getSavedObjectId(entityType) } ); diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entity_store.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entity_store.ts index 104fbf05b5159..1a47a01be7be9 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entity_store.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entity_store.ts @@ -90,6 +90,7 @@ export default ({ getService }: FtrProviderContext) => { indexPattern: '', filter: '', fieldHistoryLength: 10, + lookbackPeriod: '24h', }); }); @@ -106,6 +107,7 @@ export default ({ getService }: FtrProviderContext) => { indexPattern: '', filter: '', fieldHistoryLength: 10, + lookbackPeriod: '24h', }); }); }); @@ -124,6 +126,7 @@ export default ({ getService }: FtrProviderContext) => { indexPattern: '', filter: '', fieldHistoryLength: 10, + lookbackPeriod: '24h', }, { status: 'started', @@ -131,6 +134,7 @@ export default ({ getService }: FtrProviderContext) => { indexPattern: '', filter: '', fieldHistoryLength: 10, + lookbackPeriod: '24h', }, ]); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entity_store_nondefault_spaces.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entity_store_nondefault_spaces.ts index 64809533fec7b..2585fafc953a6 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entity_store_nondefault_spaces.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entity_store_nondefault_spaces.ts @@ -77,6 +77,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { type: 'host', filter: '', fieldHistoryLength: 10, + lookbackPeriod: '24h', indexPattern: '', }); }); @@ -96,6 +97,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { type: 'user', filter: '', fieldHistoryLength: 10, + lookbackPeriod: '24h', indexPattern: '', }); }); @@ -114,6 +116,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { type: 'host', filter: '', fieldHistoryLength: 10, + lookbackPeriod: '24h', indexPattern: '', }, { @@ -121,6 +124,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { type: 'user', filter: '', fieldHistoryLength: 10, + lookbackPeriod: '24h', indexPattern: '', }, ]);