Entity Store Config - Lookback period (elastic#206421)

## Summary This PR enables configurability of the lookbackPeriod for Security’s Entity Store through the enable API. ### Testing 1. Open the latest instance of Kibana. 2. Ensure both the Entity Store and Risk Score are enabled. 3. Navigate to Dev Tools. 4. Use the enable endpoint with lookbackPeriod, where the lookbackPeriod must be a number followed by one of the supported units (m, s, or h). ``` POST kbn:api/entity_store/enable { "lookbackPeriod": "72h" // example value } ``` 5. After enabling, use the status endpoint to confirm the configuration: ``` GET kbn:api/entity_store/status {} ``` 6. Post some documents, to check the entities available within the lookbackPeriod - confirm you can only see those within this period, and not outside of this period e.g. "5h" period, post with a timestamp of today (within 5 hours) and from yesterday and view results. ``` POST lookback-period-test/_doc { "entity_id": "LookbackPeriod Test Today", "entity_name": "LookbackPeriod Test Today", "timestamp": "2025-01-17T14:31:00Z" } ``` ``` POST lookback-period-test/_doc { "entity_id": "LookbackPeriod Test Yesterday", "entity_name": "LookbackPeriod Test Yesterday", "timestamp": "2025-01-16T14:31:00Z" } ``` 7. Can also go to entity store, check one of the transforms and check the JSON gte ranges from here too, they should show the new lookbackPeriod. --------- Co-authored-by: kibanamachine <[email protected]>
e40pud · Jan 24, 2025 · 667040f · 667040f
1 parent b08eaa7
commit 667040f
Show file tree

Hide file tree

Showing 14 changed files with 127 additions and 18 deletions.
diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml
@@ -9735,6 +9735,11 @@ paths:
                   type: string
                 indexPattern:
                   $ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern'
+                lookbackPeriod:
+                  default: 24h
+                  description: The lookback period for the entity store
+                  pattern: '[smdh]$'
+                  type: string
         description: Schema for the entity store initialization
         required: true
       responses:
@@ -50354,6 +50359,10 @@ components:
           type: string
         indexPattern:
           $ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern'
+        lookbackPeriod:
+          default: 24h
+          pattern: '[smdh]$'
+          type: string
         status:
           $ref: '#/components/schemas/Security_Entity_Analytics_API_EngineStatus'
         type:

diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml
@@ -11883,6 +11883,11 @@ paths:
                   type: string
                 indexPattern:
                   $ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern'
+                lookbackPeriod:
+                  default: 24h
+                  description: The lookback period for the entity store
+                  pattern: '[smdh]$'
+                  type: string
         description: Schema for the entity store initialization
         required: true
       responses:
@@ -57206,6 +57211,10 @@ components:
           type: string
         indexPattern:
           $ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern'
+        lookbackPeriod:
+          default: 24h
+          pattern: '[smdh]$'
+          type: string
         status:
           $ref: '#/components/schemas/Security_Entity_Analytics_API_EngineStatus'
         type:

diff --git a/x-pack/platform/packages/shared/kbn-entities-schema/src/schema/common.ts b/x-pack/platform/packages/shared/kbn-entities-schema/src/schema/common.ts
@@ -44,7 +44,9 @@ export const docCountMetricSchema = z.object({
   filter: filterSchema,
 });
 
-export const durationSchema = z.string().regex(/^\d+[m|d|s|h]$/);
+export const durationSchema = z.string().regex(/^\d+[m|s|d|h]$/, {
+  message: 'Must be a number followed by one of the units: m, s, d or h',
+});
 
 export const durationSchemaWithMinimum = (minimumMinutes: number) =>
   durationSchema.refine(

diff --git a/...security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts b/...security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts
@@ -36,6 +36,11 @@ export const EngineDescriptor = z.object({
   status: EngineStatus,
   filter: z.string().optional(),
   fieldHistoryLength: z.number().int(),
+  lookbackPeriod: z
+    .string()
+    .regex(/[smdh]$/)
+    .optional()
+    .default('24h'),
   error: z.object({}).optional(),
 });
 

diff --git a/...ity/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml b/...ity/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml
@@ -32,6 +32,10 @@ components:
           type: string
         fieldHistoryLength:
           type: integer
+        lookbackPeriod: 
+          type: string
+          default: 24h
+          pattern: '[smdh]$'
         error:
           type: object
 

diff --git a/...security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.gen.ts b/...security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.gen.ts
@@ -24,6 +24,14 @@ export const InitEntityStoreRequestBody = z.object({
    * The number of historical values to keep for each field.
    */
   fieldHistoryLength: z.number().int().optional().default(10),
+  /**
+   * The lookback period for the entity store
+   */
+  lookbackPeriod: z
+    .string()
+    .regex(/[smdh]$/)
+    .optional()
+    .default('24h'),
   indexPattern: IndexPattern.optional(),
   filter: z.string().optional(),
   entityTypes: z.array(EntityType).optional(),

diff --git a/...ity/plugins/security_solution/common/api/entity_analytics/entity_store/enable.schema.yaml b/...ity/plugins/security_solution/common/api/entity_analytics/entity_store/enable.schema.yaml
@@ -23,6 +23,11 @@ paths:
                   type: integer
                   description: The number of historical values to keep for each field.
                   default: 10
+                lookbackPeriod: 
+                  type: string
+                  description: The lookback period for the entity store
+                  default: 24h
+                  pattern: '[smdh]$'
                 indexPattern:
                   $ref: './common.schema.yaml#/components/schemas/IndexPattern'
                 filter:

diff --git a/...on/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml b/...on/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
@@ -321,6 +321,11 @@ paths:
                   type: string
                 indexPattern:
                   $ref: '#/components/schemas/IndexPattern'
+                lookbackPeriod:
+                  default: 24h
+                  description: The lookback period for the entity store
+                  pattern: '[smdh]$'
+                  type: string
         description: Schema for the entity store initialization
         required: true
       responses:
@@ -1010,6 +1015,10 @@ components:
           type: string
         indexPattern:
           $ref: '#/components/schemas/IndexPattern'
+        lookbackPeriod:
+          default: 24h
+          pattern: '[smdh]$'
+          type: string
         status:
           $ref: '#/components/schemas/EngineStatus'
         type:

diff --git a/.../openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml b/.../openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
@@ -321,6 +321,11 @@ paths:
                   type: string
                 indexPattern:
                   $ref: '#/components/schemas/IndexPattern'
+                lookbackPeriod:
+                  default: 24h
+                  description: The lookback period for the entity store
+                  pattern: '[smdh]$'
+                  type: string
         description: Schema for the entity store initialization
         required: true
       responses:
@@ -1010,6 +1015,10 @@ components:
           type: string
         indexPattern:
           $ref: '#/components/schemas/IndexPattern'
+        lookbackPeriod:
+          default: 24h
+          pattern: '[smdh]$'
+          type: string
         status:
           $ref: '#/components/schemas/EngineStatus'
         type:

diff --git a/...ns/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts b/...ns/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts
@@ -130,6 +130,18 @@ interface SearchEntitiesParams {
   sortOrder: SortOrder;
 }
 
+export const DEFAULT_INIT_ENTITY_STORE: InitEntityStoreRequestBody = {
+  indexPattern: '',
+  lookbackPeriod: '24h',
+  filter: '',
+  fieldHistoryLength: 10,
+  enrichPolicyExecutionInterval: DEFAULT_INTERVAL,
+};
+
+const DEFAULT_ENTITY_ENGINE: InitEntityEngineRequestBody & { lookbackPeriod?: string } = {
+  ...DEFAULT_INIT_ENTITY_STORE,
+};
+
 export class EntityStoreDataClient {
   private engineClient: EngineDescriptorClient;
   private assetCriticalityMigrationClient: AssetCriticalityMigrationClient;
@@ -204,19 +216,25 @@ export class EntityStoreDataClient {
   }
 
   public async enable(
-    {
-      indexPattern = '',
-      filter = '',
-      fieldHistoryLength = 10,
-      entityTypes,
-      enrichPolicyExecutionInterval,
-    }: InitEntityStoreRequestBody,
+    requestBodyOverrides: Partial<InitEntityStoreRequestBody> = {},
     { pipelineDebugMode = false }: { pipelineDebugMode?: boolean } = {}
   ): Promise<InitEntityStoreResponse> {
     if (!this.options.taskManager) {
       throw new Error('Task Manager is not available');
     }
 
+    const {
+      indexPattern,
+      lookbackPeriod,
+      filter,
+      fieldHistoryLength,
+      entityTypes,
+      enrichPolicyExecutionInterval,
+    } = {
+      ...DEFAULT_INIT_ENTITY_STORE,
+      ...requestBodyOverrides,
+    };
+
     // Immediately defer the initialization to the next tick. This way we don't block on the init preflight checks
     const run = <T>(fn: () => Promise<T>) =>
       new Promise<T>((resolve) => setTimeout(() => fn().then(resolve), 0));
@@ -233,7 +251,13 @@ export class EntityStoreDataClient {
       run(() =>
         this.init(
           entity,
-          { indexPattern, filter, fieldHistoryLength, enrichPolicyExecutionInterval },
+          {
+            indexPattern,
+            lookbackPeriod,
+            filter,
+            fieldHistoryLength,
+            enrichPolicyExecutionInterval,
+          },
           { pipelineDebugMode }
         )
       )
@@ -293,14 +317,21 @@ export class EntityStoreDataClient {
 
   public async init(
     entityType: EntityType,
-    {
-      indexPattern = '',
-      filter = '',
-      fieldHistoryLength = 10,
-      enrichPolicyExecutionInterval = DEFAULT_INTERVAL,
-    }: InitEntityEngineRequestBody,
+    InitEntityEngineRequestBodyOverrides: Partial<typeof DEFAULT_ENTITY_ENGINE> = {},
     { pipelineDebugMode = false }: { pipelineDebugMode?: boolean } = {}
   ): Promise<InitEntityEngineResponse> {
+    const mergedRequest = {
+      ...DEFAULT_ENTITY_ENGINE,
+      ...InitEntityEngineRequestBodyOverrides,
+    } as Required<typeof DEFAULT_ENTITY_ENGINE>;
+
+    const {
+      indexPattern,
+      filter,
+      fieldHistoryLength,
+      lookbackPeriod,
+      enrichPolicyExecutionInterval,
+    } = mergedRequest;
     const { experimentalFeatures } = this.options;
 
     if (entityType === EntityType.universal && !experimentalFeatures.assetInventoryStoreEnabled) {
@@ -348,13 +379,15 @@ export class EntityStoreDataClient {
     const descriptor = await this.engineClient.init(entityType, {
       filter,
       fieldHistoryLength,
+      lookbackPeriod,
       indexPattern,
     });
     this.log('debug', entityType, `Initialized engine saved object`);
 
     this.asyncSetup(
       entityType,
       fieldHistoryLength,
+      lookbackPeriod,
       enrichPolicyExecutionInterval,
       this.options.taskManager,
       indexPattern,
@@ -371,6 +404,7 @@ export class EntityStoreDataClient {
   private async asyncSetup(
     entityType: EntityType,
     fieldHistoryLength: number,
+    lookbackPeriod: string,
     enrichPolicyExecutionInterval: string,
     taskManager: TaskManagerStartContract,
     indexPattern: string,
@@ -386,7 +420,7 @@ export class EntityStoreDataClient {
       const description = createEngineDescription({
         entityType,
         namespace,
-        requestParams: { indexPattern, fieldHistoryLength },
+        requestParams: { indexPattern, fieldHistoryLength, lookbackPeriod },
         defaultIndexPatterns,
         config,
       });

diff --git a/...rity_solution/server/lib/entity_analytics/entity_store/installation/engine_description.ts b/...rity_solution/server/lib/entity_analytics/entity_store/installation/engine_description.ts
@@ -42,6 +42,7 @@ interface EngineDescriptionParams {
   requestParams?: {
     indexPattern?: string;
     fieldHistoryLength?: number;
+    lookbackPeriod?: string;
   };
   defaultIndexPatterns: string[];
 }
@@ -59,7 +60,10 @@ export const createEngineDescription = (options: EngineDescriptionParams) => {
   const settings: EntityEngineInstallationDescriptor['settings'] = {
     syncDelay: `${config.syncDelay.asSeconds()}s`,
     frequency: `${config.frequency.asSeconds()}s`,
-    lookbackPeriod: description.settings?.lookbackPeriod || DEFAULT_LOOKBACK_PERIOD,
+    lookbackPeriod:
+      requestParams?.lookbackPeriod ||
+      description.settings?.lookbackPeriod ||
+      DEFAULT_LOOKBACK_PERIOD,
     timestampField: description.settings?.timestampField || DEFAULT_TIMESTAMP_FIELD,
   };
 

diff --git a/...urity_solution/server/lib/entity_analytics/entity_store/saved_object/engine_descriptor.ts b/...urity_solution/server/lib/entity_analytics/entity_store/saved_object/engine_descriptor.ts
@@ -37,7 +37,8 @@ export class EngineDescriptorClient {
       filter,
       fieldHistoryLength,
       indexPattern,
-    }: { filter: string; fieldHistoryLength: number; indexPattern: string }
+      lookbackPeriod,
+    }: { filter: string; fieldHistoryLength: number; indexPattern: string; lookbackPeriod: string }
   ) {
     const engineDescriptor = await this.find(entityType);
 
@@ -54,6 +55,7 @@ export class EngineDescriptorClient {
         filter,
         fieldHistoryLength,
         indexPattern,
+        lookbackPeriod,
       };
       await this.deps.soClient.update<EngineDescriptor>(
         entityEngineDescriptorTypeName,
@@ -73,6 +75,7 @@ export class EngineDescriptorClient {
         indexPattern,
         filter,
         fieldHistoryLength,
+        lookbackPeriod,
       },
       { id: this.getSavedObjectId(entityType) }
     );

diff --git a/...ion/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entity_store.ts b/...ion/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entity_store.ts
@@ -90,6 +90,7 @@ export default ({ getService }: FtrProviderContext) => {
             indexPattern: '',
             filter: '',
             fieldHistoryLength: 10,
+            lookbackPeriod: '24h',
           });
         });
 
@@ -106,6 +107,7 @@ export default ({ getService }: FtrProviderContext) => {
             indexPattern: '',
             filter: '',
             fieldHistoryLength: 10,
+            lookbackPeriod: '24h',
           });
         });
       });
@@ -124,13 +126,15 @@ export default ({ getService }: FtrProviderContext) => {
               indexPattern: '',
               filter: '',
               fieldHistoryLength: 10,
+              lookbackPeriod: '24h',
             },
             {
               status: 'started',
               type: 'user',
               indexPattern: '',
               filter: '',
               fieldHistoryLength: 10,
+              lookbackPeriod: '24h',
             },
           ]);
         });

diff --git a/...tity_analytics/entity_store/trial_license_complete_tier/entity_store_nondefault_spaces.ts b/...tity_analytics/entity_store/trial_license_complete_tier/entity_store_nondefault_spaces.ts
@@ -77,6 +77,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
             type: 'host',
             filter: '',
             fieldHistoryLength: 10,
+            lookbackPeriod: '24h',
             indexPattern: '',
           });
         });
@@ -96,6 +97,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
             type: 'user',
             filter: '',
             fieldHistoryLength: 10,
+            lookbackPeriod: '24h',
             indexPattern: '',
           });
         });
@@ -114,13 +116,15 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
               type: 'host',
               filter: '',
               fieldHistoryLength: 10,
+              lookbackPeriod: '24h',
               indexPattern: '',
             },
             {
               status: 'started',
               type: 'user',
               filter: '',
               fieldHistoryLength: 10,
+              lookbackPeriod: '24h',
               indexPattern: '',
             },
           ]);