[Rules migration] Basic integration test and folder structure (elasti…

…c#11232) (elastic#206822)

## Summary

[Internal link](elastic/security-team#10820)
to the feature details

Part of elastic/security-team#11232

This PR provides:
* a structure for the SIEM Migrations Integration Tests
* simple SIEM Migrations GET API test

---------

Co-authored-by: kibanamachine <[email protected]>

.buildkite/ftr_security_serverless_configs.yml

@@ -121,6 +121,7 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/spaces/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/siem_migrations/rules/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_endpoint/configs/serverless.endpoint.config.ts
   - x-pack/test/security_solution_endpoint/configs/serverless.integrations.config.ts
   # serverless config files that run deployment-agnostic tests

.buildkite/ftr_security_stateful_configs.yml

@@ -95,6 +95,7 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/spaces/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/siem_migrations/rules/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_endpoint/configs/endpoint.config.ts
   - x-pack/test/security_solution_endpoint/configs/integrations.config.ts
   - x-pack/test/api_integration/apis/cloud_security_posture/config.ts

x-pack/test/security_solution_api_integration/package.json

@@ -52,6 +52,9 @@
     "intialize-server:explore": "node scripts/index.js server explore trial_license_complete_tier",
     "run-tests:explore": "node scripts/index.js runner explore trial_license_complete_tier",
 
+    "initialize-server:siem_migrations:trial_complete": "node ./scripts/index.js server siem_migrations trial_license_complete_tier",
+    "run-tests:siem_migrations:trial_complete": "node ./scripts/index.js runner siem_migrations trial_license_complete_tier",
+
     "genai_kb_entries:server:serverless": "npm run initialize-server:genai:trial_complete knowledge_base/entries serverless",
     "genai_kb_entries:runner:serverless": "npm run run-tests:genai:trial_complete knowledge_base/entries serverless serverlessEnv",
     "genai_kb_entries:qa:serverless": "npm run run-tests:genai:trial_complete knowledge_base/entries serverless qaPeriodicEnv",
@@ -517,6 +520,13 @@
     "explore:users:runner:qa:serverless": "npm run run-tests:explore users serverless qaPeriodicEnv",
     "explore:users:runner:qa:serverless:release": "npm run run-tests:explore users serverless qaEnv",
     "explore:users:server:ess": "npm run intialize-server:explore users ess",
-    "explore:users:runner:ess": "npm run run-tests:explore users ess essEnv"
+    "explore:users:runner:ess": "npm run run-tests:explore users ess essEnv",
+
+    "siem_migrations_rules:server:serverless": "npm run initialize-server:siem_migrations:trial_complete rules serverless",
+    "siem_migrations_rules:runner:serverless": "npm run run-tests:siem_migrations:trial_complete rules serverless serverlessEnv",
+    "siem_migrations_rules:qa:serverless": "npm run run-tests:siem_migrations:trial_complete rules serverless qaPeriodicEnv",
+    "siem_migrations_rules:qa:serverless:release": "npm run run-tests:siem_migrations:trial_complete rules serverless qaEnv",
+    "siem_migrations_rules:server:ess": "npm run initialize-server:siem_migrations:trial_complete rules ess",
+    "siem_migrations_rules:runner:ess": "npm run run-tests:siem_migrations:trial_complete rules ess essEnv"
   }
 }

x-pack/test/security_solution_api_integration/test_suites/siem_migrations/rules/trial_license_complete_tier/configs/ess.config.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const functionalConfig = await readConfigFile(
+    require.resolve('../../../../../config/ess/config.base.trial')
+  );
+
+  return {
+    ...functionalConfig.getAll(),
+    kbnTestServer: {
+      ...functionalConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...functionalConfig.get('kbnTestServer.serverArgs'),
+        `--xpack.securitySolution.enableExperimental=${JSON.stringify(['siemMigrationsEnabled'])}`,
+      ],
+    },
+    testFiles: [require.resolve('..')],
+    junit: {
+      reportName: 'SIEM Migrations Integration Tests - ESS Env - Trial License',
+    },
+  };
+}

x-pack/test/security_solution_api_integration/test_suites/siem_migrations/rules/trial_license_complete_tier/configs/serverless.config.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createTestConfig } from '../../../../../config/serverless/config.base';
+
+export default createTestConfig({
+  kbnTestServerArgs: [
+    `--xpack.securitySolution.enableExperimental=${JSON.stringify(['siemMigrationsEnabled'])}`,
+    `--xpack.securitySolutionServerless.productTypes=${JSON.stringify([
+      { product_line: 'security', product_tier: 'complete' },
+      { product_line: 'endpoint', product_tier: 'complete' },
+      { product_line: 'cloud', product_tier: 'complete' },
+    ])}`,
+  ],
+  testFiles: [require.resolve('..')],
+  junit: {
+    reportName: 'SIEM Migrations Integration Tests - Serverless Env - Complete Tier',
+  },
+});

x-pack/test/security_solution_api_integration/test_suites/siem_migrations/rules/trial_license_complete_tier/get.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from 'expect';
+import { v4 as uuidv4 } from 'uuid';
+import {
+  createMigrationRules,
+  deleteAllMigrationRules,
+  getMigrationRuleDocument,
+  migrationRulesRouteHelpersFactory,
+} from '../../utils';
+import { FtrProviderContext } from '../../../../ftr_provider_context';
+
+export default ({ getService }: FtrProviderContext) => {
+  const es = getService('es');
+  const supertest = getService('supertest');
+  const migrationRulesRoutes = migrationRulesRouteHelpersFactory(supertest);
+
+  describe('@ess @serverless @serverlessQA Get API', () => {
+    beforeEach(async () => {
+      await deleteAllMigrationRules(es);
+    });
+
+    it('should fetch existing rules within specified migration', async () => {
+      // create a document
+      const migrationId = uuidv4();
+      const migrationRuleDocument = getMigrationRuleDocument({ migration_id: migrationId });
+      await createMigrationRules(es, [migrationRuleDocument]);
+
+      const { '@timestamp': timestamp, updated_at: updatedAt, ...rest } = migrationRuleDocument;
+
+      // fetch migration rule
+      const response = await migrationRulesRoutes.get(migrationId);
+      expect(response.body.total).toEqual(1);
+      expect(response.body.data).toEqual(expect.arrayContaining([expect.objectContaining(rest)]));
+    });
+  });
+};

x-pack/test/security_solution_api_integration/test_suites/siem_migrations/rules/trial_license_complete_tier/index.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { FtrProviderContext } from '../../../../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
+  describe('@ess SecuritySolution SIEM Migrations', () => {
+    loadTestFile(require.resolve('./get'));
+  });
+}

x-pack/test/security_solution_api_integration/test_suites/siem_migrations/utils/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './rules';

x-pack/test/security_solution_api_integration/test_suites/siem_migrations/utils/rules.ts

@@ -0,0 +1,124 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import SuperTest from 'supertest';
+import type { Client } from '@elastic/elasticsearch';
+import { X_ELASTIC_INTERNAL_ORIGIN_REQUEST } from '@kbn/core-http-common';
+import { replaceParams } from '@kbn/openapi-common/shared';
+
+import { RuleMigration } from '@kbn/security-solution-plugin/common/siem_migrations/model/rule_migration.gen';
+import { INDEX_PATTERN as SIEM_MIGRATIONS_INDEX_PATTERN } from '@kbn/security-solution-plugin/server/lib/siem_migrations/rules/data/rule_migrations_data_service';
+import { SIEM_RULE_MIGRATION_PATH } from '@kbn/security-solution-plugin/common/siem_migrations/constants';
+import { GetRuleMigrationResponse } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rule_migration.gen';
+
+const SIEM_MIGRATIONS_RULES_INDEX_PATTERN = `${SIEM_MIGRATIONS_INDEX_PATTERN}-rules-default`;
+
+export type RuleMigrationDocument = Omit<RuleMigration, 'id'>;
+
+const migrationRuleDocument: RuleMigrationDocument = {
+  '@timestamp': '2025-01-13T15:17:43.571Z',
+  migration_id: '25a24356-3aab-401b-a73c-905cb8bf7a6d',
+  original_rule: {
+    id: 'https://127.0.0.1:8089/servicesNS/nobody/SA-AccessProtection/saved/searches/Access%20-%20Default%20Account%20Usage%20-%20Rule',
+    vendor: 'splunk',
+    title: 'Access - Default Account Usage - Rule',
+    description:
+      'Discovers use of default accounts (such as admin, administrator, etc.). Default accounts have default passwords and are therefore commonly targeted by attackers using brute force attack tools.',
+    query:
+      '| from datamodel:"Authentication"."Successful_Default_Authentication" | stats max("_time") as "lastTime",values("tag") as "tag",count by "dest","user","app"',
+    query_language: 'spl',
+    annotations: {
+      mitre_attack: ['T1078'],
+    },
+  },
+  status: 'completed',
+  created_by: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0',
+  updated_by: 'u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0',
+  updated_at: '2025-01-13T15:39:48.729Z',
+  comments: [
+    '## Prebuilt Rule Matching Summary\nThe Splunk rule "Access - Default Account Usage - Rule" is intended to discover the use of default accounts, which are commonly targeted by attackers using brute force attack tools. However, none of the provided Elastic Prebuilt Rules specifically address the detection of default account usage. The closest matches involve brute force attacks in general, but they do not specifically focus on default accounts. Therefore, no suitable match was found.',
+    '## Integration Matching Summary\nNo related integration found.',
+    '## Translation Summary\n\nThe provided Splunk SPL query was analyzed and translated into the equivalent ES|QL query. Here is a breakdown of the process:\n\n### Original SPL Query\n```splunk-spl\n| from datamodel:"Authentication"."Successful_Default_Authentication" \n| stats max("_time") as "lastTime",\nvalues("tag") as "tag",\ncount by "dest","user","app"\n```\n\n### Key SPL Components and Their ES|QL Equivalents:\n1. **Data Model**: `from datamodel:"Authentication"."Successful_Default_Authentication"` is not directly translatable to ES|QL. Instead, we use `FROM logs-*` to define the data source.\n2. **Stats Aggregation**: The `stats max("_time") as "lastTime", values("tag") as "tag", count by "dest","user","app"` was translated as follows:\n   - `max(_time) as lastTime` to find the latest time.\n   - `values(tag) as tag` to collect all values in the `tag` field.\n   - `count by dest, user, app` to count the occurrences grouped by `dest`, `user`, and `app`.\n\nBy analyzing these key components and their ES|QL equivalents, the translated query achieves the same results as the SPL query while adhering to the ES|QL syntax and structure.',
+  ],
+  translation_result: 'partial',
+  elastic_rule: {
+    severity: 'low',
+    integration_id: '',
+    query:
+      'FROM [indexPattern]\n| STATS lastTime = max(_time), tag = values(tag), count BY dest, user, app',
+    description:
+      'Discovers use of default accounts (such as admin, administrator, etc.). Default accounts have default passwords and are therefore commonly targeted by attackers using brute force attack tools.',
+    query_language: 'esql',
+    title: 'Access - Default Account Usage - Rule',
+  },
+};
+
+export const getMigrationRuleDocument = (
+  overrideParams: Partial<RuleMigrationDocument>
+): RuleMigrationDocument => ({
+  ...migrationRuleDocument,
+  ...overrideParams,
+});
+
+export const createMigrationRules = async (
+  es: Client,
+  rules: RuleMigrationDocument[]
+): Promise<void> => {
+  const createdAt = new Date().toISOString();
+  await es.bulk({
+    refresh: 'wait_for',
+    operations: rules.flatMap((ruleMigration) => [
+      { create: { _index: SIEM_MIGRATIONS_RULES_INDEX_PATTERN } },
+      {
+        ...ruleMigration,
+        '@timestamp': createdAt,
+        updated_at: createdAt,
+      },
+    ]),
+  });
+};
+
+export const deleteAllMigrationRules = async (es: Client): Promise<void> => {
+  await es.deleteByQuery({
+    index: [SIEM_MIGRATIONS_RULES_INDEX_PATTERN],
+    body: {
+      query: {
+        match_all: {},
+      },
+    },
+    ignore_unavailable: true,
+    refresh: true,
+  });
+};
+
+const assertStatusCode = (statusCode: number, response: SuperTest.Response) => {
+  if (response.status !== statusCode) {
+    throw new Error(
+      `Expected status code ${statusCode}, but got ${response.statusCode} \n` + response.text
+    );
+  }
+};
+
+export const migrationRulesRouteHelpersFactory = (supertest: SuperTest.Agent) => {
+  return {
+    get: async (
+      migrationId: string,
+      expectStatusCode: number = 200
+    ): Promise<{ body: GetRuleMigrationResponse }> => {
+      const response = await supertest
+        .get(replaceParams(SIEM_RULE_MIGRATION_PATH, { migration_id: migrationId }))
+        .set('kbn-xsrf', 'true')
+        .set('elastic-api-version', '1')
+        .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana')
+        .send();
+
+      assertStatusCode(expectStatusCode, response);
+
+      return response;
+    },
+  };
+};

x-pack/test/security_solution_api_integration/tsconfig.json

@@ -53,5 +53,6 @@
     "@kbn/spaces-plugin",
     "@kbn/elastic-assistant-plugin",
     "@kbn/test-suites-src",
+    "@kbn/openapi-common",
   ]
 }