From b08eaa7d5a170436d75ae909972bdd6770dea3b3 Mon Sep 17 00:00:00 2001 From: Elena Stoeva <59341489+ElenaStoeva@users.noreply.github.com> Date: Fri, 24 Jan 2025 21:55:16 +0000 Subject: [PATCH] [Index Management] Display index mode of data streams depending on `logsdb.enabled` cluster setting (#207131) Closes https://github.com/elastic/kibana/issues/206126 ## Summary This PR displays a LogsDB index mode for `logs-*-*` data streams and templates with the `logs-*-*` index pattern if the `cluster.logsdb.enabled` setting is set to true and as "Standard" if it's set to false. If the setting is not set, it is false by default in stateful (and so "Standard" index mode) and true by default in serverless (and so "LogsDB" index mode). **How to test:** In Stateful: 1. Create an index template with a `logs-*-*` index pattern 2. Create a `logs-*-*` data stream: `PUT _data_stream/logs-test-1` 3. Go to Index Management -> Index templates and click on the created index templates. Verify that the index mode is correct (**NOTE:** In stateful, the `cluster.logsdb.enabled` setting is `false` by default, while in serverless it's `true` by default. Therefore, the index mode should be `Standard` in stateful and `LogsDB` in serverless. 4. Go to Index Management -> Data Streams and find the created data stream. Verify that the index mode is correct (**NOTE:** In stateful, the `cluster.logsdb.enabled` setting is `false` by default, while in serverless it's `true` by default. Therefore, the index mode should be `Standard` in stateful and `LogsDB` in serverless. 5. In Console, set the `cluster.logsdb.enabled` setting: ``` PUT /_cluster/settings { "persistent" : { "cluster.logsdb.enabled" : true } } ``` 6. Go to the data stream/templates list and verify that the index mode of the test data stream/template that we created is displayed as LogsDB 7. Change the cluster setting to `false` and verify that the index mode is now displayed as Standard. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../common/lib/template_serialization.test.ts | 30 +++++++++ .../common/lib/template_serialization.ts | 11 ++-- .../server/lib/data_stream_serialization.ts | 16 +++-- .../api/data_streams/register_get_route.ts | 19 +++++- .../api/templates/register_get_routes.ts | 19 ++++-- .../index_management/data_streams.ts | 64 ++++++++++++++++--- .../management/index_management/templates.ts | 38 +++++++++++ .../index_management/svl_templates.helpers.ts | 11 +++- .../common/index_management/datastreams.ts | 44 +++++++++++++ .../index_management/index_templates.ts | 49 +++++++++++++- 10 files changed, 273 insertions(+), 28 deletions(-) diff --git a/x-pack/platform/plugins/shared/index_management/common/lib/template_serialization.test.ts b/x-pack/platform/plugins/shared/index_management/common/lib/template_serialization.test.ts index cb86de2660fd3..aec540aa07ca0 100644 --- a/x-pack/platform/plugins/shared/index_management/common/lib/template_serialization.test.ts +++ b/x-pack/platform/plugins/shared/index_management/common/lib/template_serialization.test.ts @@ -62,6 +62,36 @@ describe('Template serialization', () => { ).toHaveProperty('indexMode', value ?? STANDARD_INDEX_MODE); }); }); + + describe('with logs-*-* index pattern', () => { + test('deserializes to logsdb index mode when logsdb is enabled', () => { + expect( + deserializeTemplate( + { + ...defaultSerializedTemplate, + index_patterns: ['logs-*-*'], + name: 'my_template', + }, + undefined, + true + ) + ).toHaveProperty('indexMode', LOGSDB_INDEX_MODE); + }); + + test('deserializes to standard index mode when logsdb is disabled', () => { + expect( + deserializeTemplate( + { + ...defaultSerializedTemplate, + index_patterns: ['logs-*-*'], + name: 'my_template', + }, + undefined, + false + ) + ).toHaveProperty('indexMode', STANDARD_INDEX_MODE); + }); + }); }); describe('serializeTemplate()', () => { diff --git a/x-pack/platform/plugins/shared/index_management/common/lib/template_serialization.ts b/x-pack/platform/plugins/shared/index_management/common/lib/template_serialization.ts index 999023704559c..983b2525c0852 100644 --- a/x-pack/platform/plugins/shared/index_management/common/lib/template_serialization.ts +++ b/x-pack/platform/plugins/shared/index_management/common/lib/template_serialization.ts @@ -63,7 +63,8 @@ export function serializeTemplate(templateDeserialized: TemplateDeserialized): T export function deserializeTemplate( templateEs: TemplateSerialized & { name: string }, - cloudManagedTemplatePrefix?: string + cloudManagedTemplatePrefix?: string, + isLogsdbEnabled?: boolean ): TemplateDeserialized { const { name, @@ -92,7 +93,7 @@ export function deserializeTemplate( const ilmPolicyName = settings?.index?.lifecycle?.name; const indexMode = (settings?.index?.mode ?? - (indexPatterns.some((pattern) => pattern === 'logs-*-*') + (isLogsdbEnabled && indexPatterns.some((pattern) => pattern === 'logs-*-*') ? LOGSDB_INDEX_MODE : STANDARD_INDEX_MODE)) as IndexMode; @@ -168,13 +169,15 @@ export function serializeLegacyTemplate(template: TemplateDeserialized): LegacyT export function deserializeLegacyTemplate( templateEs: LegacyTemplateSerialized & { name: string }, - cloudManagedTemplatePrefix?: string + cloudManagedTemplatePrefix?: string, + isLogsdbEnabled?: boolean ): TemplateDeserialized { const { settings, aliases, mappings, ...rest } = templateEs; const deserializedTemplate = deserializeTemplate( { ...rest, template: { aliases, settings, mappings } }, - cloudManagedTemplatePrefix + cloudManagedTemplatePrefix, + isLogsdbEnabled ); return { diff --git a/x-pack/platform/plugins/shared/index_management/server/lib/data_stream_serialization.ts b/x-pack/platform/plugins/shared/index_management/server/lib/data_stream_serialization.ts index 2556fab947665..27e72356eaaaf 100644 --- a/x-pack/platform/plugins/shared/index_management/server/lib/data_stream_serialization.ts +++ b/x-pack/platform/plugins/shared/index_management/server/lib/data_stream_serialization.ts @@ -6,10 +6,14 @@ */ import { ByteSizeValue } from '@kbn/config-schema'; +import { LOGSDB_INDEX_MODE, STANDARD_INDEX_MODE } from '../../common/constants'; import { IndexMode } from '../../common/types/data_streams'; import type { DataStream, EnhancedDataStreamFromEs, Health } from '../../common'; -export function deserializeDataStream(dataStreamFromEs: EnhancedDataStreamFromEs): DataStream { +export function deserializeDataStream( + dataStreamFromEs: EnhancedDataStreamFromEs, + isLogsdbEnabled: boolean +): DataStream { const { name, timestamp_field: timeStampField, @@ -75,12 +79,16 @@ export function deserializeDataStream(dataStreamFromEs: EnhancedDataStreamFromEs globalMaxRetention, }, nextGenerationManagedBy, - indexMode: (indexMode ?? 'standard') as IndexMode, + indexMode: (indexMode ?? + (isLogsdbEnabled && /^logs-[^-]+-[^-]+$/.test(name) + ? LOGSDB_INDEX_MODE + : STANDARD_INDEX_MODE)) as IndexMode, }; } export function deserializeDataStreamList( - dataStreamsFromEs: EnhancedDataStreamFromEs[] + dataStreamsFromEs: EnhancedDataStreamFromEs[], + isLogsdbEnabled: boolean ): DataStream[] { - return dataStreamsFromEs.map((dataStream) => deserializeDataStream(dataStream)); + return dataStreamsFromEs.map((dataStream) => deserializeDataStream(dataStream, isLogsdbEnabled)); } diff --git a/x-pack/platform/plugins/shared/index_management/server/routes/api/data_streams/register_get_route.ts b/x-pack/platform/plugins/shared/index_management/server/routes/api/data_streams/register_get_route.ts index d6d03d7095ae8..a6ee8bb81c86c 100644 --- a/x-pack/platform/plugins/shared/index_management/server/routes/api/data_streams/register_get_route.ts +++ b/x-pack/platform/plugins/shared/index_management/server/routes/api/data_streams/register_get_route.ts @@ -177,6 +177,12 @@ export function registerGetAllRoute({ router, lib: { handleEsError }, config }: const { index_templates: indexTemplates } = await client.asCurrentUser.indices.getIndexTemplate(); + const { persistent, defaults } = await client.asInternalUser.cluster.getSettings({ + include_defaults: true, + }); + const isLogsdbEnabled = + (persistent?.cluster?.logsdb?.enabled ?? defaults?.cluster?.logsdb?.enabled) === 'true'; + // Only take the lifecycle of the first data stream since all data streams have the same global retention period const lifecycle = await getDataStreamLifecycle(client, dataStreams[0].name); // @ts-ignore - TS doesn't know about the `global_retention` property yet @@ -191,7 +197,9 @@ export function registerGetAllRoute({ router, lib: { handleEsError }, config }: indexTemplates, }); - return response.ok({ body: deserializeDataStreamList(enhancedDataStreams) }); + return response.ok({ + body: deserializeDataStreamList(enhancedDataStreams, isLogsdbEnabled), + }); } catch (error) { return handleEsError({ error, response }); } @@ -262,7 +270,14 @@ export function registerGetOneRoute({ router, lib: { handleEsError }, config }: globalMaxRetention, indexTemplates, }); - const body = deserializeDataStream(enhancedDataStreams[0]); + + const { persistent, defaults } = await client.asInternalUser.cluster.getSettings({ + include_defaults: true, + }); + const isLogsdbEnabled = + (persistent?.cluster?.logsdb?.enabled ?? defaults?.cluster?.logsdb?.enabled) === 'true'; + + const body = deserializeDataStream(enhancedDataStreams[0], isLogsdbEnabled); return response.ok({ body }); } diff --git a/x-pack/platform/plugins/shared/index_management/server/routes/api/templates/register_get_routes.ts b/x-pack/platform/plugins/shared/index_management/server/routes/api/templates/register_get_routes.ts index 702fb9836a672..89e94144f1140 100644 --- a/x-pack/platform/plugins/shared/index_management/server/routes/api/templates/register_get_routes.ts +++ b/x-pack/platform/plugins/shared/index_management/server/routes/api/templates/register_get_routes.ts @@ -37,8 +37,11 @@ export function registerGetAllRoute({ router, config, lib: { handleEsError } }: const { index_templates: templatesEs } = await client.asCurrentUser.indices.getIndexTemplate(); - // @ts-expect-error TemplateSerialized.index_patterns not compatible with IndicesIndexTemplate.index_patterns - const templates = deserializeTemplateList(templatesEs, cloudManagedTemplatePrefix); + const templates = deserializeTemplateList( + // @ts-expect-error TemplateSerialized.index_patterns not compatible with IndicesIndexTemplate.index_patterns + templatesEs, + cloudManagedTemplatePrefix + ); if (config.isLegacyTemplatesEnabled === false) { // If isLegacyTemplatesEnabled=false, we do not want to fetch legacy templates and return an empty array; @@ -98,6 +101,12 @@ export function registerGetOneRoute({ router, config, lib: { handleEsError } }: try { const cloudManagedTemplatePrefix = await getCloudManagedTemplatePrefix(client); + const { persistent, defaults } = await client.asInternalUser.cluster.getSettings({ + include_defaults: true, + }); + const isLogsdbEnabled = + (persistent?.cluster?.logsdb?.enabled ?? defaults?.cluster?.logsdb?.enabled) === 'true'; + if (isLegacy) { const indexTemplateByName = await client.asCurrentUser.indices.getTemplate({ name, @@ -107,7 +116,8 @@ export function registerGetOneRoute({ router, config, lib: { handleEsError } }: return response.ok({ body: deserializeLegacyTemplate( { ...indexTemplateByName[name], name }, - cloudManagedTemplatePrefix + cloudManagedTemplatePrefix, + isLogsdbEnabled ), }); } @@ -120,7 +130,8 @@ export function registerGetOneRoute({ router, config, lib: { handleEsError } }: body: deserializeTemplate( // @ts-expect-error TemplateSerialized.index_patterns not compatible with IndicesIndexTemplate.index_patterns { ...indexTemplates[0].index_template, name }, - cloudManagedTemplatePrefix + cloudManagedTemplatePrefix, + isLogsdbEnabled ), }); } diff --git a/x-pack/test/api_integration/apis/management/index_management/data_streams.ts b/x-pack/test/api_integration/apis/management/index_management/data_streams.ts index 6ea9c601f6cc3..846dc1eb342d0 100644 --- a/x-pack/test/api_integration/apis/management/index_management/data_streams.ts +++ b/x-pack/test/api_integration/apis/management/index_management/data_streams.ts @@ -14,6 +14,7 @@ import { datastreamsHelpers } from './lib/datastreams.helpers'; export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); + const es = getService('es'); const { createDataStream, @@ -164,20 +165,63 @@ export default function ({ getService }: FtrProviderContext) { }); }); - it('correctly returns index mode property', async () => { - const logsdbDataStreamName = 'logsdb-test-data-stream'; - const indexMode = 'logsdb'; + describe('index mode', () => { + it('correctly returns index mode property based on index settings', async () => { + const logsdbDataStreamName = 'logsdb-test-data-stream'; + const indexMode = 'logsdb'; - await createDataStream(logsdbDataStreamName, indexMode); + await createDataStream(logsdbDataStreamName, indexMode); - const { body: dataStream } = await supertest - .get(`${API_BASE_PATH}/data_streams/${logsdbDataStreamName}`) - .set('kbn-xsrf', 'xxx') - .expect(200); + const { body: dataStream } = await supertest + .get(`${API_BASE_PATH}/data_streams/${logsdbDataStreamName}`) + .set('kbn-xsrf', 'xxx') + .expect(200); - expect(dataStream.indexMode).to.eql(indexMode); + expect(dataStream.indexMode).to.eql(indexMode); - await deleteDataStream(logsdbDataStreamName); + await deleteDataStream(logsdbDataStreamName); + }); + + describe('index mode of logs-*-* data streams', () => { + const logsdbDataStreamName = 'logs-test-ds'; + + before(async () => { + await createDataStream(logsdbDataStreamName); + }); + + after(async () => { + await deleteDataStream(logsdbDataStreamName); + }); + + const logsdbSettings: Array<{ enabled: boolean | null; indexMode: string }> = [ + { enabled: true, indexMode: 'logsdb' }, + { enabled: false, indexMode: 'standard' }, + { enabled: null, indexMode: 'standard' }, // In stateful Kibana, the cluster.logsdb.enabled setting is false by default, so standard index mode + ]; + + logsdbSettings.forEach(({ enabled, indexMode }) => { + it(`returns ${indexMode} index mode if logsdb.enabled setting is ${enabled}`, async () => { + await es.cluster.putSettings({ + body: { + persistent: { + cluster: { + logsdb: { + enabled, + }, + }, + }, + }, + }); + + const { body: dataStream } = await supertest + .get(`${API_BASE_PATH}/data_streams/${logsdbDataStreamName}`) + .set('kbn-xsrf', 'xxx') + .expect(200); + + expect(dataStream.indexMode).to.eql(indexMode); + }); + }); + }); }); }); diff --git a/x-pack/test/api_integration/apis/management/index_management/templates.ts b/x-pack/test/api_integration/apis/management/index_management/templates.ts index 066df3120be08..7ba77cb29c721 100644 --- a/x-pack/test/api_integration/apis/management/index_management/templates.ts +++ b/x-pack/test/api_integration/apis/management/index_management/templates.ts @@ -15,6 +15,7 @@ import { getRandomString } from './lib/random'; export default function ({ getService }: FtrProviderContext) { const log = getService('log'); + const es = getService('es'); const { catTemplate, getTemplatePayload, getSerializedTemplate } = templatesHelpers(getService); const { getAllTemplates, @@ -233,6 +234,43 @@ export default function ({ getService }: FtrProviderContext) { expect(Object.keys(body).sort()).to.eql(expectedKeys); expect(Object.keys(body.template).sort()).to.eql(expectedTemplateKeys); }); + + describe('with logs-*-* index pattern', () => { + const logsdbTemplateName = 'test-logsdb-template'; + before(async () => { + const template = getTemplatePayload(logsdbTemplateName, ['logs-*-*']); + await createTemplate(template).expect(200); + }); + + after(async () => { + await deleteTemplates([{ name: logsdbTemplateName }]); + }); + + const logsdbSettings: Array<{ enabled: boolean | null; indexMode: string }> = [ + { enabled: true, indexMode: 'logsdb' }, + { enabled: false, indexMode: 'standard' }, + { enabled: null, indexMode: 'standard' }, // In stateful Kibana, the cluster.logsdb.enabled setting is false by default, so standard index mode + ]; + + logsdbSettings.forEach(({ enabled, indexMode }) => { + it(`returns ${indexMode} index mode if logsdb.enabled setting is ${enabled}`, async () => { + await es.cluster.putSettings({ + body: { + persistent: { + cluster: { + logsdb: { + enabled, + }, + }, + }, + }, + }); + + const { body } = await getOneTemplate(logsdbTemplateName).expect(200); + expect(body.indexMode).to.equal(indexMode); + }); + }); + }); }); describe('create', () => { diff --git a/x-pack/test_serverless/api_integration/services/index_management/svl_templates.helpers.ts b/x-pack/test_serverless/api_integration/services/index_management/svl_templates.helpers.ts index 8c96011936743..abe940276d0ba 100644 --- a/x-pack/test_serverless/api_integration/services/index_management/svl_templates.helpers.ts +++ b/x-pack/test_serverless/api_integration/services/index_management/svl_templates.helpers.ts @@ -5,7 +5,11 @@ * 2.0. */ -import { TemplateDeserialized, TemplateSerialized } from '@kbn/index-management-plugin/common'; +import { + TemplateDeserialized, + TemplateSerialized, + IndexMode, +} from '@kbn/index-management-plugin/common'; import { INDEX_PATTERNS } from './constants'; import { FtrProviderContext } from '../../ftr_provider_context'; @@ -51,12 +55,13 @@ export function SvlTemplatesHelpers({ getService }: FtrProviderContext) { name: string, indexPatterns: string[] = INDEX_PATTERNS, isLegacy: boolean = false, - isMappingsSourceFieldEnabled: boolean = true + isMappingsSourceFieldEnabled: boolean = true, + indexMode?: IndexMode ) => { const baseTemplate: TemplateDeserialized = { name, indexPatterns, - indexMode: 'standard', + indexMode, version: 1, template: { ...getTemplateMock(isMappingsSourceFieldEnabled) }, _kbnMeta: { diff --git a/x-pack/test_serverless/api_integration/test_suites/common/index_management/datastreams.ts b/x-pack/test_serverless/api_integration/test_suites/common/index_management/datastreams.ts index 099e1b9b3c1a6..14e1a5976b7f8 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/index_management/datastreams.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/index_management/datastreams.ts @@ -20,6 +20,8 @@ export default function ({ getService }: FtrProviderContext) { let roleAuthc: RoleCredentials; let internalReqHeader: InternalRequestHeader; const svlDatastreamsHelpers = getService('svlDatastreamsHelpers'); + const supertest = getService('supertest'); + const es = getService('es'); describe('Data streams', function () { // see details: https://github.com/elastic/kibana/issues/187372 @@ -125,6 +127,48 @@ export default function ({ getService }: FtrProviderContext) { indexMode: 'standard', }); }); + + describe('index mode of logs-*-* data streams', () => { + const logsdbDataStreamName = 'logs-test-ds'; + + before(async () => { + await svlDatastreamsHelpers.createDataStream(logsdbDataStreamName); + }); + + after(async () => { + await svlDatastreamsHelpers.deleteDataStream(logsdbDataStreamName); + }); + + const logsdbSettings: Array<{ enabled: boolean | null; indexMode: string }> = [ + { enabled: true, indexMode: 'logsdb' }, + { enabled: false, indexMode: 'standard' }, + { enabled: null, indexMode: 'logsdb' }, // In serverless Kibana, the cluster.logsdb.enabled setting is true by default, so logsdb index mode + ]; + + logsdbSettings.forEach(({ enabled, indexMode }) => { + it(`returns ${indexMode} index mode if logsdb.enabled setting is ${enabled}`, async () => { + await es.cluster.putSettings({ + body: { + persistent: { + cluster: { + logsdb: { + enabled, + }, + }, + }, + }, + }); + + const { body: dataStream } = await supertest + .get(`${API_BASE_PATH}/data_streams/${logsdbDataStreamName}`) + .set(internalReqHeader) + .set(roleAuthc.apiKeyHeader) + .expect(200); + + expect(dataStream.indexMode).to.eql(indexMode); + }); + }); + }); }); describe('Update', () => { diff --git a/x-pack/test_serverless/api_integration/test_suites/common/index_management/index_templates.ts b/x-pack/test_serverless/api_integration/test_suites/common/index_management/index_templates.ts index 1190a642a3518..fbacef1aed4f6 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/index_management/index_templates.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/index_management/index_templates.ts @@ -126,6 +126,52 @@ export default function ({ getService }: FtrProviderContext) { expect(body.name).to.eql(templateName); expect(Object.keys(body).sort()).to.eql(expectedKeys); }); + + describe('with logs-*-* index pattern', () => { + const logsdbTemplateName = 'test-logsdb-template'; + before(async () => { + const template = svlTemplatesHelpers.getTemplatePayload( + logsdbTemplateName, + ['logs-*-*'], + false, + false + ); + await svlTemplatesApi.createTemplate(template, roleAuthc).expect(200); + }); + + after(async () => { + await svlTemplatesApi.deleteTemplates([{ name: logsdbTemplateName }], roleAuthc); + }); + + const logsdbSettings: Array<{ enabled: boolean | null; indexMode: string }> = [ + { enabled: true, indexMode: 'logsdb' }, + { enabled: false, indexMode: 'standard' }, + { enabled: null, indexMode: 'logsdb' }, // In serverless Kibana, the cluster.logsdb.enabled setting is true by default, so logsdb index mode + ]; + + logsdbSettings.forEach(({ enabled, indexMode }) => { + it(`returns ${indexMode} index mode if logsdb.enabled setting is ${enabled}`, async () => { + await es.cluster.putSettings({ + body: { + persistent: { + cluster: { + logsdb: { + enabled, + }, + }, + }, + }, + }); + + const { body, status } = await supertestWithoutAuth + .get(`${API_BASE_PATH}/index_templates/${logsdbTemplateName}`) + .set(internalReqHeader) + .set(roleAuthc.apiKeyHeader); + expect(status).to.eql(200); + expect(body.indexMode).to.equal(indexMode); + }); + }); + }); }); }); @@ -135,7 +181,8 @@ export default function ({ getService }: FtrProviderContext) { `template-${getRandomString()}`, [getRandomString()], undefined, - false + false, + 'logsdb' ); const { status } = await svlTemplatesApi.createTemplate(payload, roleAuthc);