diff --git a/x-pack/platform/plugins/shared/index_management/common/lib/template_serialization.test.ts b/x-pack/platform/plugins/shared/index_management/common/lib/template_serialization.test.ts index cb86de2660fd3..aec540aa07ca0 100644 --- a/x-pack/platform/plugins/shared/index_management/common/lib/template_serialization.test.ts +++ b/x-pack/platform/plugins/shared/index_management/common/lib/template_serialization.test.ts @@ -62,6 +62,36 @@ describe('Template serialization', () => { ).toHaveProperty('indexMode', value ?? STANDARD_INDEX_MODE); }); }); + + describe('with logs-*-* index pattern', () => { + test('deserializes to logsdb index mode when logsdb is enabled', () => { + expect( + deserializeTemplate( + { + ...defaultSerializedTemplate, + index_patterns: ['logs-*-*'], + name: 'my_template', + }, + undefined, + true + ) + ).toHaveProperty('indexMode', LOGSDB_INDEX_MODE); + }); + + test('deserializes to standard index mode when logsdb is disabled', () => { + expect( + deserializeTemplate( + { + ...defaultSerializedTemplate, + index_patterns: ['logs-*-*'], + name: 'my_template', + }, + undefined, + false + ) + ).toHaveProperty('indexMode', STANDARD_INDEX_MODE); + }); + }); }); describe('serializeTemplate()', () => { diff --git a/x-pack/platform/plugins/shared/index_management/common/lib/template_serialization.ts b/x-pack/platform/plugins/shared/index_management/common/lib/template_serialization.ts index 999023704559c..983b2525c0852 100644 --- a/x-pack/platform/plugins/shared/index_management/common/lib/template_serialization.ts +++ b/x-pack/platform/plugins/shared/index_management/common/lib/template_serialization.ts @@ -63,7 +63,8 @@ export function serializeTemplate(templateDeserialized: TemplateDeserialized): T export function deserializeTemplate( templateEs: TemplateSerialized & { name: string }, - cloudManagedTemplatePrefix?: string + cloudManagedTemplatePrefix?: string, + isLogsdbEnabled?: boolean ): TemplateDeserialized { const { name, @@ -92,7 +93,7 @@ export function deserializeTemplate( const ilmPolicyName = settings?.index?.lifecycle?.name; const indexMode = (settings?.index?.mode ?? - (indexPatterns.some((pattern) => pattern === 'logs-*-*') + (isLogsdbEnabled && indexPatterns.some((pattern) => pattern === 'logs-*-*') ? LOGSDB_INDEX_MODE : STANDARD_INDEX_MODE)) as IndexMode; @@ -168,13 +169,15 @@ export function serializeLegacyTemplate(template: TemplateDeserialized): LegacyT export function deserializeLegacyTemplate( templateEs: LegacyTemplateSerialized & { name: string }, - cloudManagedTemplatePrefix?: string + cloudManagedTemplatePrefix?: string, + isLogsdbEnabled?: boolean ): TemplateDeserialized { const { settings, aliases, mappings, ...rest } = templateEs; const deserializedTemplate = deserializeTemplate( { ...rest, template: { aliases, settings, mappings } }, - cloudManagedTemplatePrefix + cloudManagedTemplatePrefix, + isLogsdbEnabled ); return { diff --git a/x-pack/platform/plugins/shared/index_management/server/lib/data_stream_serialization.ts b/x-pack/platform/plugins/shared/index_management/server/lib/data_stream_serialization.ts index 2556fab947665..27e72356eaaaf 100644 --- a/x-pack/platform/plugins/shared/index_management/server/lib/data_stream_serialization.ts +++ b/x-pack/platform/plugins/shared/index_management/server/lib/data_stream_serialization.ts @@ -6,10 +6,14 @@ */ import { ByteSizeValue } from '@kbn/config-schema'; +import { LOGSDB_INDEX_MODE, STANDARD_INDEX_MODE } from '../../common/constants'; import { IndexMode } from '../../common/types/data_streams'; import type { DataStream, EnhancedDataStreamFromEs, Health } from '../../common'; -export function deserializeDataStream(dataStreamFromEs: EnhancedDataStreamFromEs): DataStream { +export function deserializeDataStream( + dataStreamFromEs: EnhancedDataStreamFromEs, + isLogsdbEnabled: boolean +): DataStream { const { name, timestamp_field: timeStampField, @@ -75,12 +79,16 @@ export function deserializeDataStream(dataStreamFromEs: EnhancedDataStreamFromEs globalMaxRetention, }, nextGenerationManagedBy, - indexMode: (indexMode ?? 'standard') as IndexMode, + indexMode: (indexMode ?? + (isLogsdbEnabled && /^logs-[^-]+-[^-]+$/.test(name) + ? LOGSDB_INDEX_MODE + : STANDARD_INDEX_MODE)) as IndexMode, }; } export function deserializeDataStreamList( - dataStreamsFromEs: EnhancedDataStreamFromEs[] + dataStreamsFromEs: EnhancedDataStreamFromEs[], + isLogsdbEnabled: boolean ): DataStream[] { - return dataStreamsFromEs.map((dataStream) => deserializeDataStream(dataStream)); + return dataStreamsFromEs.map((dataStream) => deserializeDataStream(dataStream, isLogsdbEnabled)); } diff --git a/x-pack/platform/plugins/shared/index_management/server/routes/api/data_streams/register_get_route.ts b/x-pack/platform/plugins/shared/index_management/server/routes/api/data_streams/register_get_route.ts index d6d03d7095ae8..a6ee8bb81c86c 100644 --- a/x-pack/platform/plugins/shared/index_management/server/routes/api/data_streams/register_get_route.ts +++ b/x-pack/platform/plugins/shared/index_management/server/routes/api/data_streams/register_get_route.ts @@ -177,6 +177,12 @@ export function registerGetAllRoute({ router, lib: { handleEsError }, config }: const { index_templates: indexTemplates } = await client.asCurrentUser.indices.getIndexTemplate(); + const { persistent, defaults } = await client.asInternalUser.cluster.getSettings({ + include_defaults: true, + }); + const isLogsdbEnabled = + (persistent?.cluster?.logsdb?.enabled ?? defaults?.cluster?.logsdb?.enabled) === 'true'; + // Only take the lifecycle of the first data stream since all data streams have the same global retention period const lifecycle = await getDataStreamLifecycle(client, dataStreams[0].name); // @ts-ignore - TS doesn't know about the `global_retention` property yet @@ -191,7 +197,9 @@ export function registerGetAllRoute({ router, lib: { handleEsError }, config }: indexTemplates, }); - return response.ok({ body: deserializeDataStreamList(enhancedDataStreams) }); + return response.ok({ + body: deserializeDataStreamList(enhancedDataStreams, isLogsdbEnabled), + }); } catch (error) { return handleEsError({ error, response }); } @@ -262,7 +270,14 @@ export function registerGetOneRoute({ router, lib: { handleEsError }, config }: globalMaxRetention, indexTemplates, }); - const body = deserializeDataStream(enhancedDataStreams[0]); + + const { persistent, defaults } = await client.asInternalUser.cluster.getSettings({ + include_defaults: true, + }); + const isLogsdbEnabled = + (persistent?.cluster?.logsdb?.enabled ?? defaults?.cluster?.logsdb?.enabled) === 'true'; + + const body = deserializeDataStream(enhancedDataStreams[0], isLogsdbEnabled); return response.ok({ body }); } diff --git a/x-pack/platform/plugins/shared/index_management/server/routes/api/templates/register_get_routes.ts b/x-pack/platform/plugins/shared/index_management/server/routes/api/templates/register_get_routes.ts index 702fb9836a672..89e94144f1140 100644 --- a/x-pack/platform/plugins/shared/index_management/server/routes/api/templates/register_get_routes.ts +++ b/x-pack/platform/plugins/shared/index_management/server/routes/api/templates/register_get_routes.ts @@ -37,8 +37,11 @@ export function registerGetAllRoute({ router, config, lib: { handleEsError } }: const { index_templates: templatesEs } = await client.asCurrentUser.indices.getIndexTemplate(); - // @ts-expect-error TemplateSerialized.index_patterns not compatible with IndicesIndexTemplate.index_patterns - const templates = deserializeTemplateList(templatesEs, cloudManagedTemplatePrefix); + const templates = deserializeTemplateList( + // @ts-expect-error TemplateSerialized.index_patterns not compatible with IndicesIndexTemplate.index_patterns + templatesEs, + cloudManagedTemplatePrefix + ); if (config.isLegacyTemplatesEnabled === false) { // If isLegacyTemplatesEnabled=false, we do not want to fetch legacy templates and return an empty array; @@ -98,6 +101,12 @@ export function registerGetOneRoute({ router, config, lib: { handleEsError } }: try { const cloudManagedTemplatePrefix = await getCloudManagedTemplatePrefix(client); + const { persistent, defaults } = await client.asInternalUser.cluster.getSettings({ + include_defaults: true, + }); + const isLogsdbEnabled = + (persistent?.cluster?.logsdb?.enabled ?? defaults?.cluster?.logsdb?.enabled) === 'true'; + if (isLegacy) { const indexTemplateByName = await client.asCurrentUser.indices.getTemplate({ name, @@ -107,7 +116,8 @@ export function registerGetOneRoute({ router, config, lib: { handleEsError } }: return response.ok({ body: deserializeLegacyTemplate( { ...indexTemplateByName[name], name }, - cloudManagedTemplatePrefix + cloudManagedTemplatePrefix, + isLogsdbEnabled ), }); } @@ -120,7 +130,8 @@ export function registerGetOneRoute({ router, config, lib: { handleEsError } }: body: deserializeTemplate( // @ts-expect-error TemplateSerialized.index_patterns not compatible with IndicesIndexTemplate.index_patterns { ...indexTemplates[0].index_template, name }, - cloudManagedTemplatePrefix + cloudManagedTemplatePrefix, + isLogsdbEnabled ), }); } diff --git a/x-pack/test/api_integration/apis/management/index_management/data_streams.ts b/x-pack/test/api_integration/apis/management/index_management/data_streams.ts index 6ea9c601f6cc3..846dc1eb342d0 100644 --- a/x-pack/test/api_integration/apis/management/index_management/data_streams.ts +++ b/x-pack/test/api_integration/apis/management/index_management/data_streams.ts @@ -14,6 +14,7 @@ import { datastreamsHelpers } from './lib/datastreams.helpers'; export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); + const es = getService('es'); const { createDataStream, @@ -164,20 +165,63 @@ export default function ({ getService }: FtrProviderContext) { }); }); - it('correctly returns index mode property', async () => { - const logsdbDataStreamName = 'logsdb-test-data-stream'; - const indexMode = 'logsdb'; + describe('index mode', () => { + it('correctly returns index mode property based on index settings', async () => { + const logsdbDataStreamName = 'logsdb-test-data-stream'; + const indexMode = 'logsdb'; - await createDataStream(logsdbDataStreamName, indexMode); + await createDataStream(logsdbDataStreamName, indexMode); - const { body: dataStream } = await supertest - .get(`${API_BASE_PATH}/data_streams/${logsdbDataStreamName}`) - .set('kbn-xsrf', 'xxx') - .expect(200); + const { body: dataStream } = await supertest + .get(`${API_BASE_PATH}/data_streams/${logsdbDataStreamName}`) + .set('kbn-xsrf', 'xxx') + .expect(200); - expect(dataStream.indexMode).to.eql(indexMode); + expect(dataStream.indexMode).to.eql(indexMode); - await deleteDataStream(logsdbDataStreamName); + await deleteDataStream(logsdbDataStreamName); + }); + + describe('index mode of logs-*-* data streams', () => { + const logsdbDataStreamName = 'logs-test-ds'; + + before(async () => { + await createDataStream(logsdbDataStreamName); + }); + + after(async () => { + await deleteDataStream(logsdbDataStreamName); + }); + + const logsdbSettings: Array<{ enabled: boolean | null; indexMode: string }> = [ + { enabled: true, indexMode: 'logsdb' }, + { enabled: false, indexMode: 'standard' }, + { enabled: null, indexMode: 'standard' }, // In stateful Kibana, the cluster.logsdb.enabled setting is false by default, so standard index mode + ]; + + logsdbSettings.forEach(({ enabled, indexMode }) => { + it(`returns ${indexMode} index mode if logsdb.enabled setting is ${enabled}`, async () => { + await es.cluster.putSettings({ + body: { + persistent: { + cluster: { + logsdb: { + enabled, + }, + }, + }, + }, + }); + + const { body: dataStream } = await supertest + .get(`${API_BASE_PATH}/data_streams/${logsdbDataStreamName}`) + .set('kbn-xsrf', 'xxx') + .expect(200); + + expect(dataStream.indexMode).to.eql(indexMode); + }); + }); + }); }); }); diff --git a/x-pack/test/api_integration/apis/management/index_management/templates.ts b/x-pack/test/api_integration/apis/management/index_management/templates.ts index 066df3120be08..7ba77cb29c721 100644 --- a/x-pack/test/api_integration/apis/management/index_management/templates.ts +++ b/x-pack/test/api_integration/apis/management/index_management/templates.ts @@ -15,6 +15,7 @@ import { getRandomString } from './lib/random'; export default function ({ getService }: FtrProviderContext) { const log = getService('log'); + const es = getService('es'); const { catTemplate, getTemplatePayload, getSerializedTemplate } = templatesHelpers(getService); const { getAllTemplates, @@ -233,6 +234,43 @@ export default function ({ getService }: FtrProviderContext) { expect(Object.keys(body).sort()).to.eql(expectedKeys); expect(Object.keys(body.template).sort()).to.eql(expectedTemplateKeys); }); + + describe('with logs-*-* index pattern', () => { + const logsdbTemplateName = 'test-logsdb-template'; + before(async () => { + const template = getTemplatePayload(logsdbTemplateName, ['logs-*-*']); + await createTemplate(template).expect(200); + }); + + after(async () => { + await deleteTemplates([{ name: logsdbTemplateName }]); + }); + + const logsdbSettings: Array<{ enabled: boolean | null; indexMode: string }> = [ + { enabled: true, indexMode: 'logsdb' }, + { enabled: false, indexMode: 'standard' }, + { enabled: null, indexMode: 'standard' }, // In stateful Kibana, the cluster.logsdb.enabled setting is false by default, so standard index mode + ]; + + logsdbSettings.forEach(({ enabled, indexMode }) => { + it(`returns ${indexMode} index mode if logsdb.enabled setting is ${enabled}`, async () => { + await es.cluster.putSettings({ + body: { + persistent: { + cluster: { + logsdb: { + enabled, + }, + }, + }, + }, + }); + + const { body } = await getOneTemplate(logsdbTemplateName).expect(200); + expect(body.indexMode).to.equal(indexMode); + }); + }); + }); }); describe('create', () => { diff --git a/x-pack/test_serverless/api_integration/services/index_management/svl_templates.helpers.ts b/x-pack/test_serverless/api_integration/services/index_management/svl_templates.helpers.ts index 8c96011936743..abe940276d0ba 100644 --- a/x-pack/test_serverless/api_integration/services/index_management/svl_templates.helpers.ts +++ b/x-pack/test_serverless/api_integration/services/index_management/svl_templates.helpers.ts @@ -5,7 +5,11 @@ * 2.0. */ -import { TemplateDeserialized, TemplateSerialized } from '@kbn/index-management-plugin/common'; +import { + TemplateDeserialized, + TemplateSerialized, + IndexMode, +} from '@kbn/index-management-plugin/common'; import { INDEX_PATTERNS } from './constants'; import { FtrProviderContext } from '../../ftr_provider_context'; @@ -51,12 +55,13 @@ export function SvlTemplatesHelpers({ getService }: FtrProviderContext) { name: string, indexPatterns: string[] = INDEX_PATTERNS, isLegacy: boolean = false, - isMappingsSourceFieldEnabled: boolean = true + isMappingsSourceFieldEnabled: boolean = true, + indexMode?: IndexMode ) => { const baseTemplate: TemplateDeserialized = { name, indexPatterns, - indexMode: 'standard', + indexMode, version: 1, template: { ...getTemplateMock(isMappingsSourceFieldEnabled) }, _kbnMeta: { diff --git a/x-pack/test_serverless/api_integration/test_suites/common/index_management/datastreams.ts b/x-pack/test_serverless/api_integration/test_suites/common/index_management/datastreams.ts index 099e1b9b3c1a6..14e1a5976b7f8 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/index_management/datastreams.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/index_management/datastreams.ts @@ -20,6 +20,8 @@ export default function ({ getService }: FtrProviderContext) { let roleAuthc: RoleCredentials; let internalReqHeader: InternalRequestHeader; const svlDatastreamsHelpers = getService('svlDatastreamsHelpers'); + const supertest = getService('supertest'); + const es = getService('es'); describe('Data streams', function () { // see details: https://github.com/elastic/kibana/issues/187372 @@ -125,6 +127,48 @@ export default function ({ getService }: FtrProviderContext) { indexMode: 'standard', }); }); + + describe('index mode of logs-*-* data streams', () => { + const logsdbDataStreamName = 'logs-test-ds'; + + before(async () => { + await svlDatastreamsHelpers.createDataStream(logsdbDataStreamName); + }); + + after(async () => { + await svlDatastreamsHelpers.deleteDataStream(logsdbDataStreamName); + }); + + const logsdbSettings: Array<{ enabled: boolean | null; indexMode: string }> = [ + { enabled: true, indexMode: 'logsdb' }, + { enabled: false, indexMode: 'standard' }, + { enabled: null, indexMode: 'logsdb' }, // In serverless Kibana, the cluster.logsdb.enabled setting is true by default, so logsdb index mode + ]; + + logsdbSettings.forEach(({ enabled, indexMode }) => { + it(`returns ${indexMode} index mode if logsdb.enabled setting is ${enabled}`, async () => { + await es.cluster.putSettings({ + body: { + persistent: { + cluster: { + logsdb: { + enabled, + }, + }, + }, + }, + }); + + const { body: dataStream } = await supertest + .get(`${API_BASE_PATH}/data_streams/${logsdbDataStreamName}`) + .set(internalReqHeader) + .set(roleAuthc.apiKeyHeader) + .expect(200); + + expect(dataStream.indexMode).to.eql(indexMode); + }); + }); + }); }); describe('Update', () => { diff --git a/x-pack/test_serverless/api_integration/test_suites/common/index_management/index_templates.ts b/x-pack/test_serverless/api_integration/test_suites/common/index_management/index_templates.ts index 1190a642a3518..fbacef1aed4f6 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/index_management/index_templates.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/index_management/index_templates.ts @@ -126,6 +126,52 @@ export default function ({ getService }: FtrProviderContext) { expect(body.name).to.eql(templateName); expect(Object.keys(body).sort()).to.eql(expectedKeys); }); + + describe('with logs-*-* index pattern', () => { + const logsdbTemplateName = 'test-logsdb-template'; + before(async () => { + const template = svlTemplatesHelpers.getTemplatePayload( + logsdbTemplateName, + ['logs-*-*'], + false, + false + ); + await svlTemplatesApi.createTemplate(template, roleAuthc).expect(200); + }); + + after(async () => { + await svlTemplatesApi.deleteTemplates([{ name: logsdbTemplateName }], roleAuthc); + }); + + const logsdbSettings: Array<{ enabled: boolean | null; indexMode: string }> = [ + { enabled: true, indexMode: 'logsdb' }, + { enabled: false, indexMode: 'standard' }, + { enabled: null, indexMode: 'logsdb' }, // In serverless Kibana, the cluster.logsdb.enabled setting is true by default, so logsdb index mode + ]; + + logsdbSettings.forEach(({ enabled, indexMode }) => { + it(`returns ${indexMode} index mode if logsdb.enabled setting is ${enabled}`, async () => { + await es.cluster.putSettings({ + body: { + persistent: { + cluster: { + logsdb: { + enabled, + }, + }, + }, + }, + }); + + const { body, status } = await supertestWithoutAuth + .get(`${API_BASE_PATH}/index_templates/${logsdbTemplateName}`) + .set(internalReqHeader) + .set(roleAuthc.apiKeyHeader); + expect(status).to.eql(200); + expect(body.indexMode).to.equal(indexMode); + }); + }); + }); }); }); @@ -135,7 +181,8 @@ export default function ({ getService }: FtrProviderContext) { `template-${getRandomString()}`, [getRandomString()], undefined, - false + false, + 'logsdb' ); const { status } = await svlTemplatesApi.createTemplate(payload, roleAuthc);