diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java index 00fb285ab34..260c1965a20 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java @@ -39,6 +39,8 @@ import com.microsoft.applicationinsights.agent.internal.exporter.AgentSpanExporter; import com.microsoft.applicationinsights.agent.internal.exporter.ExporterUtils; import com.microsoft.applicationinsights.agent.internal.httpclient.LazyHttpClient; +import com.microsoft.applicationinsights.agent.internal.keytransaction.KeyTransactionConfigSupplier; +import com.microsoft.applicationinsights.agent.internal.keytransaction.KeyTransactionSpanProcessor; import com.microsoft.applicationinsights.agent.internal.legacyheaders.AiLegacyHeaderSpanProcessor; import com.microsoft.applicationinsights.agent.internal.processors.ExporterWithLogProcessor; import com.microsoft.applicationinsights.agent.internal.processors.ExporterWithSpanProcessor; @@ -550,6 +552,10 @@ private static SdkTracerProviderBuilder configureTracing( tracerProvider.addSpanProcessor(new AiLegacyHeaderSpanProcessor()); } + if (KeyTransactionConfigSupplier.KEY_TRANSACTIONS_ENABLED) { + tracerProvider.addSpanProcessor(new KeyTransactionSpanProcessor()); + } + return tracerProvider; } @@ -745,6 +751,9 @@ private static void drop( @Override public void afterAutoConfigure(OpenTelemetrySdk sdk) { + + KeyTransactionSpanProcessor.initMeterProvider(sdk.getMeterProvider()); + Runtime.getRuntime() .addShutdownHook( new Thread( diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionConfig.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionConfig.java new file mode 100644 index 00000000000..64d963f33d5 --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionConfig.java @@ -0,0 +1,146 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.keytransaction; + +import com.azure.json.JsonReader; +import com.azure.json.JsonToken; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import java.io.IOException; +import java.util.List; + +public class KeyTransactionConfig { + + private String name; + private List startCriteria; + private List endCriteria; + + String getName() { + return name; + } + + List getStartCriteria() { + return startCriteria; + } + + List getEndCriteria() { + return endCriteria; + } + + public static KeyTransactionConfig fromJson(JsonReader jsonReader) throws IOException { + return jsonReader.readObject( + (reader) -> { + KeyTransactionConfig deserializedValue = new KeyTransactionConfig(); + + while (reader.nextToken() != JsonToken.END_OBJECT) { + String fieldName = reader.getFieldName(); + reader.nextToken(); + if ("Name".equals(fieldName)) { + deserializedValue.name = reader.getString(); + } else if ("StartCriteria".equals(fieldName)) { + deserializedValue.startCriteria = reader.readArray(Criterion::fromJson); + } else if ("EndCriteria".equals(fieldName)) { + deserializedValue.endCriteria = reader.readArray(Criterion::fromJson); + } else { + reader.skipChildren(); + } + } + + return deserializedValue; + }); + } + + public static boolean matches(Attributes attributes, List criteria) { + for (Criterion criterion : criteria) { + String value = attributes.get(criterion.field); + switch (criterion.operator) { + case EQUALS: + if (value == null || !value.equals(criterion.value)) { + return false; + } + break; + case STARTSWITH: + if (value == null || !value.startsWith(criterion.value)) { + return false; + } + break; + case CONTAINS: + if (value == null || !value.contains(criterion.value)) { + return false; + } + break; + default: + // unexpected operator + return false; + } + } + return true; + } + + // TODO (not for hackathon) expand this to work with non-String attributes + // a bit tricky since Attributes#get(AttributeKey) requires a known type + // (without iterating over all attributes) + public static class Criterion { + private AttributeKey field; + private String value; + private Operator operator; + + // visible for testing + AttributeKey getField() { + return field; + } + + // visible for testing + String getValue() { + return value; + } + + // visible for testing + Operator getOperator() { + return operator; + } + + public static Criterion fromJson(JsonReader jsonReader) throws IOException { + return jsonReader.readObject( + (reader) -> { + Criterion deserializedValue = new Criterion(); + + while (reader.nextToken() != JsonToken.END_OBJECT) { + String fieldName = reader.getFieldName(); + reader.nextToken(); + if ("Field".equals(fieldName)) { + deserializedValue.field = AttributeKey.stringKey(reader.getString()); + } else if ("Operator".equals(fieldName)) { + deserializedValue.operator = Operator.from(reader.getString()); + } else if ("Value".equals(fieldName)) { + deserializedValue.value = reader.getString(); + } else { + reader.skipChildren(); + } + } + + return deserializedValue; + }); + } + } + + public enum Operator { + EQUALS, + STARTSWITH, + CONTAINS; + + private static Operator from(String value) { + switch (value) { + case "==": + return EQUALS; + case "startswith": + return STARTSWITH; + case "contains": + return CONTAINS; + default: + throw new IllegalStateException("Unexpected operator: " + value); + } + } + } +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionConfigSupplier.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionConfigSupplier.java new file mode 100644 index 00000000000..25cb0886ea0 --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionConfigSupplier.java @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.keytransaction; + +import static java.util.Collections.emptyList; +import static java.util.stream.Collectors.toList; + +import com.azure.json.JsonProviders; +import java.io.IOException; +import java.util.List; +import java.util.function.Supplier; + +public class KeyTransactionConfigSupplier implements Supplier> { + + public static final boolean KEY_TRANSACTIONS_ENABLED = true; + public static final boolean USE_HARDCODED_CONFIG = false; + + // TODO remove reliance on global + private static final KeyTransactionConfigSupplier instance = new KeyTransactionConfigSupplier(); + + static { + if (USE_HARDCODED_CONFIG) { + instance.set(hardcodedDemo()); + } + } + + public static KeyTransactionConfigSupplier getInstance() { + return instance; + } + + private volatile List configs = emptyList(); + + private KeyTransactionConfigSupplier() {} + + @Override + public List get() { + return configs; + } + + public void set(List configs) { + this.configs = configs; + } + + private static List hardcodedDemo() { + List configs; + try { + configs = + NewResponse.fromJson( + JsonProviders.createReader( + "{\n" + + " \"itemsReceived\": 13,\n" + + " \"itemsAccepted\": 13,\n" + + " \"appId\": null,\n" + + " \"errors\": [],\n" + + " \"sdkConfiguration\": [\n" + + " {\n" + + " \"Key\": \"Transaction\",\n" + + " \"Value\": {\n" + + " \"Name\": \"EarthOrbit\",\n" + + " \"StartCriteria\": [\n" + + " {\n" + + " \"Field\": \"url.path\",\n" + + " \"Operator\": \"==\",\n" + + " \"Value\": \"/earth\"\n" + + " }\n" + + " ],\n" + + " \"EndCriteria\": []\n" + + " }\n" + + " },\n" + + " {\n" + + " \"Key\": \"Transaction\",\n" + + " \"Value\": {\n" + + " \"Name\": \"MarsMission\",\n" + + " \"StartCriteria\": [\n" + + " {\n" + + " \"Field\": \"url.path\",\n" + + " \"Operator\": \"==\",\n" + + " \"Value\": \"/mars\"\n" + + " }\n" + + " ],\n" + + " \"EndCriteria\": [\n" + + " {\n" + + " \"Field\": \"messaging.operation\",\n" + + " \"Operator\": \"==\",\n" + + " \"Value\": \"process\"\n" + + " },\n" + + " {\n" + + " \"Field\": \"messaging.destination.name\",\n" + + " \"Operator\": \"==\",\n" + + " \"Value\": \"space\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + " ]\n" + + "}\n")) + .getSdkConfigurations() + .stream() + .map(SdkConfiguration::getValue) + .collect(toList()); + } catch (IOException e) { + throw new IllegalStateException(e); + } + return configs; + } +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionSampler.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionSampler.java new file mode 100644 index 00000000000..d9e67db3211 --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionSampler.java @@ -0,0 +1,180 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.keytransaction; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.trace.data.LinkData; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import io.opentelemetry.sdk.trace.samplers.SamplingDecision; +import io.opentelemetry.sdk.trace.samplers.SamplingResult; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +public final class KeyTransactionSampler implements Sampler { + + private final Supplier> configs; + private final Sampler fallback; + + private KeyTransactionSampler(Supplier> configs, Sampler fallback) { + this.configs = configs; + this.fallback = fallback; + } + + public static KeyTransactionSampler create( + Supplier> configs, Sampler fallback) { + return new KeyTransactionSampler(configs, fallback); + } + + @Override + public SamplingResult shouldSample( + Context parentContext, + String traceId, + String name, + SpanKind spanKind, + Attributes attributes, + List parentLinks) { + + SpanContext spanContext = Span.fromContext(parentContext).getSpanContext(); + if (spanContext.isValid() && !spanContext.isRemote()) { + // for now only applying to local root spans + return fallback.shouldSample(parentContext, traceId, name, spanKind, attributes, parentLinks); + } + + List configs = this.configs.get(); + + Set existingKeyTransactions = + KeyTransactionTraceState.getKeyTransactionStartTimes(spanContext.getTraceState()).keySet(); + + List startKeyTransactions = new ArrayList<>(); + List endKeyTransactions = new ArrayList<>(); + for (KeyTransactionConfig config : configs) { + + if (existingKeyTransactions.contains(config.getName())) { + // consider ending it + if (!config.getEndCriteria().isEmpty() + && KeyTransactionConfig.matches(attributes, config.getEndCriteria())) { + endKeyTransactions.add(config.getName()); + } + } else { + // consider starting it + if (KeyTransactionConfig.matches(attributes, config.getStartCriteria())) { + startKeyTransactions.add(config.getName()); + // consider ending it right away + if (config.getEndCriteria().isEmpty() + || KeyTransactionConfig.matches(attributes, config.getEndCriteria())) { + endKeyTransactions.add(config.getName()); + } + } + } + } + + // always delegate to fallback sampler to give it a chance to also modify trace state + // or capture additional attributes + SamplingResult result = + fallback.shouldSample(parentContext, traceId, name, spanKind, attributes, parentLinks); + + if (existingKeyTransactions.isEmpty() && startKeyTransactions.isEmpty()) { + return result; + } + return new TransactionSamplingResult( + existingKeyTransactions, startKeyTransactions, endKeyTransactions, result); + } + + @Override + public String getDescription() { + return String.format("TransactionSampler{root:%s}", fallback.getDescription()); + } + + @Override + public String toString() { + return getDescription(); + } + + private static class TransactionSamplingResult implements SamplingResult { + + private final Collection existingKeyTransactions; + private final Collection startKeyTransactions; + private final Collection endKeyTransactions; + private final SamplingResult delegate; + + private TransactionSamplingResult( + Collection existingKeyTransactions, + Collection startKeyTransactions, + Collection endKeyTransactions, + SamplingResult delegate) { + + this.existingKeyTransactions = existingKeyTransactions; + this.startKeyTransactions = startKeyTransactions; + this.endKeyTransactions = endKeyTransactions; + this.delegate = delegate; + } + + @Override + public SamplingDecision getDecision() { + // always capture 100% of key transaction spans + return SamplingDecision.RECORD_AND_SAMPLE; + } + + @Override + public Attributes getAttributes() { + AttributesBuilder builder = delegate.getAttributes().toBuilder(); + + for (String startKeyTransaction : startKeyTransactions) { + builder.put("key_transaction." + startKeyTransaction, true); + builder.put("key_transaction.started." + startKeyTransaction, true); + } + + for (String existingKeyTransaction : existingKeyTransactions) { + builder.put("key_transaction." + existingKeyTransaction, true); + } + + for (String existingKeyTransaction : endKeyTransactions) { + builder.put("key_transaction.ended." + existingKeyTransaction, true); + } + + return builder.build(); + } + + @Override + public TraceState getUpdatedTraceState(TraceState parentTraceState) { + // TODO can we remove ended key transactions from trace state? + // maybe not, since the "end" span could itself still have downstream synchronous flows + // that need to be stamped and will complete before the "end" span itself completes + + TraceState updatedTraceState = delegate.getUpdatedTraceState(parentTraceState); + + if (startKeyTransactions.isEmpty()) { + return updatedTraceState; + } + + // may not match span start time exactly + long startTime = System.currentTimeMillis(); + + String newValue = + startKeyTransactions.stream() + .map(name -> name + ":" + startTime) + .collect(Collectors.joining(";")); + + String existingValue = updatedTraceState.get(KeyTransactionTraceState.TRACE_STATE_KEY); + + if (existingValue != null && !existingValue.isEmpty()) { + newValue = existingValue + ";" + newValue; + } + + return updatedTraceState.toBuilder() + .put(KeyTransactionTraceState.TRACE_STATE_KEY, newValue) + .build(); + } + } +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionSpanProcessor.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionSpanProcessor.java new file mode 100644 index 00000000000..fc8fb459bf0 --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionSpanProcessor.java @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.keytransaction; + +import static io.opentelemetry.api.common.AttributeType.BOOLEAN; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.MeterProvider; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.trace.ReadWriteSpan; +import io.opentelemetry.sdk.trace.ReadableSpan; +import io.opentelemetry.sdk.trace.SpanProcessor; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class KeyTransactionSpanProcessor implements SpanProcessor { + + // TODO remove global state + private static volatile DoubleHistogram keyTransactionHistogram; + + public static void initMeterProvider(MeterProvider meterProvider) { + keyTransactionHistogram = + meterProvider + .get("keytransactions") + .histogramBuilder("key_transaction.duration") + .setDescription("Key transaction duration") + .setUnit("s") + .build(); + } + + @Override + @SuppressWarnings("unchecked") + public void onStart(Context context, ReadWriteSpan readWriteSpan) { + // copy key_transaction. attributes down to child spans + Span parentSpan = Span.fromContext(context); + if (parentSpan instanceof ReadableSpan) { + ((ReadableSpan) parentSpan) + .getAttributes() + .forEach( + (key, value) -> { + if (key.getKey().startsWith("key_transaction.") + && !key.getKey().startsWith("key_transaction.started.") + && !key.getKey().startsWith("key_transaction.ended.") + && key.getType() == BOOLEAN + && value instanceof Boolean) { + readWriteSpan.setAttribute((AttributeKey) key, (Boolean) value); + } + }); + } + } + + @Override + public boolean isStartRequired() { + return true; + } + + @Override + public void onEnd(ReadableSpan readableSpan) { + + if (keyTransactionHistogram == null) { + return; + } + + SpanContext parentSpanContext = readableSpan.getParentSpanContext(); + if (parentSpanContext.isValid() && !parentSpanContext.isRemote()) { + // only generating metrics for local root spans + return; + } + + List keyTransactionNames = new ArrayList<>(); + List startedKeyTransactions = new ArrayList<>(); + List endedKeyTransactions = new ArrayList<>(); + readableSpan + .getAttributes() + .forEach( + (attributeKey, value) -> { + String key = attributeKey.getKey(); + if (key.startsWith("key_transaction.started.")) { + startedKeyTransactions.add(key.substring("key_transaction.started.".length())); + } else if (key.startsWith("key_transaction.ended.")) { + endedKeyTransactions.add(key.substring("key_transaction.ended.".length())); + } else if (key.startsWith("key_transaction.")) { + keyTransactionNames.add(key.substring("key_transaction.".length())); + } + }); + + if (keyTransactionNames.isEmpty()) { + return; + } + + Map keyTransactionStartTimes = + KeyTransactionTraceState.getKeyTransactionStartTimes( + readableSpan.getSpanContext().getTraceState()); + + long endTime = System.currentTimeMillis(); + + for (String keyTransactionName : keyTransactionNames) { + long startTime = keyTransactionStartTimes.get(keyTransactionName); + + double duration = (endTime - startTime) / 1000.0; + + AttributesBuilder attributes = + Attributes.builder().put("key_transaction", keyTransactionName); + + if (startedKeyTransactions.contains(keyTransactionName)) { + attributes.put("key_transaction.started", true); + } + if (endedKeyTransactions.contains(keyTransactionName)) { + attributes.put("key_transaction.ended", true); + } + + keyTransactionHistogram.record(duration, attributes.build()); + } + } + + @Override + public boolean isEndRequired() { + return true; + } +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionTelemetryPipelineListener.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionTelemetryPipelineListener.java new file mode 100644 index 00000000000..001d07faf7e --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionTelemetryPipelineListener.java @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.keytransaction; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.stream.Collectors.toList; + +import com.azure.json.JsonProviders; +import com.azure.json.JsonReader; +import com.azure.monitor.opentelemetry.exporter.implementation.localstorage.LocalStorageTelemetryPipelineListener; +import com.azure.monitor.opentelemetry.exporter.implementation.logging.OperationLogger; +import com.azure.monitor.opentelemetry.exporter.implementation.pipeline.TelemetryItemExporter; +import com.azure.monitor.opentelemetry.exporter.implementation.pipeline.TelemetryPipelineListener; +import com.azure.monitor.opentelemetry.exporter.implementation.pipeline.TelemetryPipelineRequest; +import com.azure.monitor.opentelemetry.exporter.implementation.pipeline.TelemetryPipelineResponse; +import io.opentelemetry.sdk.common.CompletableResultCode; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class KeyTransactionTelemetryPipelineListener implements TelemetryPipelineListener { + + private static final Logger logger = + LoggerFactory.getLogger(KeyTransactionTelemetryPipelineListener.class); + + private static final OperationLogger operationLogger = + new OperationLogger(TelemetryItemExporter.class, "Parsing response from ingestion service"); + + @Override + public void onResponse(TelemetryPipelineRequest request, TelemetryPipelineResponse response) { + + if (logger.isDebugEnabled()) { + logger.debug("request: {}", requestToString(request)); + logger.debug("response: {}", response.getBody()); + } + + NewResponse parsedResponse; + try (JsonReader reader = JsonProviders.createReader(response.getBody())) { + parsedResponse = NewResponse.fromJson(reader); + operationLogger.recordSuccess(); + } catch (IOException e) { + operationLogger.recordFailure(e.getMessage(), e); + return; + } + + if (parsedResponse.getSdkConfigurations() != null + && !KeyTransactionConfigSupplier.USE_HARDCODED_CONFIG) { + KeyTransactionConfigSupplier.getInstance() + .set( + parsedResponse.getSdkConfigurations().stream() + .map(SdkConfiguration::getValue) + .collect(toList())); + } + } + + @Override + public void onException( + TelemetryPipelineRequest telemetryPipelineRequest, String s, Throwable throwable) { + // ignore + } + + @Override + public CompletableResultCode shutdown() { + return CompletableResultCode.ofSuccess(); + } + + private static String requestToString(TelemetryPipelineRequest request) { + List originalByteBuffers = request.getByteBuffers(); + byte[] gzippedBytes = convertByteBufferListToByteArray(originalByteBuffers); + byte[] ungzippedBytes = LocalStorageTelemetryPipelineListener.ungzip(gzippedBytes); + return new String(ungzippedBytes, UTF_8); + } + + // convert a list of byte buffers to a big byte array + private static byte[] convertByteBufferListToByteArray(List byteBuffers) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + for (ByteBuffer buffer : byteBuffers) { + byte[] arr = new byte[buffer.remaining()]; + buffer.get(arr); + try { + baos.write(arr); + } catch (IOException e) { + // this should never happen since ByteArrayOutputStream doesn't throw IOException + throw new IllegalStateException(e); + } + } + + return baos.toByteArray(); + } +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionTraceState.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionTraceState.java new file mode 100644 index 00000000000..8204032680d --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionTraceState.java @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.keytransaction; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; + +import io.opentelemetry.api.trace.TraceState; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +class KeyTransactionTraceState { + + // NOTE: dots are not valid in trace state keys, so we use underscores + // example: "microsoft_kt=mykeytransaction:starttimemillis;mykeytransaction2:starttimemillis" + static final String TRACE_STATE_KEY = "microsoft_kt"; + + static Set getKeyTransactionNames(TraceState traceState) { + return getKeyTransactionNames(traceState.get(TRACE_STATE_KEY)); + } + + // visible for testing + @SuppressWarnings("MixedMutabilityReturnType") + static Set getKeyTransactionNames(String value) { + if (value == null) { + return emptySet(); + } + + Set names = new HashSet<>(); + for (String part : value.split(";")) { + int index = part.lastIndexOf(':'); + if (index == -1) { + // invalid format, ignore + continue; + } + names.add(part.substring(0, index)); + } + + return names; + } + + static Map getKeyTransactionStartTimes(TraceState traceState) { + return getKeyTransactionStartTimes(traceState.get(TRACE_STATE_KEY)); + } + + // visible for testing + @SuppressWarnings("MixedMutabilityReturnType") + static Map getKeyTransactionStartTimes(String value) { + if (value == null) { + return emptyMap(); + } + + Map names = new HashMap<>(); + for (String part : value.split(";")) { + int index = part.lastIndexOf(':'); + if (index == -1) { + // invalid format, ignore + continue; + } + String key = part.substring(0, index); + long val; + try { + val = Long.parseLong(part.substring(index + 1)); + } catch (NumberFormatException e) { + // invalid format, ignore + continue; + } + names.put(key, val); + } + + return names; + } + + private KeyTransactionTraceState() {} +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/NewResponse.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/NewResponse.java new file mode 100644 index 00000000000..1512aeda96c --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/NewResponse.java @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.keytransaction; + +import com.azure.json.JsonReader; +import com.azure.json.JsonToken; +import java.io.IOException; +import java.util.List; + +class NewResponse { + + private List sdkConfigurations; + + public List getSdkConfigurations() { + return sdkConfigurations; + } + + static NewResponse fromJson(JsonReader jsonReader) throws IOException { + return jsonReader.readObject( + (reader) -> { + NewResponse deserializedValue = new NewResponse(); + + while (reader.nextToken() != JsonToken.END_OBJECT) { + String fieldName = reader.getFieldName(); + reader.nextToken(); + if ("sdkConfiguration".equals(fieldName)) { + deserializedValue.sdkConfigurations = reader.readArray(SdkConfiguration::fromJson); + } else { + reader.skipChildren(); + } + } + + return deserializedValue; + }); + } +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/SdkConfiguration.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/SdkConfiguration.java new file mode 100644 index 00000000000..62e3fba02ab --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/keytransaction/SdkConfiguration.java @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.keytransaction; + +import com.azure.json.JsonReader; +import com.azure.json.JsonToken; +import java.io.IOException; + +public class SdkConfiguration { + + private String key; + private KeyTransactionConfig value; // for hackathon only supporting one type of value + + public String getKey() { + return key; + } + + public KeyTransactionConfig getValue() { + return value; + } + + static SdkConfiguration fromJson(JsonReader jsonReader) throws IOException { + return jsonReader.readObject( + (reader) -> { + SdkConfiguration deserializedValue = new SdkConfiguration(); + + while (reader.nextToken() != JsonToken.END_OBJECT) { + String fieldName = reader.getFieldName(); + reader.nextToken(); + if ("Key".equals(fieldName)) { + deserializedValue.key = reader.getString(); + } else if ("Value".equals(fieldName)) { + deserializedValue.value = KeyTransactionConfig.fromJson(reader); + } else { + reader.skipChildren(); + } + } + + return deserializedValue; + }); + } +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/Samplers.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/Samplers.java index 8f6dfaecd42..a55385e961f 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/Samplers.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/Samplers.java @@ -5,6 +5,8 @@ import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.SamplingOverride; +import com.microsoft.applicationinsights.agent.internal.keytransaction.KeyTransactionConfigSupplier; +import com.microsoft.applicationinsights.agent.internal.keytransaction.KeyTransactionSampler; import io.opentelemetry.sdk.trace.samplers.Sampler; import java.util.List; import java.util.stream.Collectors; @@ -46,16 +48,21 @@ public static Sampler getSampler( new AiOverrideSampler(requestSamplingOverrides, dependencySamplingOverrides, sampler); } - if (!samplingPreview.parentBased) { - return sampler; + if (samplingPreview.parentBased) { + // when using parent-based sampling, sampling overrides still take precedence + + // IMPORTANT, the parent-based sampler is useful for interop with other sampling mechanisms, + // as + // it will ensure consistent traces, however it does not accurately compute item counts, since + // item counts are not propagated in trace state (yet) + sampler = Sampler.parentBasedBuilder(sampler).build(); } - // when using parent-based sampling, sampling overrides still take precedence + if (KeyTransactionConfigSupplier.KEY_TRANSACTIONS_ENABLED) { + sampler = KeyTransactionSampler.create(KeyTransactionConfigSupplier.getInstance(), sampler); + } - // IMPORTANT, the parent-based sampler is useful for interop with other sampling mechanisms, as - // it will ensure consistent traces, however it does not accurately compute item counts, since - // item counts are not propagated in trace state (yet) - return Sampler.parentBasedBuilder(sampler).build(); + return sampler; } private Samplers() {} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryClient.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryClient.java index f719e8ee12f..7126228d8e1 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryClient.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryClient.java @@ -34,6 +34,8 @@ import com.azure.monitor.opentelemetry.exporter.implementation.utils.TempDirs; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; import com.microsoft.applicationinsights.agent.internal.httpclient.LazyHttpClient; +import com.microsoft.applicationinsights.agent.internal.keytransaction.KeyTransactionConfigSupplier; +import com.microsoft.applicationinsights.agent.internal.keytransaction.KeyTransactionTelemetryPipelineListener; import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.resources.Resource; import java.io.File; @@ -254,6 +256,12 @@ private BatchItemProcessor initBatchItemProcessor( false)); } + if (KeyTransactionConfigSupplier.KEY_TRANSACTIONS_ENABLED) { + telemetryPipelineListener = + TelemetryPipelineListener.composite( + telemetryPipelineListener, new KeyTransactionTelemetryPipelineListener()); + } + return BatchItemProcessor.builder( new TelemetryItemExporter(telemetryPipeline, telemetryPipelineListener)) .setMaxQueueSize(exportQueueCapacity) diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionConfigTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionConfigTest.java new file mode 100644 index 00000000000..0e31688a080 --- /dev/null +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionConfigTest.java @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.keytransaction; + +import static com.microsoft.applicationinsights.agent.internal.keytransaction.KeyTransactionConfig.Operator.EQUALS; +import static org.assertj.core.api.Assertions.assertThat; + +import com.azure.json.JsonProviders; +import java.io.InputStream; +import org.junit.jupiter.api.Test; + +class KeyTransactionConfigTest { + + @Test + void test() throws Exception { + InputStream in = KeyTransactionConfigTest.class.getResourceAsStream("demo.json"); + NewResponse parsedResponse = NewResponse.fromJson(JsonProviders.createReader(in)); + + assertThat(parsedResponse.getSdkConfigurations()).hasSize(2); + + SdkConfiguration earthOrbitSdkConfiguration = parsedResponse.getSdkConfigurations().get(0); + assertThat(earthOrbitSdkConfiguration.getKey()).isEqualTo("Transaction"); + + KeyTransactionConfig earthOrbitKeyTransactionConfig = earthOrbitSdkConfiguration.getValue(); + assertThat(earthOrbitKeyTransactionConfig.getName()).isEqualTo("EarthOrbit"); + + assertThat(earthOrbitKeyTransactionConfig.getStartCriteria()).hasSize(1); + KeyTransactionConfig.Criterion earthOrbitStartCriteria = + earthOrbitKeyTransactionConfig.getStartCriteria().get(0); + assertThat(earthOrbitStartCriteria.getField().getKey()).isEqualTo("url.path"); + assertThat(earthOrbitStartCriteria.getOperator()).isEqualTo(EQUALS); + assertThat(earthOrbitStartCriteria.getValue()).isEqualTo("/earth"); + + assertThat(earthOrbitKeyTransactionConfig.getEndCriteria()).isEmpty(); + + SdkConfiguration marsMissionSdkConfiguration = parsedResponse.getSdkConfigurations().get(1); + assertThat(marsMissionSdkConfiguration.getKey()).isEqualTo("Transaction"); + + KeyTransactionConfig marsMissionKeyTransactionConfig = marsMissionSdkConfiguration.getValue(); + assertThat(marsMissionKeyTransactionConfig.getName()).isEqualTo("MarsMission"); + + assertThat(marsMissionKeyTransactionConfig.getStartCriteria()).hasSize(1); + KeyTransactionConfig.Criterion marsMissionStartCriteria = + marsMissionKeyTransactionConfig.getStartCriteria().get(0); + assertThat(marsMissionStartCriteria.getField().getKey()).isEqualTo("url.path"); + assertThat(marsMissionStartCriteria.getOperator()).isEqualTo(EQUALS); + assertThat(marsMissionStartCriteria.getValue()).isEqualTo("/mars"); + + assertThat(marsMissionKeyTransactionConfig.getEndCriteria()).hasSize(2); + + KeyTransactionConfig.Criterion marsMissionEndCriteria1 = + marsMissionKeyTransactionConfig.getEndCriteria().get(0); + assertThat(marsMissionEndCriteria1.getField().getKey()).isEqualTo("messaging.operation"); + assertThat(marsMissionEndCriteria1.getOperator()).isEqualTo(EQUALS); + assertThat(marsMissionEndCriteria1.getValue()).isEqualTo("process"); + + KeyTransactionConfig.Criterion marsMissionEndCriteria2 = + marsMissionKeyTransactionConfig.getEndCriteria().get(1); + assertThat(marsMissionEndCriteria2.getField().getKey()).isEqualTo("messaging.destination.name"); + assertThat(marsMissionEndCriteria2.getOperator()).isEqualTo(EQUALS); + assertThat(marsMissionEndCriteria2.getValue()).isEqualTo("space"); + } +} diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionTraceStateTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionTraceStateTest.java new file mode 100644 index 00000000000..f5a6ad826e4 --- /dev/null +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/keytransaction/KeyTransactionTraceStateTest.java @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.keytransaction; + +import static com.microsoft.applicationinsights.agent.internal.keytransaction.KeyTransactionTraceState.getKeyTransactionStartTimes; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Map; +import org.junit.jupiter.api.Test; + +class KeyTransactionTraceStateTest { + + @Test + void testNull() { + assertThat(getKeyTransactionStartTimes((String) null)).isEmpty(); + } + + @Test + void testEmpty() { + assertThat(getKeyTransactionStartTimes("")).isEmpty(); + } + + @Test + void testSingle() { + Map startTimes = getKeyTransactionStartTimes("abc:123"); + assertThat(startTimes).containsOnlyKeys("abc"); + assertThat(startTimes.get("abc")).isEqualTo(123L); + } + + @Test + void testMultiple() { + Map startTimes = getKeyTransactionStartTimes("abc:123;qrs:456;xyz:789"); + assertThat(startTimes).containsOnlyKeys("abc", "qrs", "xyz"); + assertThat(startTimes.get("abc")).isEqualTo(123L); + assertThat(startTimes.get("qrs")).isEqualTo(456L); + assertThat(startTimes.get("xyz")).isEqualTo(789L); + } +} diff --git a/agent/agent-tooling/src/test/resources/com/microsoft/applicationinsights/agent/internal/keytransaction/demo.json b/agent/agent-tooling/src/test/resources/com/microsoft/applicationinsights/agent/internal/keytransaction/demo.json new file mode 100644 index 00000000000..a5db272f97a --- /dev/null +++ b/agent/agent-tooling/src/test/resources/com/microsoft/applicationinsights/agent/internal/keytransaction/demo.json @@ -0,0 +1,47 @@ +{ + "itemsReceived": 13, + "itemsAccepted": 13, + "appId": null, + "errors": [], + "sdkConfiguration": [ + { + "Key": "Transaction", + "Value": { + "Name": "EarthOrbit", + "StartCriteria": [ + { + "Field": "url.path", + "Operator": "==", + "Value": "/earth" + } + ], + "EndCriteria": [] + } + }, + { + "Key": "Transaction", + "Value": { + "Name": "MarsMission", + "StartCriteria": [ + { + "Field": "url.path", + "Operator": "==", + "Value": "/mars" + } + ], + "EndCriteria": [ + { + "Field": "messaging.operation", + "Operator": "==", + "Value": "process" + }, + { + "Field": "messaging.destination.name", + "Operator": "==", + "Value": "space" + } + ] + } + } + ] +} diff --git a/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/fakeingestion/MockedAppInsightsIngestionServlet.java b/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/fakeingestion/MockedAppInsightsIngestionServlet.java index 6fdbb3d6a0b..5f0b6f95467 100644 --- a/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/fakeingestion/MockedAppInsightsIngestionServlet.java +++ b/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/fakeingestion/MockedAppInsightsIngestionServlet.java @@ -138,6 +138,8 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws I type2envelope.put(baseType, envelope); } } + + resp.getWriter().print("{}"); } @Override