From 41acb04333847cdb32fd048a14326bde55b85f6f Mon Sep 17 00:00:00 2001 From: JophieQu Date: Thu, 18 Sep 2025 11:11:17 +0800 Subject: [PATCH 01/69] feat: support pprof feature --- .../command/CommandDeserializer.java | 2 + .../component/command/PprofTaskCommand.java | 82 ++++++ oap-server/server-core/pom.xml | 5 + .../oap/server/core/CoreModule.java | 10 + .../oap/server/core/CoreModuleProvider.java | 9 + .../server/core/cache/CacheUpdateTimer.java | 48 ++++ .../oap/server/core/cache/PprofTaskCache.java | 93 +++++++ .../server/core/command/CommandService.java | 15 ++ .../profiling/pprof/PprofMutationService.java | 147 +++++++++++ .../profiling/pprof/PprofQueryService.java | 122 +++++++++ .../storage/PprofProfilingDataDispatcher.java | 41 +++ .../storage/PprofProfilingDataRecord.java | 105 ++++++++ .../pprof/storage/PprofTaskLogRecord.java | 99 ++++++++ .../pprof/storage/PprofTaskRecord.java | 116 +++++++++ .../oap/server/core/query/PprofTaskLog.java | 44 ++++ .../query/input/PprofAnalyzationRequest.java | 31 +++ .../query/input/PprofTaskCreationRequest.java | 35 +++ .../query/input/PprofTaskListRequest.java | 12 + .../core/query/type/PprofAnalyzation.java | 28 +++ .../core/query/type/PprofEventType.java | 42 ++++ .../core/query/type/PprofStackElement.java | 38 +++ .../core/query/type/PprofStackTree.java | 62 +++++ .../oap/server/core/query/type/PprofTask.java | 44 ++++ .../query/type/PprofTaskCreationResult.java | 24 ++ .../query/type/PprofTaskCreationType.java | 8 + .../core/query/type/PprofTaskListResult.java | 32 +++ .../query/type/PprofTaskLogOperationType.java | 54 ++++ .../core/query/type/PprofTaskProgress.java | 31 +++ .../core/source/DefaultScopeDefine.java | 3 + .../core/source/PprofProfilingData.java | 50 ++++ .../server/core/storage/StorageModule.java | 6 + .../profiling/pprof/IPprofDataQueryDAO.java | 36 +++ .../pprof/IPprofTaskLogQueryDAO.java | 14 ++ .../profiling/pprof/IPprofTaskQueryDAO.java | 47 ++++ .../library-pprof-parser/pom.xml | 86 +++++++ .../pprof/parser/PprofMergeBuilder.java | 84 +++++++ .../library/pprof/parser/PprofParser.java | 53 ++++ .../server/library/pprof/type/FrameTree.java | 38 +++ .../library/pprof/type/FrameTreeBuilder.java | 102 ++++++++ .../src/main/proto/profile.proto | 233 ++++++++++++++++++ .../server/library/util/CollectionUtils.java | 4 + oap-server/server-library/pom.xml | 1 + .../query/graphql/GraphQLQueryProvider.java | 6 +- .../query/graphql/resolver/PprofMutation.java | 55 +++++ .../query/graphql/resolver/PprofQuery.java | 88 +++++++ oap-server/server-receiver-plugin/pom.xml | 1 + .../receiver-proto/pom.xml | 8 + .../skywalking-pprof-receiver-plugin/pom.xml | 53 ++++ .../receiver/pprof/module/PprofModule.java | 34 +++ .../pprof/module/PprofModuleConfig.java | 46 ++++ .../pprof/provider/PprofModuleProvider.java | 88 +++++++ .../provider/handler/PprofServiceHandler.java | 130 ++++++++++ .../PprofByteBufCollectionObserver.java | 158 ++++++++++++ .../stream/PprofCollectionMetaData.java | 36 +++ .../stream/PprofFileCollectionObserver.java | 170 +++++++++++++ ...ing.oap.server.library.module.ModuleDefine | 19 ++ ...g.oap.server.library.module.ModuleProvider | 19 ++ .../skywalking-sharing-server-plugin/pom.xml | 8 + oap-server/server-starter/pom.xml | 5 + .../src/main/resources/application.yml | 16 +- .../banyandb/BanyanDBStorageConfig.java | 2 + .../banyandb/BanyanDBStorageProvider.java | 17 ++ .../stream/BanyanDBPprofDataQueryDAO.java | 59 +++++ .../stream/BanyanDBPprofTaskLogQueryDAO.java | 82 ++++++ .../stream/BanyanDBPprofTaskQueryDAO.java | 147 +++++++++++ .../StorageModuleElasticsearchProvider.java | 12 + .../query/PprofTaskLogQueryEsDAO.java | 84 +++++++ .../query/PprofTaskQueryEsDAO.java | 147 +++++++++++ 68 files changed, 3624 insertions(+), 2 deletions(-) create mode 100644 apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofQueryService.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PprofTaskLog.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofAnalyzationRequest.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskCreationRequest.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofAnalyzation.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofEventType.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackElement.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackTree.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTask.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskListResult.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskLogOperationType.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskProgress.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/PprofProfilingData.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofDataQueryDAO.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskQueryDAO.java create mode 100755 oap-server/server-library/library-pprof-parser/pom.xml create mode 100644 oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java create mode 100644 oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java create mode 100755 oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java create mode 100644 oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java create mode 100755 oap-server/server-library/library-pprof-parser/src/main/proto/profile.proto create mode 100644 oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofMutation.java create mode 100644 oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofQuery.java create mode 100644 oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/pom.xml create mode 100644 oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModule.java create mode 100644 oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java create mode 100644 oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/PprofModuleProvider.java create mode 100644 oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java create mode 100644 oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java create mode 100644 oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofCollectionMetaData.java create mode 100644 oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java create mode 100644 oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine create mode 100644 oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider create mode 100644 oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java create mode 100644 oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskLogQueryDAO.java create mode 100644 oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java create mode 100644 oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java create mode 100644 oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java diff --git a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/CommandDeserializer.java b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/CommandDeserializer.java index 6323e6906a12..4bd94b8bc0e0 100644 --- a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/CommandDeserializer.java +++ b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/CommandDeserializer.java @@ -29,6 +29,8 @@ public static BaseCommand deserialize(final Command command) { return ConfigurationDiscoveryCommand.DESERIALIZER.deserialize(command); } else if (AsyncProfilerTaskCommand.NAME.equals(commandName)) { return AsyncProfilerTaskCommand.DESERIALIZER.deserialize(command); + } else if (PprofTaskCommand.NAME.equals(commandName)) { + return PprofTaskCommand.DESERIALIZER.deserialize(command); } throw new UnsupportedCommandException(command); } diff --git a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java new file mode 100644 index 000000000000..3cbf9b5f4750 --- /dev/null +++ b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java @@ -0,0 +1,82 @@ +package org.apache.skywalking.oap.server.network.trace.component.command; + +import org.apache.skywalking.apm.network.common.v3.Command; +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import java.util.List; +import lombok.Getter; + +@Getter +public class PprofTaskCommand extends BaseCommand implements Serializable, Deserializable { + public static final Deserializable DESERIALIZER = new PprofTaskCommand("", "", "", 0, 0, 0); + public static final String NAME = "PprofTaskQuery"; + /** + * pprof taskId + */ + private String taskId; + // Type of profiling (CPU/Heap/Block/Mutex/Goroutine/Threadcreate/Allocs) + private String events; + // unit is minute + private long duration; + // Unix timestamp in milliseconds when the task was created + private long createTime; + // + private int dumpPeriod; + + public PprofTaskCommand(String serialNumber, String taskId, String events, + long duration, long createTime, int dumpPeriod) { + super(NAME, serialNumber); + this.taskId = taskId; + this.duration = duration; + this.createTime = createTime; + this.dumpPeriod = dumpPeriod; + this.events = events; + } + + // public PprofTaskCommand(String serialNumber, String taskId, + // long duration, long startTime, long createTime, int dumpPeriod) { + // super(NAME, serialNumber); + // this.taskId = taskId; + // this.duration = duration; + // this.startTime = startTime; + // this.createTime = createTime; + // this.dumpPeriod = dumpPeriod; + // } + + @Override + public PprofTaskCommand deserialize(Command command) { + final List argsList = command.getArgsList(); + String taskId = null; + String events = null; + long duration = 0; + long createTime = 0; + int dumpPeriod = 0; + String serialNumber = null; + for (final KeyStringValuePair pair : argsList) { + if ("SerialNumber".equals(pair.getKey())) { + serialNumber = pair.getValue(); + } else if ("TaskId".equals(pair.getKey())) { + taskId = pair.getValue(); + } else if ("Events".equals(pair.getKey())) { + events = pair.getValue(); + } else if ("Duration".equals(pair.getKey())) { + duration = Long.parseLong(pair.getValue()); + } else if ("CreateTime".equals(pair.getKey())) { + createTime = Long.parseLong(pair.getValue()); + } else if ("DumpPeriod".equals(pair.getKey())) { + dumpPeriod = Integer.parseInt(pair.getValue()); + } + } + return new PprofTaskCommand(serialNumber, taskId, events, duration, createTime, dumpPeriod); + } + + @Override + public Command.Builder serialize() { + final Command.Builder builder = commandBuilder(); + builder.addArgs(KeyStringValuePair.newBuilder().setKey("TaskId").setValue(taskId)) + .addArgs(KeyStringValuePair.newBuilder().setKey("Events").setValue(events)) + .addArgs(KeyStringValuePair.newBuilder().setKey("Duration").setValue(String.valueOf(duration))) + .addArgs(KeyStringValuePair.newBuilder().setKey("CreateTime").setValue(String.valueOf(createTime))) + .addArgs(KeyStringValuePair.newBuilder().setKey("DumpPeriod").setValue(String.valueOf(dumpPeriod))); + return builder; + } +} \ No newline at end of file diff --git a/oap-server/server-core/pom.xml b/oap-server/server-core/pom.xml index 784da5435c03..f6d233587ad2 100644 --- a/oap-server/server-core/pom.xml +++ b/oap-server/server-core/pom.xml @@ -44,6 +44,11 @@ library-async-profiler-jfr-parser ${project.version} + + org.apache.skywalking + library-pprof-parser + ${project.version} + org.apache.skywalking telemetry-api diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java index 91509759eab3..b6d1734d2f43 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java @@ -22,6 +22,7 @@ import java.util.List; import org.apache.skywalking.oap.server.core.analysis.meter.MeterSystem; import org.apache.skywalking.oap.server.core.cache.AsyncProfilerTaskCache; +import org.apache.skywalking.oap.server.core.cache.PprofTaskCache; import org.apache.skywalking.oap.server.core.cache.NetworkAddressAliasCache; import org.apache.skywalking.oap.server.core.cache.ProfileTaskCache; import org.apache.skywalking.oap.server.core.command.CommandService; @@ -41,6 +42,8 @@ import org.apache.skywalking.oap.server.core.profiling.continuous.ContinuousProfilingQueryService; import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingMutationService; import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingQueryService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofMutationService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofQueryService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskMutationService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskQueryService; import org.apache.skywalking.oap.server.core.query.AggregationQueryService; @@ -106,6 +109,7 @@ public Class[] services() { addManagementService(classes); addEBPFProfilingService(classes); addAsyncProfilerService(classes); + addPprofService(classes); classes.add(CommandService.class); classes.add(HierarchyService.class); @@ -137,6 +141,12 @@ private void addAsyncProfilerService(List classes) { classes.add(AsyncProfilerTaskCache.class); } + private void addPprofService(List classes) { + classes.add(PprofMutationService.class); + classes.add(PprofQueryService.class); + classes.add(PprofTaskCache.class); + } + private void addOALService(List classes) { classes.add(OALEngineLoaderService.class); } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java index a9963f6545a2..c8d88d8fdf47 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java @@ -35,6 +35,7 @@ import org.apache.skywalking.oap.server.core.cache.AsyncProfilerTaskCache; import org.apache.skywalking.oap.server.core.cache.CacheUpdateTimer; import org.apache.skywalking.oap.server.core.cache.NetworkAddressAliasCache; +import org.apache.skywalking.oap.server.core.cache.PprofTaskCache; import org.apache.skywalking.oap.server.core.cache.ProfileTaskCache; import org.apache.skywalking.oap.server.core.cluster.ClusterCoordinator; import org.apache.skywalking.oap.server.core.cluster.ClusterModule; @@ -65,6 +66,8 @@ import org.apache.skywalking.oap.server.core.profiling.continuous.ContinuousProfilingQueryService; import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingMutationService; import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingQueryService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofMutationService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofQueryService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskMutationService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskQueryService; import org.apache.skywalking.oap.server.core.query.AggregationQueryService; @@ -331,6 +334,12 @@ TTLStatusQuery.class, new TTLStatusQuery( AsyncProfilerQueryService.class, new AsyncProfilerQueryService(getManager())); this.registerServiceImplementation( AsyncProfilerTaskCache.class, new AsyncProfilerTaskCache(getManager(), moduleConfig)); + this.registerServiceImplementation( + PprofMutationService.class, new PprofMutationService(getManager())); + this.registerServiceImplementation( + PprofQueryService.class, new PprofQueryService(getManager())); + this.registerServiceImplementation( + PprofTaskCache.class, new PprofTaskCache(getManager(), moduleConfig)); this.registerServiceImplementation( EBPFProfilingMutationService.class, new EBPFProfilingMutationService(getManager())); this.registerServiceImplementation( diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/CacheUpdateTimer.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/CacheUpdateTimer.java index c38f90daccd0..17f7cee530f7 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/CacheUpdateTimer.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/CacheUpdateTimer.java @@ -27,12 +27,15 @@ import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.storage.AsyncProfilerTaskRecord; import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerTask; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; import org.apache.skywalking.oap.server.library.util.CollectionUtils; import org.apache.skywalking.oap.server.library.util.RunnableWithExceptionProtection; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.analysis.DisableRegister; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.manual.networkalias.NetworkAddressAlias; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskRecord; import org.apache.skywalking.oap.server.core.query.type.ProfileTask; import org.apache.skywalking.oap.server.core.storage.StorageModule; @@ -46,6 +49,8 @@ public enum CacheUpdateTimer { private AsyncProfilerTaskCache asyncProfilerTaskCache; private IAsyncProfilerTaskQueryDAO asyncProfilerTaskQueryDAO; + private PprofTaskCache pprofTaskCache; + private IPprofTaskQueryDAO pprofTaskQueryDAO; private int ttl = 10; @@ -72,6 +77,10 @@ private void update(ModuleDefineHolder moduleDefineHolder) { if (!DisableRegister.INSTANCE.include(AsyncProfilerTaskRecord.INDEX_NAME)) { updateAsyncProfilerTask(moduleDefineHolder); } + + if (!DisableRegister.INSTANCE.include(PprofTaskRecord.INDEX_NAME)) { + updatePprofTask(moduleDefineHolder); + } } /** @@ -163,4 +172,43 @@ private void updateAsyncProfilerTask(ModuleDefineHolder moduleDefineHolder) { return; } + + private PprofTaskCache getPprofTaskCache(ModuleDefineHolder moduleDefineHolder) { + if (pprofTaskCache == null) { + pprofTaskCache = moduleDefineHolder.find(CoreModule.NAME) + .provider() + .getService(PprofTaskCache.class); + } + return pprofTaskCache; + } + + private IPprofTaskQueryDAO getPprofTaskQueryDAO(ModuleDefineHolder moduleDefineHolder) { + if (pprofTaskQueryDAO == null) { + pprofTaskQueryDAO = moduleDefineHolder.find(StorageModule.NAME) + .provider() + .getService(IPprofTaskQueryDAO.class); + } + return pprofTaskQueryDAO; + } + + private void updatePprofTask(ModuleDefineHolder moduleDefineHolder) { + PprofTaskCache taskCache = getPprofTaskCache(moduleDefineHolder); + IPprofTaskQueryDAO taskQueryDAO = getPprofTaskQueryDAO(moduleDefineHolder); + + try { + List taskList = taskQueryDAO.getTaskList( + null, taskCache.getCacheStartTimeBucket(), taskCache.getCacheEndTimeBucket(), null + ); + if (CollectionUtils.isEmpty(taskList)) { + return; + } + + for (PprofTask task : taskList) { + taskCache.saveTask(task.getServiceId(), task); + } + + } catch (IOException e) { + log.warn("Unable to update pprof task cache", e); + } + } } \ No newline at end of file diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java new file mode 100644 index 000000000000..9ce7e60defc2 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.cache; + +import org.apache.skywalking.oap.server.library.module.Service; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.apache.skywalking.oap.server.core.CoreModuleConfig; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +public class PprofTaskCache implements Service { + private static final Logger LOGGER = LoggerFactory.getLogger(PprofTaskCache.class); + + private final Cache serviceId2taskCache; + private final ModuleManager moduleManager; + + private IPprofTaskQueryDAO taskQueryDAO; + + public PprofTaskCache(ModuleManager moduleManager, CoreModuleConfig moduleConfig) { + this.moduleManager = moduleManager; + long initialSize = moduleConfig.getMaxSizeOfProfileTask() / 10L; + int initialCapacitySize = (int) (initialSize > Integer.MAX_VALUE ? Integer.MAX_VALUE : initialSize); + + serviceId2taskCache = CacheBuilder.newBuilder() + .initialCapacity(initialCapacitySize) + .maximumSize(moduleConfig.getMaxSizeOfProfileTask()) + // remove old profile task data - extend to 10 minutes to ensure data availability + .expireAfterWrite(Duration.ofMinutes(10)) + .build(); + } + + private IPprofTaskQueryDAO getTaskQueryDAO() { + if (Objects.isNull(taskQueryDAO)) { + taskQueryDAO = moduleManager.find(StorageModule.NAME) + .provider() + .getService(IPprofTaskQueryDAO.class); + } + return taskQueryDAO; + } + + public PprofTask getPprofTask(String serviceId) { + PprofTask task = serviceId2taskCache.getIfPresent(serviceId); + return task; + } + + public void saveTask(String serviceId, PprofTask task) { + if (task == null) { + return ; + } + + serviceId2taskCache.put(serviceId, task); + } + + /** + * use for every db query, -5min start time + */ + public long getCacheStartTimeBucket() { + return TimeBucket.getRecordTimeBucket(System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(5)); + } + + /** + * use for every db query, +5min end time(because search through task's create time) + */ + public long getCacheEndTimeBucket() { + return TimeBucket.getRecordTimeBucket(System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(5)); + } + +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/command/CommandService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/command/CommandService.java index ac7ef401769a..40a8de67e06c 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/command/CommandService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/command/CommandService.java @@ -33,6 +33,7 @@ import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerEventType; import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerTask; import org.apache.skywalking.oap.server.core.query.type.EBPFProfilingTaskExtension; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; import org.apache.skywalking.oap.server.library.util.CollectionUtils; import org.apache.skywalking.oap.server.library.util.StringUtil; import org.apache.skywalking.oap.server.network.trace.component.command.AsyncProfilerTaskCommand; @@ -40,6 +41,7 @@ import org.apache.skywalking.oap.server.network.trace.component.command.ContinuousProfilingPolicyCommand; import org.apache.skywalking.oap.server.network.trace.component.command.EBPFProfilingTaskCommand; import org.apache.skywalking.oap.server.network.trace.component.command.EBPFProfilingTaskExtensionConfig; +import org.apache.skywalking.oap.server.network.trace.component.command.PprofTaskCommand; import org.apache.skywalking.oap.server.network.trace.component.command.ProfileTaskCommand; import org.apache.skywalking.oap.server.core.query.type.ProfileTask; import org.apache.skywalking.oap.server.library.module.ModuleManager; @@ -72,6 +74,19 @@ public AsyncProfilerTaskCommand newAsyncProfileTaskCommand(AsyncProfilerTask tas eventNames, task.getExecArgs(), task.getCreateTime()); } + /** + * Create a new pprof task command for Go agents + */ + public PprofTaskCommand newPprofTaskCommand(PprofTask task) { + final String serialNumber = UUID.randomUUID().toString(); + String events = ""; + if (task.getEvents() != null) { + events = task.getEvents().getName(); + } + return new PprofTaskCommand(serialNumber, task.getId(), events, + task.getDuration(), task.getCreateTime(), task.getDumpPeriod()); + } + /** * Used to notify the eBPF Profiling task to the eBPF agent side */ diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java new file mode 100644 index 000000000000..c909790df570 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof; + +import lombok.RequiredArgsConstructor; +import org.apache.skywalking.oap.server.library.module.Service; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import java.io.IOException; +import java.util.List; +import org.apache.skywalking.oap.server.core.analysis.worker.NoneStreamProcessor; +import java.util.concurrent.TimeUnit; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskCreationResult; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskCreationType; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; + +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RequiredArgsConstructor +public class PprofMutationService implements Service { + private final ModuleManager moduleManager; + + private IPprofTaskQueryDAO taskQueryDAO; + + private IPprofTaskQueryDAO getPprofTaskDAO() { + if (taskQueryDAO == null) { + this.taskQueryDAO = moduleManager.find(StorageModule.NAME) + .provider() + .getService(IPprofTaskQueryDAO.class); + } + return taskQueryDAO; + } + + public PprofTaskCreationResult createTask(String serviceId, + List serviceInstanceIds, + int duration, + PprofEventType events, + int dumpPeriod) throws IOException { + long createTime = System.currentTimeMillis(); + // check data + PprofTaskCreationResult checkResult = checkDataSuccess( + serviceId, serviceInstanceIds, duration, createTime, events + ); + if (checkResult != null) { + return checkResult; + } + + // create task + PprofTaskRecord task = new PprofTaskRecord(); + String taskId = createTime + Const.ID_CONNECTOR + serviceId; + task.setTaskId(taskId); + task.setServiceId(serviceId); + task.setServiceInstanceIdsFromList(serviceInstanceIds); + task.setDuration(duration); + task.setEvents(events.toString()); + task.setDumpPeriod(dumpPeriod); + task.setCreateTime(createTime); + task.setTimeBucket(TimeBucket.getRecordTimeBucket(createTime)); + NoneStreamProcessor.getInstance().in(task); + return PprofTaskCreationResult.builder() + .id(task.id().build()) + .code(PprofTaskCreationType.SUCCESS) + .build(); + } + + private PprofTaskCreationResult checkDataSuccess(String serviceId, + List serviceInstanceIds, + int duration, + long createTime, + PprofEventType events) throws IOException { + String checkArgumentMessage = checkArgumentError(serviceId, serviceInstanceIds, duration, events); + if (checkArgumentMessage != null) { + return PprofTaskCreationResult.builder() + .code(PprofTaskCreationType.ARGUMENT_ERROR) + .errorReason(checkArgumentMessage) + .build(); + } + String checkTaskProfilingMessage = checkTaskProfiling(serviceId, createTime); + if (checkTaskProfilingMessage != null) { + return PprofTaskCreationResult.builder() + .code(PprofTaskCreationType.ALREADY_PROFILING_ERROR) + .errorReason(checkTaskProfilingMessage) + .build(); + } + return null; + } + + private String checkArgumentError(String serviceId, + List serviceInstanceIds, + int duration, + PprofEventType events) { + if (serviceId == null) { + return "service cannot be null"; + } + if (duration <= 0) { + return "duration cannot be negative"; + } + if (events == null) { + return "events cannot be empty"; + } + if (CollectionUtils.isEmpty(serviceInstanceIds)) { + return "serviceInstanceIds cannot be empty"; + } + return null; + } + + private String checkTaskProfiling(String serviceId, + long createTime) throws IOException { + // Each service can only enable one task at a time + long endTimeBucket = TimeBucket.getMinuteTimeBucket(createTime); + final List alreadyHaveTaskList = getPprofTaskDAO().getTaskList( + serviceId, null, endTimeBucket, 1 + ); + if (CollectionUtils.isNotEmpty(alreadyHaveTaskList)) { + for (PprofTask task : alreadyHaveTaskList) { + if (task.getCreateTime() + TimeUnit.SECONDS.toMillis(task.getDuration()) >= createTime) { + // if the endTime is greater or equal than the createTime of the newly created task, i.e. there is overlap between two tasks, it is an invalid case, it will return an error + return "current service already has monitor pprof task execute at this time"; + } + } + } + return null; + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofQueryService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofQueryService.java new file mode 100644 index 000000000000..2d456996ac02 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofQueryService.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof; + +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.library.module.Service; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import java.util.Objects; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import java.io.IOException; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; +import org.apache.skywalking.oap.server.core.query.type.PprofStackTree; +import org.apache.skywalking.oap.server.library.pprof.parser.PprofMergeBuilder; +import java.util.List; +import com.google.gson.Gson; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RequiredArgsConstructor +public class PprofQueryService implements Service { + private static final Gson GSON = new Gson(); + + private final ModuleManager moduleManager; + + private IPprofTaskQueryDAO taskQueryDAO; + private IPprofDataQueryDAO dataQueryDAO; + private IPprofTaskLogQueryDAO logQueryDAO; + + private IPprofTaskQueryDAO getTaskQueryDAO() { + if (taskQueryDAO == null) { + this.taskQueryDAO = moduleManager.find(StorageModule.NAME) + .provider() + .getService(IPprofTaskQueryDAO.class); + } + return taskQueryDAO; + } + + private IPprofDataQueryDAO getPprofDataQueryDAO() { + if (dataQueryDAO == null) { + this.dataQueryDAO = moduleManager.find(StorageModule.NAME) + .provider() + .getService(IPprofDataQueryDAO.class); + } + return dataQueryDAO; + } + + private IPprofTaskLogQueryDAO getTaskLogQueryDAO() { + if (logQueryDAO == null) { + this.logQueryDAO = moduleManager.find(StorageModule.NAME) + .provider() + .getService(IPprofTaskLogQueryDAO.class); + } + return logQueryDAO; + } + + public List queryTask(String serviceId, Duration duration, Integer limit) throws IOException { + Long startTimeBucket = null; + Long endTimeBucket = null; + if (Objects.nonNull(duration)) { + startTimeBucket = duration.getStartTimeBucketInSec(); + endTimeBucket = duration.getEndTimeBucketInSec(); + } + List tasks = getTaskQueryDAO().getTaskList(serviceId, startTimeBucket, endTimeBucket, limit); + return tasks; + } + + public PprofStackTree queryPprofData(String taskId, List instanceIds) throws IOException { + List pprofDataList = getPprofDataQueryDAO().getByTaskIdAndInstances(taskId, instanceIds); + List trees = pprofDataList.stream() + .map(data -> GSON.fromJson(new String(data.getDataBinary()), FrameTree.class)) + .collect(Collectors.toList()); + FrameTree resultTree = new PprofMergeBuilder() + .merge(trees) + .build(); + return new PprofStackTree(resultTree); + } + + public List queryPprofTaskLogs(String taskId) throws IOException { + List taskLogList = getTaskLogQueryDAO().getTaskLogList(); + return findMatchedLogs(taskId, taskLogList); + } + + private List findMatchedLogs(final String taskID, final List allLogs) { + return allLogs.stream() + .filter(l -> Objects.equals(l.getId(), taskID)) + .map(this::extendTaskLog) + .collect(Collectors.toList()); + } + + private PprofTaskLog extendTaskLog(PprofTaskLog log) { + final IDManager.ServiceInstanceID.InstanceIDDefinition instanceIDDefinition = IDManager.ServiceInstanceID + .analysisId(log.getInstanceId()); + log.setInstanceName(instanceIDDefinition.getName()); + return log; + } + +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java new file mode 100644 index 000000000000..5cdf1dd2b1ed --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + package org.apache.skywalking.oap.server.core.profiling.pprof.storage; + +import com.google.gson.Gson; +import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher; +import org.apache.skywalking.oap.server.core.source.PprofProfilingData; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; + +public class PprofProfilingDataDispatcher implements SourceDispatcher { + private static final Gson GSON = new Gson(); + + @Override + public void dispatch(PprofProfilingData source) { + PprofProfilingDataRecord record = new PprofProfilingDataRecord(); + record.setTaskId(source.getTaskId()); + record.setInstanceId(source.getInstanceId()); + record.setEventType(source.getEventType().toString()); + record.setDataBinary(GSON.toJson(source.getFrameTree()).getBytes()); + record.setUploadTime(source.getUploadTime()); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(source.getUploadTime())); + RecordStreamProcessor.getInstance().in(record); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java new file mode 100644 index 000000000000..17164411a257 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof.storage; + +import lombok.Data; +import org.apache.skywalking.oap.server.core.analysis.record.Record; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB; +import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.core.analysis.Stream; +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.PPROF_PROFILING_DATA; +import org.apache.skywalking.oap.server.core.storage.StorageID; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; +import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; +import com.google.common.hash.Hashing; +import java.nio.charset.StandardCharsets; + +@Data +@Stream(name = PprofProfilingDataRecord.INDEX_NAME, scopeId = PPROF_PROFILING_DATA, + builder = PprofProfilingDataRecord.Builder.class, processor = RecordStreamProcessor.class) +@BanyanDB.TimestampColumn(PprofProfilingDataRecord.UPLOAD_TIME) +public class PprofProfilingDataRecord extends Record { + public static final String INDEX_NAME = "pprof_profiling_data"; + + public static final String TASK_ID = "task_id"; + public static final String EVENT_TYPE = "event_type"; + public static final String INSTANCE_ID = "instance_id"; + public static final String DATA_BINARY = "data_binary"; + public static final String UPLOAD_TIME = "upload_time"; + + @Column(name = TASK_ID) + private String taskId; + + @Column(name = INSTANCE_ID) + @BanyanDB.SeriesID(index = 0) + private String instanceId; + + @Column(name = EVENT_TYPE) + private String eventType; + + @Column(name = UPLOAD_TIME) + private long uploadTime; + + @Column(name = DATA_BINARY, storageOnly = true) + private byte[] dataBinary; + + @Override + public StorageID id() { + return new StorageID().appendMutant( + new String[]{ + TASK_ID, + INSTANCE_ID, + EVENT_TYPE, + UPLOAD_TIME + }, + Hashing.sha256().newHasher() + .putString(taskId, StandardCharsets.UTF_8) + .putString(instanceId, StandardCharsets.UTF_8) + .putString(eventType, StandardCharsets.UTF_8) + .putLong(uploadTime) + .hash().toString() + ); + } + + public static class Builder implements StorageBuilder { + @Override + public PprofProfilingDataRecord storage2Entity(final Convert2Entity converter) { + final PprofProfilingDataRecord dataTraffic = new PprofProfilingDataRecord(); + dataTraffic.setTimeBucket(((Number) converter.get(TIME_BUCKET)).longValue()); + dataTraffic.setTaskId((String) converter.get(TASK_ID)); + dataTraffic.setInstanceId((String) converter.get(INSTANCE_ID)); + dataTraffic.setUploadTime(((Number) converter.get(UPLOAD_TIME)).longValue()); + dataTraffic.setEventType((String) converter.get(EVENT_TYPE)); + dataTraffic.setDataBinary(converter.getBytes(DATA_BINARY)); + return dataTraffic; + } + + @Override + public void entity2Storage(final PprofProfilingDataRecord storageData, final Convert2Storage converter) { + converter.accept(TIME_BUCKET, storageData.getTimeBucket()); + converter.accept(TASK_ID, storageData.getTaskId()); + converter.accept(INSTANCE_ID, storageData.getInstanceId()); + converter.accept(UPLOAD_TIME, storageData.getUploadTime()); + converter.accept(EVENT_TYPE, storageData.getEventType()); + converter.accept(DATA_BINARY, storageData.getDataBinary()); + } + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java new file mode 100644 index 000000000000..f3ee5ec7338e --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof.storage; + +import org.apache.skywalking.oap.server.core.analysis.Stream; +import org.apache.skywalking.oap.server.core.analysis.record.Record; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.source.ScopeDeclaration; +import org.apache.skywalking.oap.server.core.storage.StorageID; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB; +import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearch; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; +import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.PPROF_TASK_LOG; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +@ScopeDeclaration(id = PPROF_TASK_LOG, name = "PprofTaskLog") +@Stream(name = PprofTaskLogRecord.INDEX_NAME, scopeId = PPROF_TASK_LOG, builder = PprofTaskLogRecord.Builder.class, processor = RecordStreamProcessor.class) +@BanyanDB.TimestampColumn(PprofTaskLogRecord.TIMESTAMP) +public class PprofTaskLogRecord extends Record { + public static final String INDEX_NAME = "pprof_task_log"; + public static final String TASK_ID = "task_id"; + public static final String INSTANCE_ID = "instance_id"; + public static final String OPERATION_TYPE = "operation_type"; + public static final String OPERATION_TIME = "operation_time"; + public static final String TIMESTAMP = "timestamp"; + + @Column(name = TASK_ID) + private String taskId; + @Column(name = INSTANCE_ID) + @BanyanDB.SeriesID(index = 0) + private String instanceId; + @Column(name = OPERATION_TYPE, storageOnly = true) + private int operationType; + @ElasticSearch.EnableDocValues + @Column(name = OPERATION_TIME) + private long operationTime; + @Getter + @Setter + @ElasticSearch.EnableDocValues + @Column(name = TIMESTAMP) + private long timestamp; + + @Override + public StorageID id() { + return new StorageID() + .append(TASK_ID, getTaskId()) + .append(INSTANCE_ID, getInstanceId()) + .append(OPERATION_TYPE, getOperationType()) + .append(OPERATION_TIME, getOperationTime()); + } + + public static class Builder implements StorageBuilder { + @Override + public PprofTaskLogRecord storage2Entity(final Convert2Entity converter) { + final PprofTaskLogRecord log = new PprofTaskLogRecord(); + log.setTaskId((String) converter.get(TASK_ID)); + log.setInstanceId((String) converter.get(INSTANCE_ID)); + log.setOperationType(((Number) converter.get(OPERATION_TYPE)).intValue()); + log.setOperationTime(((Number) converter.get(OPERATION_TIME)).longValue()); + log.setTimestamp(((Number) converter.get(TIMESTAMP)).longValue()); + log.setTimeBucket(((Number) converter.get(TIME_BUCKET)).longValue()); + return log; + } + + @Override + public void entity2Storage(final PprofTaskLogRecord storageData, final Convert2Storage converter) { + converter.accept(TASK_ID, storageData.getTaskId()); + converter.accept(INSTANCE_ID, storageData.getInstanceId()); + converter.accept(OPERATION_TYPE, storageData.getOperationType()); + converter.accept(OPERATION_TIME, storageData.getOperationTime()); + converter.accept(TIME_BUCKET, storageData.getTimeBucket()); + converter.accept(TIMESTAMP, storageData.getTimestamp()); + } + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java new file mode 100644 index 000000000000..e00090d7fa05 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof.storage; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.analysis.Stream; +import org.apache.skywalking.oap.server.core.analysis.config.NoneStream; +import org.apache.skywalking.oap.server.core.analysis.worker.NoneStreamProcessor; +import org.apache.skywalking.oap.server.core.source.ScopeDeclaration; +import org.apache.skywalking.oap.server.core.storage.StorageID; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB; +import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearch; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; +import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; +import java.util.List; + +import com.google.gson.Gson; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.PPROF_TASK; + +/** + * Pprof task database bean, use none stream + */ +@Getter +@Setter +@ScopeDeclaration(id = PPROF_TASK, name = "PprofTask") +@Stream(name = PprofTaskRecord.INDEX_NAME, scopeId = PPROF_TASK, builder = PprofTaskRecord.Builder.class, processor = NoneStreamProcessor.class) +@BanyanDB.TimestampColumn(PprofTaskRecord.CREATE_TIME) +public class PprofTaskRecord extends NoneStream { + private static final Gson GSON = new Gson(); + + public static final String INDEX_NAME = "pprof_task"; + public static final String TASK_ID = "task_id"; + public static final String SERVICE_ID = "service_id"; + public static final String SERVICE_INSTANCE_IDS = "service_instance_ids"; + public static final String CREATE_TIME = "create_time"; + public static final String EVENT_TYPES = "events"; + public static final String DURATION = "duration"; + public static final String DUMP_PERIOD = "dump_period"; + + @Override + public StorageID id() { + return new StorageID().append(TASK_ID, taskId); + } + + @Column(name = SERVICE_ID) + @BanyanDB.SeriesID(index = 0) + private String serviceId; + @Column(name = SERVICE_INSTANCE_IDS) + private String serviceInstanceIds; + @Column(name = TASK_ID) + private String taskId; + @ElasticSearch.EnableDocValues + @Column(name = CREATE_TIME) + private long createTime; + @ElasticSearch.EnableDocValues + @Column(name = EVENT_TYPES) + private String events; + @Column(name = DURATION) + private int duration; + @Column(name = DUMP_PERIOD) + private int dumpPeriod; + + public static class Builder implements StorageBuilder { + + @Override + public PprofTaskRecord storage2Entity(final Convert2Entity converter) { + PprofTaskRecord record = new PprofTaskRecord(); + record.setServiceId((String) converter.get(SERVICE_ID)); + record.setServiceInstanceIds((String) converter.get(SERVICE_INSTANCE_IDS)); + record.setTaskId((String) converter.get(TASK_ID)); + record.setCreateTime(((Number) converter.get(CREATE_TIME)).longValue()); + record.setEvents((String) converter.get(EVENT_TYPES)); + record.setDuration(((Number) converter.get(DURATION)).intValue()); + record.setDumpPeriod(((Number) converter.get(DUMP_PERIOD)).intValue()); + record.setTimeBucket(((Number) converter.get(TIME_BUCKET)).longValue()); + return record; + } + + @Override + public void entity2Storage(final PprofTaskRecord storageData, final Convert2Storage converter) { + converter.accept(SERVICE_ID, storageData.getServiceId()); + converter.accept(SERVICE_INSTANCE_IDS, storageData.getServiceInstanceIds()); + converter.accept(TASK_ID, storageData.getTaskId()); + converter.accept(CREATE_TIME, storageData.getCreateTime()); + converter.accept(EVENT_TYPES, storageData.getEvents()); + converter.accept(DURATION, storageData.getDuration()); + converter.accept(DUMP_PERIOD, storageData.getDumpPeriod()); + converter.accept(TIME_BUCKET, storageData.getTimeBucket()); + } + } + + public void setServiceInstanceIdsFromList(List serviceInstanceIds) { + this.serviceInstanceIds = GSON.toJson(serviceInstanceIds); + } + +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PprofTaskLog.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PprofTaskLog.java new file mode 100644 index 000000000000..471bde9cdfaa --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PprofTaskLog.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; + +@Setter +@Getter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class PprofTaskLog { + // task id + private String id; + + // instance + private String instanceId; + private String instanceName; + + // operation + private PprofTaskLogOperationType operationType; + private long operationTime; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofAnalyzationRequest.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofAnalyzationRequest.java new file mode 100644 index 000000000000..cb62845d2bee --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofAnalyzationRequest.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.input; + +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +@Getter +@Setter +public class PprofAnalyzationRequest { + private String taskId; + private List instanceIds; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskCreationRequest.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskCreationRequest.java new file mode 100644 index 000000000000..af19a9466d1e --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskCreationRequest.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.input; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; + +import java.util.List; + +@Getter +@Setter +public class PprofTaskCreationRequest { + private String serviceId; + private List serviceInstanceIds; + private int duration; + private PprofEventType events; + private int dumpPeriod; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java new file mode 100644 index 000000000000..db789a56228a --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java @@ -0,0 +1,12 @@ +package org.apache.skywalking.oap.server.core.query.input; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class PprofTaskListRequest { + private String serviceId; + private Duration queryDuration; + private Integer limit; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofAnalyzation.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofAnalyzation.java new file mode 100644 index 000000000000..ef56f268acad --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofAnalyzation.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class PprofAnalyzation { + private PprofStackTree tree; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofEventType.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofEventType.java new file mode 100644 index 000000000000..333b0b061dd9 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofEventType.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public enum PprofEventType { + CPU(0, "cpu"), + HEAP(1, "heap"), + BLOCK(2, "block"), + MUTEX(3, "mutex"), + GOROUTINE(4, "goroutine"), + THREADCREATE(5, "threadcreate"), + ALLOCS(6, "allocs"); + + + private final int code; + private final String name; + + public static PprofEventType valueOfString(String event) { + return PprofEventType.valueOf(event); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackElement.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackElement.java new file mode 100644 index 000000000000..b82f3e1ec670 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackElement.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter + +public class PprofStackElement { + // work for tree building, id matches multiple parentId + private int id; + private int parentId; + + // stack code signature + private String codeSignature; + + private long total; + private long self; + +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackTree.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackTree.java new file mode 100644 index 000000000000..2600133dd98b --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackTree.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import com.google.common.collect.Lists; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; +import java.util.List; +import java.util.Objects; + +@Setter +@Getter +@NoArgsConstructor +public class PprofStackTree { + private List elements; + + private int idGen = 0; + + public PprofStackTree(FrameTree tree) { + this.elements = convertTree(-1, tree); + } + + private List convertTree(int parentId, FrameTree tree) { + PprofStackElement pprofStackElement = new PprofStackElement(); + pprofStackElement.setId(idGen++); + pprofStackElement.setParentId(parentId); + pprofStackElement.setCodeSignature(tree.getSignature()); + pprofStackElement.setTotal(tree.getTotal()); + pprofStackElement.setSelf(tree.getSelf()); + + List children = tree.getChildren(); + List result = Lists.newArrayList(pprofStackElement); + if (Objects.isNull(children) || children.isEmpty()) { + return result; + } + + for (FrameTree child : children) { + List childElements = convertTree(pprofStackElement.getId(), child); + result.addAll(childElements); + } + + return result; + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTask.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTask.java new file mode 100644 index 000000000000..3d8e4e3ba5bc --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTask.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Setter +@Getter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class PprofTask { + + private String id; + private String serviceId; + private List serviceInstanceIds; + private PprofEventType events; + private long createTime; + private int duration; + private int dumpPeriod; + +} \ No newline at end of file diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java new file mode 100644 index 000000000000..1e9ca4d8596c --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java @@ -0,0 +1,24 @@ +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +/** + * create pprof task result + */ +@Setter +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class PprofTaskCreationResult { + // ErrorReason gives detailed reason for the exception, if the code returned represents a kind of failure. + private String errorReason; + // Code defines the status of the response, i.e. success or failure. + private PprofTaskCreationType code; + // Task id, if code is SUCCESS. + private String id; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java new file mode 100644 index 000000000000..ddf62db10a05 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java @@ -0,0 +1,8 @@ +package org.apache.skywalking.oap.server.core.query.type; + +public enum PprofTaskCreationType { + SUCCESS, + ARGUMENT_ERROR, + ALREADY_PROFILING_ERROR, + ; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskListResult.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskListResult.java new file mode 100644 index 000000000000..e3ee7a75a99d --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskListResult.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.util.List; + +@Data +@AllArgsConstructor +public class PprofTaskListResult { + private String errorReason; + private List tasks; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskLogOperationType.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskLogOperationType.java new file mode 100644 index 000000000000..e220ceb23d37 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskLogOperationType.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import java.util.HashMap; +import java.util.Map; + +public enum PprofTaskLogOperationType { + NOTIFIED(1), // when sniffer has execution finished to report + EXECUTION_FINISHED(2), // when sniffer has execution finished to report + PPROF_UPLOAD_FILE_TOO_LARGE_ERROR(3), // when sniffer finished task but jfr file is to large that oap server can not receive + EXECUTION_TASK_ERROR(4) // when sniffer fails to execute its task + ; + + private final int code; + private static final Map CACHE = new HashMap(); + + static { + for (PprofTaskLogOperationType val : PprofTaskLogOperationType.values()) { + CACHE.put(val.getCode(), val); + } + } + + /** + * Parse operation type by code + */ + public static PprofTaskLogOperationType parse(int code) { + return CACHE.get(code); + } + + PprofTaskLogOperationType(int code) { + this.code = code; + } + + public int getCode() { + return this.code; + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskProgress.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskProgress.java new file mode 100644 index 000000000000..991e6e4f6d95 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskProgress.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.Data; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; + +import java.util.List; + +@Data +public class PprofTaskProgress { + private List logs; + private List errorInstanceIds; + private List successInstanceIds; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java index 1d51cc193e18..f2018b4a0984 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java @@ -151,6 +151,9 @@ public class DefaultScopeDefine { public static final int BROWSER_APP_RESOURCE_PERF = 88; public static final int BROWSER_APP_WEB_INTERACTION_PAGE_PERF = 89; public static final int SW_SPAN_ATTACHED_EVENT = 90; + public static final int PPROF_TASK = 91; + public static final int PPROF_PROFILING_DATA = 92; + public static final int PPROF_TASK_LOG = 93; /** * Catalog of scope, the metrics processor could use this to group all generated metrics by oal rt. diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/PprofProfilingData.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/PprofProfilingData.java new file mode 100644 index 000000000000..d9c23835f86c --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/PprofProfilingData.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.source; + +import lombok.Data; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.PPROF_PROFILING_DATA; + +@Data +@ScopeDeclaration(id = PPROF_PROFILING_DATA, name = "PprofProfilingData") +@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class) +public class PprofProfilingData extends Source { + private volatile String entityId; + + @Override + public int scope() { + return PPROF_PROFILING_DATA; + } + + @Override + public String getEntityId() { + if (entityId == null) { + return taskId + instanceId + eventType.name() + uploadTime; + } + return entityId; + } + + private String taskId; + private String instanceId; + private long uploadTime; + private PprofEventType eventType; + private Object frameTree; +} \ No newline at end of file diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java index 89ac015cd0b4..cd711dedeaa7 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java @@ -24,6 +24,9 @@ import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IJFRDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.continuous.IContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IServiceLabelDAO; import org.apache.skywalking.oap.server.core.storage.profiling.trace.IProfileTaskLogQueryDAO; @@ -96,6 +99,9 @@ public Class[] services() { IAsyncProfilerTaskQueryDAO.class, IAsyncProfilerTaskLogQueryDAO.class, IJFRDataQueryDAO.class, + IPprofTaskQueryDAO.class, + IPprofTaskLogQueryDAO.class, + IPprofDataQueryDAO.class, StorageTTLStatusQuery.class }; } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofDataQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofDataQueryDAO.java new file mode 100644 index 000000000000..523fa73d419b --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofDataQueryDAO.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.oap.server.core.storage.profiling.pprof; + +import org.apache.skywalking.oap.server.library.module.Service; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import java.io.IOException; +import java.util.List; + +public interface IPprofDataQueryDAO extends Service { + /** + * get pprof data record + * + * @param taskId taskId + * @param instanceIds instances of successfully uploaded file and parsed + * @return record list + */ + List getByTaskIdAndInstances(final String taskId, List instanceIds) throws IOException; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java new file mode 100644 index 000000000000..78571479204a --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java @@ -0,0 +1,14 @@ +package org.apache.skywalking.oap.server.core.storage.profiling.pprof; + +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.storage.DAO; + +import java.io.IOException; +import java.util.List; + +public interface IPprofTaskLogQueryDAO extends DAO { + /** + * search all task log list in appoint task id + */ + List getTaskLogList() throws IOException; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskQueryDAO.java new file mode 100644 index 000000000000..4c4945654512 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskQueryDAO.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.storage.profiling.pprof; + +import org.apache.skywalking.oap.server.core.storage.DAO; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import java.io.IOException; +import java.util.List; + +public interface IPprofTaskQueryDAO extends DAO { + + /** + * search task list in appoint time bucket + * + * @param serviceId monitor service id, maybe null + * @param startTimeBucket time bucket bigger than or equals, nullable + * @param endTimeBucket time bucket smaller than or equals, nullable + * @param limit limit count, if null means query all + */ + List getTaskList(final String serviceId, final Long startTimeBucket, + final Long endTimeBucket, final Integer limit) throws IOException; + + /** + * query profile task by id + * + * @param id taskId + * @return task data + */ + PprofTask getById(final String id) throws IOException; + +} diff --git a/oap-server/server-library/library-pprof-parser/pom.xml b/oap-server/server-library/library-pprof-parser/pom.xml new file mode 100755 index 000000000000..196e38a441bd --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/pom.xml @@ -0,0 +1,86 @@ + + + 4.0.0 + + + org.apache.skywalking + server-library + ${revision} + + + library-pprof-parser + + + 17 + 17 + UTF-8 + true + + + + + com.google.protobuf + protobuf-java + 3.25.5 + + + com.google.code.gson + gson + 2.10.1 + + + org.projectlombok + lombok + 1.18.30 + provided + + + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.6.1 + + com.google.protobuf:protoc:3.25.5:exe:${os.detected.classifier} + + + + + compile + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.3.0 + + + add-generated-sources + generate-sources + + add-source + + + + target/generated-sources/protobuf/java + + + + + + + + + kr.motd.maven + os-maven-plugin + 1.7.0 + + + + \ No newline at end of file diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java new file mode 100644 index 000000000000..20b2cf57570b --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.pprof.parser; + +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class PprofMergeBuilder { + private final Node root = new Node("root"); + + public PprofMergeBuilder merge(List trees) { + if (trees == null || trees.isEmpty()) { + return this; + } + for (FrameTree tree : trees) { + merge0(root, tree); + } + return this; + } + + public PprofMergeBuilder merge(FrameTree tree) { + merge0(root, tree); + return this; + } + + private void merge0(Node node, FrameTree tree) { + if (tree == null) { + return; + } + if (tree.getChildren() != null) { + for (FrameTree childTree : tree.getChildren()) { + Node child = getOrAddChild(node, childTree.getSignature()); + merge0(child, childTree); + } + } + node.total += tree.getTotal(); + node.self += tree.getSelf(); + } + + private Node getOrAddChild(Node parent, String signature) { + return parent.children.computeIfAbsent(signature, Node::new); + } + + public FrameTree build() { + return toFrameTree(root); + } + + private FrameTree toFrameTree(Node node) { + FrameTree tree = new FrameTree(node.signature, node.total, node.self); + for (Node child : node.children.values()) { + tree.getChildren().add(toFrameTree(child)); + } + return tree; + } + + private static class Node { + final String signature; + long total; + long self; + final Map children = new HashMap<>(); + + Node(String signature) { + this.signature = signature; + } + } +} diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java new file mode 100644 index 000000000000..eacbf2d55f18 --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.pprof.parser; + +import com.google.perftools.profiles.ProfileProto; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTreeBuilder; + +/** + * Parses pprof protobuf format files and converts them to frame trees. + */ +public class PprofParser { + + public static FrameTree dumpTree(ByteBuffer buf) throws IOException { + ProfileProto.Profile profile = ProfileProto.Profile.parseFrom(buf); + FrameTree tree = new FrameTreeBuilder(profile).build(); + return tree; + } + + public static FrameTree dumpTree(String filePath) throws IOException { + File file = new File(filePath); + if (!file.exists()) { + throw new IOException("Pprof file not found: " + filePath); + } + + InputStream fileStream = new FileInputStream(file); + ProfileProto.Profile profile = ProfileProto.Profile.parseFrom(fileStream); + FrameTree tree = new FrameTreeBuilder(profile).build(); + return tree; + + } +} \ No newline at end of file diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java new file mode 100755 index 000000000000..2b1587a8ec53 --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.pprof.type; + +import com.google.gson.annotations.SerializedName; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import java.util.ArrayList; +import java.util.List; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class FrameTree { + @SerializedName("name") + private String signature; + @SerializedName("value") + private long total; + private long self; + private final List children = new ArrayList<>(); +} diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java new file mode 100644 index 000000000000..3fe4a3b04089 --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.pprof.type; + +import com.google.perftools.profiles.ProfileProto; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Data +@NoArgsConstructor +@AllArgsConstructor +class RawFrameTree { + private long locationId; + private long total; + private long self; + private final Map children = new HashMap<>(); +} + +public class FrameTreeBuilder { + private final ProfileProto.Profile profile; + private final RawFrameTree root; + + public FrameTreeBuilder(ProfileProto.Profile profile) { + this.profile = profile; + this.root = new RawFrameTree(0, 0, 0); + } + + private FrameTree parseTree(RawFrameTree rawTree) { + FrameTree tree = new FrameTree(getSignature(rawTree.getLocationId()), rawTree.getTotal(), rawTree.getSelf()); + for (RawFrameTree rawChild : rawTree.getChildren().values()) { + FrameTree child = parseTree(rawChild); + tree.getChildren().add(child); + } + return tree; + } + + private String getSignature(long locationId) { + if (locationId == 0) { + return "root"; + } + ProfileProto.Location location = profile.getLocation((int) locationId - 1); + return location.getLineList().stream().map((line) -> { + ProfileProto.Function function = profile.getFunction((int) line.getFunctionId() - 1); + String functionName = profile.getStringTable((int) function.getName()); + return functionName + ":" + line.getLine(); + }).collect(Collectors.joining(";")); + } + + public FrameTree build() { + for (ProfileProto.Sample sample : profile.getSampleList()) { + mergeSample(sample); + } + return parseTree(this.root); + } + + private void mergeSample(ProfileProto.Sample sample) { + // merge sample data + Map children = root.getChildren(); + List locationIdList = new ArrayList<>(sample.getLocationIdList()); + // from root to leaf + Collections.reverse(locationIdList); + int size = locationIdList.size(); + for (int i = 0; i < size; i++) { + boolean isEnd = i == size - 1; + long locationId = locationIdList.get(i); + if (children.containsKey(locationId)) { + RawFrameTree child = children.get(locationId); + child.setTotal(child.getTotal() + 1); + child.setSelf(child.getSelf() + (isEnd ? 1 : 0)); + children = child.getChildren(); + } else { + RawFrameTree child = new RawFrameTree(locationId, 1, (isEnd ? 1 : 0)); + children.put(locationId, child); + children = child.getChildren(); + } + } + root.setTotal(root.getTotal() + 1); + } +} \ No newline at end of file diff --git a/oap-server/server-library/library-pprof-parser/src/main/proto/profile.proto b/oap-server/server-library/library-pprof-parser/src/main/proto/profile.proto new file mode 100755 index 000000000000..60216a3cfc8f --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/proto/profile.proto @@ -0,0 +1,233 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Profile is a common stacktrace profile format. +// +// Measurements represented with this format should follow the +// following conventions: +// +// - Consumers should treat unset optional fields as if they had been +// set with their default value. +// +// - When possible, measurements should be stored in "unsampled" form +// that is most useful to humans. There should be enough +// information present to determine the original sampled values. +// +// - On-disk, the serialized proto must be gzip-compressed. +// +// - The profile is represented as a set of samples, where each sample +// references a sequence of locations, and where each location belongs +// to a mapping. +// - There is a N->1 relationship from sample.location_id entries to +// locations. For every sample.location_id entry there must be a +// unique Location with that id. +// - There is an optional N->1 relationship from locations to +// mappings. For every nonzero Location.mapping_id there must be a +// unique Mapping with that id. + +syntax = "proto3"; + +package perftools.profiles; + +option java_package = "com.google.perftools.profiles"; +option java_outer_classname = "ProfileProto"; + +message Profile { + // A description of the samples associated with each Sample.value. + // For a cpu profile this might be: + // [["cpu","nanoseconds"]] or [["wall","seconds"]] or [["syscall","count"]] + // For a heap profile, this might be: + // [["allocations","count"], ["space","bytes"]], + // If one of the values represents the number of events represented + // by the sample, by convention it should be at index 0 and use + // sample_type.unit == "count". + repeated ValueType sample_type = 1; + // The set of samples recorded in this profile. + repeated Sample sample = 2; + // Mapping from address ranges to the image/binary/library mapped + // into that address range. mapping[0] will be the main binary. + repeated Mapping mapping = 3; + // Locations referenced by samples. + repeated Location location = 4; + // Functions referenced by locations. + repeated Function function = 5; + // A common table for strings referenced by various messages. + // string_table[0] must always be "". + repeated string string_table = 6; + // frames with Function.function_name fully matching the following + // regexp will be dropped from the samples, along with their successors. + int64 drop_frames = 7; // Index into string table. + // frames with Function.function_name fully matching the following + // regexp will be kept, even if it matches drop_frames. + int64 keep_frames = 8; // Index into string table. + + // The following fields are informational, do not affect + // interpretation of results. + + // Time of collection (UTC) represented as nanoseconds past the epoch. + int64 time_nanos = 9; + // Duration of the profile, if a duration makes sense. + int64 duration_nanos = 10; + // The kind of events between sampled occurrences. + // e.g [ "cpu","cycles" ] or [ "heap","bytes" ] + ValueType period_type = 11; + // The number of events between sampled occurrences. + int64 period = 12; + // Free-form text associated with the profile. The text is displayed as is + // to the user by the tools that read profiles (e.g. by pprof). This field + // should not be used to store any machine-readable information, it is only + // for human-friendly content. The profile must stay functional if this field + // is cleaned. + repeated int64 comment = 13; // Indices into string table. + // Index into the string table of the type of the preferred sample + // value. If unset, clients should default to the last sample value. + int64 default_sample_type = 14; + // Documentation link for this profile. The URL must be absolute, + // e.g., http://pprof.example.com/cpu-profile.html + // + // The URL may be missing if the profile was generated by older code or code + // that did not bother to supply a link. + int64 doc_url = 15; // Index into string table. +} + +// ValueType describes the semantics and measurement units of a value. +message ValueType { + int64 type = 1; // Index into string table. + int64 unit = 2; // Index into string table. +} + +// Each Sample records values encountered in some program +// context. The program context is typically a stack trace, perhaps +// augmented with auxiliary information like the thread-id, some +// indicator of a higher level request being handled etc. +message Sample { + // The ids recorded here correspond to a Profile.location.id. + // The leaf is at location_id[0]. + repeated uint64 location_id = 1; + // The type and unit of each value is defined by the corresponding + // entry in Profile.sample_type. All samples must have the same + // number of values, the same as the length of Profile.sample_type. + // When aggregating multiple samples into a single sample, the + // result has a list of values that is the element-wise sum of the + // lists of the originals. + repeated int64 value = 2; + // label includes additional context for this sample. It can include + // things like a thread id, allocation size, etc. + // + // NOTE: While possible, having multiple values for the same label key is + // strongly discouraged and should never be used. Most tools (e.g. pprof) do + // not have good (or any) support for multi-value labels. And an even more + // discouraged case is having a string label and a numeric label of the same + // name on a sample. Again, possible to express, but should not be used. + repeated Label label = 3; +} + +message Label { + // Index into string table. An annotation for a sample (e.g. + // "allocation_size") with an associated value. + // Keys with "pprof::" prefix are reserved for internal use by pprof. + int64 key = 1; + + // At most one of the following must be present + int64 str = 2; // Index into string table + int64 num = 3; + + // Should only be present when num is present. + // Specifies the units of num. + // Use arbitrary string (for example, "requests") as a custom count unit. + // If no unit is specified, consumer may apply heuristic to deduce the unit. + // Consumers may also interpret units like "bytes" and "kilobytes" as memory + // units and units like "seconds" and "nanoseconds" as time units, + // and apply appropriate unit conversions to these. + int64 num_unit = 4; // Index into string table +} + +message Mapping { + // Unique nonzero id for the mapping. + uint64 id = 1; + // Address at which the binary (or DLL) is loaded into memory. + uint64 memory_start = 2; + // The limit of the address range occupied by this mapping. + uint64 memory_limit = 3; + // Offset in the binary that corresponds to the first mapped address. + uint64 file_offset = 4; + // The object this entry is loaded from. This can be a filename on + // disk for the main binary and shared libraries, or virtual + // abstractions like "[vdso]". + int64 filename = 5; // Index into string table + // A string that uniquely identifies a particular program version + // with high probability. E.g., for binaries generated by GNU tools, + // it could be the contents of the .note.gnu.build-id field. + int64 build_id = 6; // Index into string table + + // The following fields indicate the resolution of symbolic info. + bool has_functions = 7; + bool has_filenames = 8; + bool has_line_numbers = 9; + bool has_inline_frames = 10; +} + +// Describes function and line table debug information. +message Location { + // Unique nonzero id for the location. A profile could use + // instruction addresses or any integer sequence as ids. + uint64 id = 1; + // The id of the corresponding profile.Mapping for this location. + // It can be unset if the mapping is unknown or not applicable for + // this profile type. + uint64 mapping_id = 2; + // The instruction address for this location, if available. It + // should be within [Mapping.memory_start...Mapping.memory_limit] + // for the corresponding mapping. A non-leaf address may be in the + // middle of a call instruction. It is up to display tools to find + // the beginning of the instruction if necessary. + uint64 address = 3; + // Multiple line indicates this location has inlined functions, + // where the last entry represents the caller into which the + // preceding entries were inlined. + // + // E.g., if memcpy() is inlined into printf: + // line[0].function_name == "memcpy" + // line[1].function_name == "printf" + repeated Line line = 4; + // Provides an indication that multiple symbols map to this location's + // address, for example due to identical code folding by the linker. In that + // case the line information above represents one of the multiple + // symbols. This field must be recomputed when the symbolization state of the + // profile changes. + bool is_folded = 5; +} + +message Line { + // The id of the corresponding profile.Function for this line. + uint64 function_id = 1; + // Line number in source code. + int64 line = 2; + // Column number in source code. + int64 column = 3; +} + +message Function { + // Unique nonzero id for the function. + uint64 id = 1; + // Name of the function, in human-readable form if available. + int64 name = 2; // Index into string table + // Name of the function, as identified by the system. + // For instance, it can be a C++ mangled name. + int64 system_name = 3; // Index into string table + // Source file containing the function. + int64 filename = 4; // Index into string table + // Line number in source file. + int64 start_line = 5; +} \ No newline at end of file diff --git a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/CollectionUtils.java b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/CollectionUtils.java index d41bce7f2b49..110a4f341e8b 100644 --- a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/CollectionUtils.java +++ b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/CollectionUtils.java @@ -32,6 +32,10 @@ public static boolean isEmpty(List list) { return list == null || list.size() == 0; } + public static boolean isEmpty(String str) { + return str == null || str.length() == 0; + } + public static boolean isEmpty(Set set) { return set == null || set.size() == 0; } diff --git a/oap-server/server-library/pom.xml b/oap-server/server-library/pom.xml index 3e92ba599bae..93481db0742a 100644 --- a/oap-server/server-library/pom.xml +++ b/oap-server/server-library/pom.xml @@ -36,6 +36,7 @@ library-datacarrier-queue library-kubernetes-support library-async-profiler-jfr-parser + library-pprof-parser library-integration-test diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java index 9d77c7e0e60f..4232710c0856 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java @@ -47,6 +47,8 @@ import org.apache.skywalking.oap.query.graphql.resolver.OndemandLogQuery; import org.apache.skywalking.oap.query.graphql.resolver.ProfileMutation; import org.apache.skywalking.oap.query.graphql.resolver.ProfileQuery; +import org.apache.skywalking.oap.query.graphql.resolver.PprofMutation; +import org.apache.skywalking.oap.query.graphql.resolver.PprofQuery; import org.apache.skywalking.oap.query.graphql.resolver.Query; import org.apache.skywalking.oap.query.graphql.resolver.RecordsQuery; import org.apache.skywalking.oap.query.graphql.resolver.TopNRecordsQuery; @@ -150,7 +152,9 @@ public void prepare() throws ServiceNotProvidedException { .resolvers(new RecordsQuery(getManager())) .file("query-protocol/hierarchy.graphqls").resolvers(new HierarchyQuery(getManager())) .file("query-protocol/async-profiler.graphqls") - .resolvers(new AsyncProfilerQuery(getManager()), new AsyncProfilerMutation(getManager())); + .resolvers(new AsyncProfilerQuery(getManager()), new AsyncProfilerMutation(getManager())) + .file("query-protocol/pprof.graphqls") + .resolvers(new PprofQuery(getManager()), new PprofMutation(getManager())); if (config.isEnableOnDemandPodLog()) { schemaBuilder diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofMutation.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofMutation.java new file mode 100644 index 000000000000..eb98f8d7825a --- /dev/null +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofMutation.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.query.graphql.resolver; + +import graphql.kickstart.tools.GraphQLMutationResolver; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofMutationService; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskCreationResult; +import org.apache.skywalking.oap.server.core.query.input.PprofTaskCreationRequest; + +import java.io.IOException; + +@Slf4j +public class PprofMutation implements GraphQLMutationResolver { + private final ModuleManager moduleManager; + + private PprofMutationService mutationService; + + public PprofMutation(ModuleManager moduleManager) { + this.moduleManager = moduleManager; + } + + private PprofMutationService getPprofMutationService() { + if (mutationService == null) { + this.mutationService = moduleManager.find(CoreModule.NAME) + .provider() + .getService(PprofMutationService.class); + } + return mutationService; + } + + public PprofTaskCreationResult createPprofTask(PprofTaskCreationRequest request) throws IOException { + PprofMutationService pprofMutationService = getPprofMutationService(); + return pprofMutationService.createTask(request.getServiceId(), request.getServiceInstanceIds(), + request.getDuration(), request.getEvents(), request.getDumpPeriod()); + } +} \ No newline at end of file diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofQuery.java new file mode 100644 index 000000000000..9b1096cfbdec --- /dev/null +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofQuery.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.query.graphql.resolver; + +import org.apache.skywalking.oap.server.core.CoreModule; +import groovy.util.logging.Slf4j; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import graphql.kickstart.tools.GraphQLQueryResolver; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofQueryService; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskListResult; +import org.apache.skywalking.oap.server.core.query.input.PprofTaskListRequest; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskProgress; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.oap.server.core.query.type.PprofAnalyzation; +import org.apache.skywalking.oap.server.core.query.type.PprofStackTree; +import org.apache.skywalking.oap.server.core.query.input.PprofAnalyzationRequest; +import java.util.ArrayList; +import java.io.IOException; +import java.util.List; + +@Slf4j +public class PprofQuery implements GraphQLQueryResolver { + private final ModuleManager moduleManager; + + private PprofQueryService queryService; + + public PprofQuery(ModuleManager moduleManager) { + this.moduleManager = moduleManager; + } + + private PprofQueryService getPprofQueryService() { + if (queryService == null) { + this.queryService = moduleManager.find(CoreModule.NAME).provider().getService(PprofQueryService.class); + } + return queryService; + } + + public PprofTaskListResult queryPprofTaskList(PprofTaskListRequest request) throws IOException { + List tasks = getPprofQueryService().queryTask( + request.getServiceId(), request.getQueryDuration(), request.getLimit() + ); + return new PprofTaskListResult(null, tasks); + } + + public PprofAnalyzation queryPprofAnalyze(PprofAnalyzationRequest request) throws IOException { + PprofStackTree eventFrameTrees = getPprofQueryService().queryPprofData( + request.getTaskId(), request.getInstanceIds() + ); + return new PprofAnalyzation(eventFrameTrees); + } + + public PprofTaskProgress queryPprofTaskProgress(String taskId) throws IOException { + PprofTaskProgress pprofTaskProgress = new PprofTaskProgress(); + List logs = getPprofQueryService().queryPprofTaskLogs(taskId); + pprofTaskProgress.setLogs(logs); + List errorInstances = new ArrayList<>(); + List successInstances = new ArrayList<>(); + logs.forEach(log -> { + if (PprofTaskLogOperationType.EXECUTION_FINISHED.equals(log.getOperationType())) { + successInstances.add(log.getInstanceId()); + } else if (PprofTaskLogOperationType.EXECUTION_TASK_ERROR.equals(log.getOperationType()) + || PprofTaskLogOperationType.PPROF_UPLOAD_FILE_TOO_LARGE_ERROR.equals(log.getOperationType())) { + errorInstances.add(log.getInstanceId()); + } + }); + pprofTaskProgress.setErrorInstanceIds(errorInstances); + pprofTaskProgress.setSuccessInstanceIds(successInstances); + return pprofTaskProgress; + } +} diff --git a/oap-server/server-receiver-plugin/pom.xml b/oap-server/server-receiver-plugin/pom.xml index 17892a71a48e..068a86167c62 100644 --- a/oap-server/server-receiver-plugin/pom.xml +++ b/oap-server/server-receiver-plugin/pom.xml @@ -49,6 +49,7 @@ skywalking-telegraf-receiver-plugin aws-firehose-receiver skywalking-async-profiler-receiver-plugin + skywalking-pprof-receiver-plugin diff --git a/oap-server/server-receiver-plugin/receiver-proto/pom.xml b/oap-server/server-receiver-plugin/receiver-proto/pom.xml index 0c057f133cef..aaff44d8c39a 100644 --- a/oap-server/server-receiver-plugin/receiver-proto/pom.xml +++ b/oap-server/server-receiver-plugin/receiver-proto/pom.xml @@ -41,6 +41,14 @@ flatbuffers-java provided + + com.google.protobuf + protobuf-java + + + com.google.protobuf + protobuf-java-util + diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/pom.xml b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/pom.xml new file mode 100644 index 000000000000..985a5b30702f --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/pom.xml @@ -0,0 +1,53 @@ + + + + + + server-receiver-plugin + org.apache.skywalking + ${revision} + + 4.0.0 + + skywalking-pprof-receiver-plugin + + + + org.apache.skywalking + library-module + ${project.version} + + + org.apache.skywalking + skywalking-sharing-server-plugin + ${project.version} + + + org.apache.skywalking + apm-network + ${project.version} + + + org.apache.skywalking + library-pprof-parser + ${project.version} + + + diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModule.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModule.java new file mode 100644 index 000000000000..f768f1a7ec91 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModule.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.module; + +import org.apache.skywalking.oap.server.library.module.ModuleDefine; + +public class PprofModule extends ModuleDefine { + public static final String NAME = "receiver-pprof"; + + public PprofModule() { + super(NAME); + } + + @Override + public Class[] services() { + return new Class[] {}; + } +} \ No newline at end of file diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java new file mode 100644 index 000000000000..99f5b70fc078 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.module; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; + +@Getter +@Setter +public class PprofModuleConfig extends ModuleConfig { + /** + * Used to manage the maximum size of the pprof file that can be received, the unit is Byte + * default is 30M + */ + private int pprofMaxSize = 4 * 1024; + /** + * default is true + *

+ * If memoryParserEnabled is true, then PprofByteBufCollectionObserver will be enabled + * will use memory to receive pprof files without writing files (this is currently used). + * This can prevent the oap server from crashing due to no volume mounting. + *

+ * If memoryParserEnabled is false, then PprofFileCollectionObserver will be enabled + * which uses createTemp to write files and then reads the files for parsing. + * The advantage of this is that it reduces memory and prevents the oap server from crashing due to + * insufficient memory, but it may report an error due to no volume mounting. + */ + private boolean memoryParserEnabled = true; +} diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/PprofModuleProvider.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/PprofModuleProvider.java new file mode 100644 index 000000000000..0ff1aacad79e --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/PprofModuleProvider.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.provider; + +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.server.GRPCHandlerRegister; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; +import org.apache.skywalking.oap.server.library.module.ModuleDefine; +import org.apache.skywalking.oap.server.library.module.ModuleProvider; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; +import org.apache.skywalking.oap.server.receiver.pprof.module.PprofModule; +import org.apache.skywalking.oap.server.receiver.pprof.module.PprofModuleConfig; +import org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler; +import org.apache.skywalking.oap.server.receiver.sharing.server.SharingServerModule; + +public class PprofModuleProvider extends ModuleProvider { + private PprofModuleConfig config; + + @Override + public String name() { + return "default"; + } + + @Override + public Class module() { + return PprofModule.class; + } + + @Override + public ConfigCreator newConfigCreator() { + return new ConfigCreator() { + @Override + public Class type() { + return PprofModuleConfig.class; + } + + @Override + public void onInitialized(final PprofModuleConfig initialized) { + config = initialized; + } + }; + } + + @Override + public void prepare() throws ServiceNotProvidedException, ModuleStartException { + + } + + @Override + public void start() throws ServiceNotProvidedException, ModuleStartException { + GRPCHandlerRegister grpcHandlerRegister = getManager().find(SharingServerModule.NAME) + .provider() + .getService(GRPCHandlerRegister.class); + PprofServiceHandler pprofServiceHandler = new PprofServiceHandler(getManager(), + config.getPprofMaxSize(), config.isMemoryParserEnabled()); + grpcHandlerRegister.addHandler(pprofServiceHandler); + } + + @Override + public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException { + } + + @Override + public String[] requiredModules() { + return new String[]{ + CoreModule.NAME, + SharingServerModule.NAME + }; + } + +} diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java new file mode 100644 index 000000000000..6fdf30538cd0 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.provider.handler; + +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; +import java.io.IOException; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.apm.network.pprof.v10.PprofData; +import org.apache.skywalking.apm.network.pprof.v10.PprofTaskGrpc; +import org.apache.skywalking.apm.network.pprof.v10.PprofCollectionResponse; +import org.apache.skywalking.apm.network.pprof.v10.PprofMetaData; + +import org.apache.skywalking.apm.network.common.v3.Commands; +import org.apache.skywalking.apm.network.pprof.v10.PprofTaskCommandQuery; + +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.command.CommandService; +import java.util.Objects; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.cache.PprofTaskCache; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskLogRecord; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import java.util.concurrent.TimeUnit; + +//import org.apache.skywalking.oap.server.core.storage.StorageModule; +//import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.server.grpc.GRPCHandler; +import org.apache.skywalking.oap.server.network.trace.component.command.PprofTaskCommand; +import org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream.PprofByteBufCollectionObserver; +import org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream.PprofCollectionMetaData; +import org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream.PprofFileCollectionObserver; + +@Slf4j +public class PprofServiceHandler extends PprofTaskGrpc.PprofTaskImplBase implements GRPCHandler { + + private final IPprofTaskQueryDAO taskDAO; + private final SourceReceiver sourceReceiver; + private final CommandService commandService; + private final PprofTaskCache taskCache; + private final int pprofMaxSize; + private final boolean memoryParserEnabled; + + public PprofServiceHandler(ModuleManager moduleManager, int pprofMaxSize, boolean memoryParserEnabled) { + this.taskDAO = moduleManager.find(StorageModule.NAME).provider().getService(IPprofTaskQueryDAO.class); + this.commandService = moduleManager.find(CoreModule.NAME).provider().getService(CommandService.class); + this.sourceReceiver = moduleManager.find(CoreModule.NAME).provider().getService(SourceReceiver.class); + this.taskCache = moduleManager.find(CoreModule.NAME).provider().getService(PprofTaskCache.class); + this.pprofMaxSize = pprofMaxSize; + this.memoryParserEnabled = memoryParserEnabled; + } + + @Override + public StreamObserver collect(StreamObserver responseObserver) { + return memoryParserEnabled ? + new PprofByteBufCollectionObserver(taskDAO, responseObserver, sourceReceiver, pprofMaxSize) + : new PprofFileCollectionObserver(taskDAO, responseObserver, sourceReceiver, pprofMaxSize); + } + + @Override + public void getPprofTaskCommands(PprofTaskCommandQuery request, StreamObserver responseObserver) { + String serviceId = IDManager.ServiceID.buildId(request.getService(), true); + String serviceInstanceId = IDManager.ServiceInstanceID.buildId(serviceId, request.getServiceInstance()); + PprofTask task = taskCache.getPprofTask(serviceId); + // if task is null or createTime is less than lastCommandTime, return empty commands + if (Objects.isNull(task) || task.getCreateTime() <= request.getLastCommandTime() || + (!CollectionUtils.isEmpty(task.getServiceInstanceIds()) && !task.getServiceInstanceIds().contains(serviceInstanceId))) { + responseObserver.onNext(Commands.newBuilder().build()); + responseObserver.onCompleted(); + return; + } + + PprofTaskCommand pprofTaskCommand = commandService.newPprofTaskCommand(task); + Commands commands = Commands.newBuilder().addCommands(pprofTaskCommand.serialize()).build(); + responseObserver.onNext(commands); + responseObserver.onCompleted(); + recordPprofTaskLog(task, serviceInstanceId, PprofTaskLogOperationType.NOTIFIED); + } + + public static void recordPprofTaskLog(PprofTask task, String instanceId, PprofTaskLogOperationType operationType) { + PprofTaskLogRecord logRecord = new PprofTaskLogRecord(); + logRecord.setTaskId(task.getId()); + logRecord.setInstanceId(instanceId); + logRecord.setOperationType(operationType.getCode()); + logRecord.setOperationTime(System.currentTimeMillis()); + long timestamp = task.getCreateTime() + TimeUnit.SECONDS.toMillis(task.getDuration()); + logRecord.setTimestamp(timestamp); + logRecord.setTimeBucket(TimeBucket.getRecordTimeBucket(timestamp)); + RecordStreamProcessor.getInstance().in(logRecord); + } + + public static PprofCollectionMetaData parseMetaData(PprofMetaData metaData, IPprofTaskQueryDAO taskDAO) throws IOException { + String taskId = metaData.getTaskId(); + PprofTask task = taskDAO.getById(taskId); + String serviceId = IDManager.ServiceID.buildId(metaData.getService(), true); + String serviceInstanceId = IDManager.ServiceInstanceID.buildId(serviceId, metaData.getServiceInstance()); + + return PprofCollectionMetaData.builder() + .task(task) + .serviceId(serviceId) + .instanceId(serviceInstanceId) + .type(metaData.getType()) + .contentSize(metaData.getContentSize()) + .uploadTime(System.currentTimeMillis()) + .build(); + } +} \ No newline at end of file diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java new file mode 100644 index 000000000000..5b9f10abba0e --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream; + +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.library.pprof.parser.PprofParser; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; +import org.apache.skywalking.apm.network.pprof.v10.PprofData; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.apm.network.pprof.v10.PprofCollectionResponse; +import org.apache.skywalking.apm.network.pprof.v10.PprofProfilingStatus; +import org.apache.skywalking.oap.server.core.source.PprofProfilingData; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import static org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler.parseMetaData; +import static org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler.recordPprofTaskLog; + +@Slf4j +public class PprofByteBufCollectionObserver implements StreamObserver { + private final IPprofTaskQueryDAO taskDAO; + private final StreamObserver responseObserver; + private final SourceReceiver sourceReceiver; + private final int pprofMaxSize; + private PprofCollectionMetaData taskMetaData; + private ByteBuffer buf; + + public PprofByteBufCollectionObserver(IPprofTaskQueryDAO taskDAO, + StreamObserver responseObserver, + SourceReceiver sourceReceiver, int pprofMaxSize) { + this.taskDAO = taskDAO; + this.responseObserver = responseObserver; + this.sourceReceiver = sourceReceiver; + this.pprofMaxSize = pprofMaxSize; + } + + @Override + public void onNext(PprofData pprofData) { + try { + if (Objects.isNull(taskMetaData) && pprofData.hasMetadata()) { + taskMetaData = parseMetaData(pprofData.getMetadata(), taskDAO); + if (PprofProfilingStatus.PPROF_PROFILING_SUCCESS.equals(taskMetaData.getType())) { + int size = taskMetaData.getContentSize(); + log.info("pprofMaxSize: {}, Pprof data size: {}", pprofMaxSize, size); + if (pprofMaxSize >= size) { + buf = ByteBuffer.allocate(size); + // Send success response to allow client to continue uploading + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus(PprofProfilingStatus.PPROF_PROFILING_SUCCESS) + .build()); + + log.info("Started collecting pprof data in memory - service: {}, serviceInstance: {}, size: {} bytes", + pprofData.getMetadata().getService(), pprofData.getMetadata().getServiceInstance(), size); + } else { + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus(PprofProfilingStatus.PPROF_TERMINATED_BY_OVERSIZE) + .build()); + recordPprofTaskLog(taskMetaData.getTask(), taskMetaData.getInstanceId(), PprofTaskLogOperationType.PPROF_UPLOAD_FILE_TOO_LARGE_ERROR); + log.warn("Pprof file size {} exceeds maximum allowed size {} for service: {}, serviceInstance: {}", + size, pprofMaxSize, pprofData.getMetadata().getService(), pprofData.getMetadata().getServiceInstance()); + } + } else { + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus(PprofProfilingStatus.PPROF_EXECUTION_TASK_ERROR) + .build()); + recordPprofTaskLog(taskMetaData.getTask(), taskMetaData.getInstanceId(), PprofTaskLogOperationType.EXECUTION_TASK_ERROR); + log.error("Received execution error from agent - service: {}, serviceInstance: {}, status: {}", + pprofData.getMetadata().getService(), pprofData.getMetadata().getServiceInstance(), taskMetaData.getType()); + } + } else if (pprofData.hasContent()) { + if (buf != null) { + pprofData.getContent().copyTo(buf); + log.info("Received {} bytes of pprof data", pprofData.getContent().size()); + } + } + } catch (IOException e) { + log.error("Error processing pprof data", e); + responseObserver.onError(Status.INTERNAL.withDescription("Error processing pprof data: " + e.getMessage()).asRuntimeException()); + } + } + + @Override + public void onError(Throwable throwable) { + Status status = Status.fromThrowable(throwable); + if (Status.CANCELLED.getCode() == status.getCode()) { + if (log.isDebugEnabled()) { + log.debug(throwable.getMessage(), throwable); + } + return; + } + log.error("Error in receiving pprof profiling data", throwable); + + } + + @Override + public void onCompleted() { + responseObserver.onCompleted(); + if (Objects.nonNull(buf)) { + buf.flip(); + try { + parseAndStorageData(taskMetaData, buf); + } catch (IOException e) { + log.error("Failed to parse and store pprof data", e); + } + } + } + + private void parseAndStorageData(PprofCollectionMetaData taskMetaData, ByteBuffer buf) throws IOException { + PprofTask task = taskMetaData.getTask(); + if (task == null) { + log.error("Pprof instanceId:{} has not been assigned a task but still uploaded data", taskMetaData.getInstanceId()); + return; + } + recordPprofTaskLog(task, taskMetaData.getInstanceId(), PprofTaskLogOperationType.EXECUTION_FINISHED); + parsePprofAndStorage(taskMetaData, buf); + } + + public void parsePprofAndStorage(PprofCollectionMetaData taskMetaData, ByteBuffer buf) throws IOException { + log.info("Parsing pprof file for service: {}, instance: {}", + taskMetaData.getServiceId(), taskMetaData.getInstanceId()); + PprofTask task = taskMetaData.getTask(); + FrameTree tree = PprofParser.dumpTree(buf); + PprofProfilingData data = new PprofProfilingData(); + data.setEventType(PprofEventType.valueOfString(task.getEvents().name())); + data.setFrameTree(tree); + data.setTaskId(task.getId()); + data.setInstanceId(taskMetaData.getInstanceId()); + data.setUploadTime(taskMetaData.getUploadTime()); + log.info("data eventType: {}", data.getEventType()); + log.info("data frameTree: {}", tree); + log.info("data taskId: {}", task.getId()); + log.info("data instanceId: {}", taskMetaData.getInstanceId()); + log.info("data uploadTime: {}", taskMetaData.getUploadTime()); + sourceReceiver.receive(data); + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofCollectionMetaData.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofCollectionMetaData.java new file mode 100644 index 000000000000..b0986361d97c --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofCollectionMetaData.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream; + +import org.apache.skywalking.oap.server.core.query.type.PprofTask; + +import org.apache.skywalking.apm.network.pprof.v10.PprofProfilingStatus; +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class PprofCollectionMetaData { + private PprofTask task; + private String serviceId; + private String instanceId; + private int contentSize; + private PprofProfilingStatus type; + private long uploadTime; +} diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java new file mode 100644 index 000000000000..a5cd831e9685 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream; + +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; +import lombok.SneakyThrows; +import org.apache.skywalking.apm.network.pprof.v10.PprofData; +import org.apache.skywalking.apm.network.pprof.v10.PprofCollectionResponse; +import org.apache.skywalking.apm.network.pprof.v10.PprofProfilingStatus; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.source.PprofProfilingData; +import org.apache.skywalking.oap.server.library.pprof.parser.PprofParser; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import static org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler.recordPprofTaskLog; +import static org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler.parseMetaData; + +@Slf4j +public class PprofFileCollectionObserver implements StreamObserver { + private final IPprofTaskQueryDAO taskDAO; + private final StreamObserver responseObserver; + private final SourceReceiver sourceReceiver; + private final int pprofMaxSize; + private PprofCollectionMetaData taskMetaData; + private Path tempFile; + private FileOutputStream fileOutputStream; + + public PprofFileCollectionObserver(IPprofTaskQueryDAO taskDAO, + StreamObserver responseObserver, + SourceReceiver sourceReceiver, int pprofMaxSize) { + this.taskDAO = taskDAO; + this.responseObserver = responseObserver; + this.sourceReceiver = sourceReceiver; + this.pprofMaxSize = pprofMaxSize; + } + + @SneakyThrows + @Override + public void onNext(PprofData pprofData) { + if (Objects.isNull(taskMetaData) && pprofData.hasMetadata()) { + taskMetaData = parseMetaData(pprofData.getMetadata(), taskDAO); + + if (PprofProfilingStatus.PPROF_PROFILING_SUCCESS.equals(taskMetaData.getType())) { + int size = taskMetaData.getContentSize(); + if (pprofMaxSize >= size) { + // Create temporary file for pprof data + tempFile = Files.createTempFile(taskMetaData.getTask().getId() + taskMetaData.getInstanceId() + System.currentTimeMillis(), ".pprof"); + fileOutputStream = new FileOutputStream(tempFile.toFile()); + + // Send success response to allow client to continue uploading + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus(PprofProfilingStatus.PPROF_PROFILING_SUCCESS) + .build()); + } else { + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus(PprofProfilingStatus.PPROF_TERMINATED_BY_OVERSIZE) + .build()); + recordPprofTaskLog(taskMetaData.getTask(), taskMetaData.getInstanceId(), PprofTaskLogOperationType.PPROF_UPLOAD_FILE_TOO_LARGE_ERROR); + } + } else { + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus(PprofProfilingStatus.PPROF_EXECUTION_TASK_ERROR) + .build()); + recordPprofTaskLog(taskMetaData.getTask(), taskMetaData.getInstanceId(), PprofTaskLogOperationType.EXECUTION_TASK_ERROR); + } + } else if (pprofData.hasContent()) { + if (fileOutputStream != null) { + fileOutputStream.write(pprofData.getContent().toByteArray()); + + if (log.isDebugEnabled()) { + log.debug("Received {} bytes of pprof data", pprofData.getContent().size()); + } + } + } + } + + @Override + public void onError(Throwable throwable) { + Status status = Status.fromThrowable(throwable); + if (Status.CANCELLED.getCode() == status.getCode()) { + if (log.isDebugEnabled()) { + log.debug("Pprof data collection cancelled: {}", throwable.getMessage()); + } + } else { + log.error("Error in receiving pprof profiling data", throwable); + } + + // Clean up resources + closeFileStream(); + } + + @Override + @SneakyThrows + public void onCompleted() { + responseObserver.onCompleted(); + + if (Objects.nonNull(tempFile)) { + closeFileStream(); + parseAndStorageData(taskMetaData, tempFile.toAbsolutePath().toString()); + } + } + + private void closeFileStream() { + if (fileOutputStream != null) { + try { + fileOutputStream.close(); + fileOutputStream = null; + } catch (IOException e) { + log.error("Failed to close file output stream", e); + } + } + } + + @SneakyThrows + private void parseAndStorageData(PprofCollectionMetaData taskMetaData, String fileName) { + PprofTask task = taskMetaData.getTask(); + if (task == null) { + log.error("Pprof instanceId:{} has not been assigned a task but still uploaded data", taskMetaData.getInstanceId()); + return; + } + recordPprofTaskLog(task, taskMetaData.getInstanceId(), PprofTaskLogOperationType.EXECUTION_FINISHED); + parsePprofAndStorage(taskMetaData, fileName); + } + + public void parsePprofAndStorage(PprofCollectionMetaData taskMetaData, + String fileName) throws IOException { + log.info("Parsing pprof file for service: {}, instance: {}", + taskMetaData.getServiceId(), taskMetaData.getInstanceId()); + PprofTask task = taskMetaData.getTask(); + FrameTree tree = PprofParser.dumpTree(fileName); + PprofProfilingData data = new PprofProfilingData(); + data.setEventType(PprofEventType.valueOfString(task.getEvents().name())); + data.setFrameTree(tree); + data.setTaskId(task.getId()); + data.setInstanceId(taskMetaData.getInstanceId()); + data.setUploadTime(taskMetaData.getUploadTime()); + log.info("data eventType: {}", data.getEventType()); + log.info("data frameTree: {}", tree); + log.info("data taskId: {}", task.getId()); + log.info("data instanceId: {}", taskMetaData.getInstanceId()); + log.info("data uploadTime: {}", taskMetaData.getUploadTime()); + sourceReceiver.receive(data); + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine new file mode 100644 index 000000000000..3ef9c8a560ef --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.oap.server.receiver.pprof.module.PprofModule diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider new file mode 100644 index 000000000000..e4e90929ba62 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.oap.server.receiver.pprof.provider.PprofModuleProvider diff --git a/oap-server/server-receiver-plugin/skywalking-sharing-server-plugin/pom.xml b/oap-server/server-receiver-plugin/skywalking-sharing-server-plugin/pom.xml index 5cf914ba25c2..19b45143d0ea 100644 --- a/oap-server/server-receiver-plugin/skywalking-sharing-server-plugin/pom.xml +++ b/oap-server/server-receiver-plugin/skywalking-sharing-server-plugin/pom.xml @@ -27,4 +27,12 @@ skywalking-sharing-server-plugin jar + + + + org.apache.skywalking + library-server + ${project.version} + + \ No newline at end of file diff --git a/oap-server/server-starter/pom.xml b/oap-server/server-starter/pom.xml index d1971572aa90..7d44453471ae 100644 --- a/oap-server/server-starter/pom.xml +++ b/oap-server/server-starter/pom.xml @@ -176,6 +176,11 @@ skywalking-async-profiler-receiver-plugin ${project.version} + + org.apache.skywalking + skywalking-pprof-receiver-plugin + ${project.version} + org.apache.skywalking skywalking-telegraf-receiver-plugin diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index 30c7f4d2fa58..50b66c9c8de9 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -151,7 +151,7 @@ storage: # Since 10.2.0, the banyandb configuration is separated to an independent configuration file: `bydb.yaml`. elasticsearch: namespace: ${SW_NAMESPACE:""} - clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:localhost:9200} + clusterNodes: 192.168.50.242:9200 protocol: ${SW_STORAGE_ES_HTTP_PROTOCOL:"http"} connectTimeout: ${SW_STORAGE_ES_CONNECT_TIMEOUT:3000} socketTimeout: ${SW_STORAGE_ES_SOCKET_TIMEOUT:30000} @@ -298,6 +298,20 @@ receiver-async-profiler: # It is recommended to use physical file mode when volume mounting is used or the tmp directory has sufficient storage. memoryParserEnabled: ${SW_RECEIVER_ASYNC_PROFILER_MEMORY_PARSER_ENABLED:true} +receiver-pprof: + selector: ${SW_RECEIVER_PPROF:default} + default: + # Used to manage the maximum size of the pprof file that can be received, the unit is Byte, default is 30M + pprofMaxSize: ${SW_RECEIVER_PPROF_MAX_SIZE:31457280} + # Used to determine whether to receive pprof in memory file or physical file mode + # + # The memory file mode have fewer local file system limitations, so they are by default. But it costs more memory. + # + # The physical file mode will use less memory when parsing and is more friendly to parsing large files. + # However, if the storage of the tmp directory in the container is insufficient, the oap server instance may crash. + # It is recommended to use physical file mode when volume mounting is used or the tmp directory has sufficient storage. + memoryParserEnabled: ${SW_RECEIVER_PPROF_MEMORY_PARSER_ENABLED:true} + receiver-zabbix: selector: ${SW_RECEIVER_ZABBIX:-} default: diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java index e32891c3e472..0f533990ea02 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java @@ -97,6 +97,8 @@ public static class Global { */ private int asyncProfilerTaskQueryMaxSize; + private int pprofTaskQueryMaxSize; + private int resultWindowMaxSize = 10000; private int metadataQueryMaxSize = 5000; private int segmentQueryMaxSize = 200; diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java index 62f6dea5ba15..d1f58d35bff1 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java @@ -40,6 +40,9 @@ import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IJFRDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.continuous.IContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingDataDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingScheduleDAO; @@ -80,6 +83,9 @@ import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBAlarmQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBAsyncProfilerTaskQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBPprofDataQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBPprofTaskQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBBrowserLogQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBEBPFProfilingDataDAO; @@ -199,6 +205,17 @@ IAsyncProfilerTaskLogQueryDAO.class, new BanyanDBAsyncProfilerTaskLogQueryDAO(cl this.config.getGlobal().getAsyncProfilerTaskQueryMaxSize() )); this.registerServiceImplementation(IJFRDataQueryDAO.class, new BanyanDBJFRDataQueryDAO(client)); + this.registerServiceImplementation( + IPprofTaskQueryDAO.class, new BanyanDBPprofTaskQueryDAO(client, + this.config.getGlobal().getPprofTaskQueryMaxSize() + )); + this.registerServiceImplementation( + IPprofTaskLogQueryDAO.class, new BanyanDBPprofTaskLogQueryDAO(client, + this.config.getGlobal().getPprofTaskQueryMaxSize() + )); + this.registerServiceImplementation( + IPprofDataQueryDAO.class, new BanyanDBPprofDataQueryDAO(client) + ); this.registerServiceImplementation( StorageTTLStatusQuery.class, new BanyanDBTTLStatusQuery(config) diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java new file mode 100644 index 000000000000..7d03db2f5aac --- /dev/null +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java @@ -0,0 +1,59 @@ +package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream; + +import com.google.common.collect.ImmutableSet; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; +import java.util.Set; +import java.util.List; +import java.io.IOException; +import java.util.ArrayList; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient; +import org.apache.skywalking.banyandb.v1.client.StreamQuery; +import org.apache.skywalking.banyandb.v1.client.StreamQueryResponse; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.banyandb.v1.client.RowEntity; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBConverter; + +public class BanyanDBPprofDataQueryDAO extends AbstractBanyanDBDAO implements IPprofDataQueryDAO { + private static final Set TAGS = ImmutableSet.of( + PprofProfilingDataRecord.TASK_ID, + PprofProfilingDataRecord.INSTANCE_ID, + PprofProfilingDataRecord.EVENT_TYPE, + PprofProfilingDataRecord.UPLOAD_TIME, + PprofProfilingDataRecord.DATA_BINARY + ); + + public BanyanDBPprofDataQueryDAO(BanyanDBStorageClient client) { + super(client); + } + + @Override + public List getByTaskIdAndInstances(String taskId, List instanceIds) throws IOException { + if (StringUtil.isBlank(taskId)) { + return new ArrayList<>(); + } + StreamQueryResponse resp = query(false, PprofProfilingDataRecord.INDEX_NAME, TAGS, + new QueryBuilder() { + @Override + protected void apply(StreamQuery query) { + query.and(eq(PprofProfilingDataRecord.TASK_ID, taskId)); + if (CollectionUtils.isNotEmpty(instanceIds)) { + query.and(in(PprofProfilingDataRecord.INSTANCE_ID, instanceIds)); + } + } + }); + List records = new ArrayList<>(resp.size()); + for (final RowEntity entity : resp.getElements()) { + records.add(buildProfilingDataRecord(entity)); + } + + return records; + } + + private PprofProfilingDataRecord buildProfilingDataRecord(RowEntity entity) { + final PprofProfilingDataRecord.Builder builder = new PprofProfilingDataRecord.Builder(); + BanyanDBConverter.StorageToStream storageToStream = new BanyanDBConverter.StorageToStream(PprofProfilingDataRecord.INDEX_NAME, entity); + return builder.storage2Entity(storageToStream); + } +} diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskLogQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskLogQueryDAO.java new file mode 100644 index 000000000000..cfd9d4d42daf --- /dev/null +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskLogQueryDAO.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream; + +import com.google.common.collect.ImmutableSet; +import org.apache.skywalking.banyandb.v1.client.Element; +import org.apache.skywalking.banyandb.v1.client.StreamQuery; +import org.apache.skywalking.banyandb.v1.client.StreamQueryResponse; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskLogRecord; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +/** + * {@link PprofTaskLogRecord} is a stream + */ +public class BanyanDBPprofTaskLogQueryDAO extends AbstractBanyanDBDAO implements IPprofTaskLogQueryDAO { + private static final Set TAGS = ImmutableSet.of( + PprofTaskLogRecord.OPERATION_TIME, + PprofTaskLogRecord.INSTANCE_ID, + PprofTaskLogRecord.TASK_ID, + PprofTaskLogRecord.OPERATION_TYPE + ); + + private final int queryMaxSize; + + public BanyanDBPprofTaskLogQueryDAO(BanyanDBStorageClient client, int taskQueryMaxSize) { + super(client); + // query log size use pprof task query max size * per log count + this.queryMaxSize = taskQueryMaxSize * 50; + } + + @Override + public List getTaskLogList() throws IOException { + StreamQueryResponse resp = query(false, PprofTaskLogRecord.INDEX_NAME, TAGS, + new QueryBuilder() { + @Override + public void apply(StreamQuery query) { + query.setLimit(BanyanDBPprofTaskLogQueryDAO.this.queryMaxSize); + } + }); + + final LinkedList tasks = new LinkedList<>(); + for (final Element element : resp.getElements()) { + tasks.add(buildPprofTaskLog(element)); + } + return tasks; + } + + private PprofTaskLog buildPprofTaskLog(Element data) { + int operationTypeInt = ((Number) data.getTagValue(PprofTaskLogRecord.OPERATION_TYPE)).intValue(); + PprofTaskLogOperationType operationType = PprofTaskLogOperationType.parse(operationTypeInt); + return PprofTaskLog.builder() + .id(data.getTagValue(PprofTaskLogRecord.TASK_ID)) + .instanceId(data.getTagValue(PprofTaskLogRecord.INSTANCE_ID)) + .operationType(operationType) + .operationTime(((Number) data.getTagValue(PprofTaskLogRecord.OPERATION_TIME)).longValue()) + .build(); + } +} diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java new file mode 100644 index 000000000000..58488382f6f4 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream; + +import com.google.common.collect.ImmutableSet; +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import org.apache.skywalking.banyandb.v1.client.AbstractQuery; +import org.apache.skywalking.banyandb.v1.client.RowEntity; +import org.apache.skywalking.banyandb.v1.client.StreamQuery; +import org.apache.skywalking.banyandb.v1.client.StreamQueryResponse; +import org.apache.skywalking.banyandb.v1.client.TimestampRange; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient; +import lombok.extern.slf4j.Slf4j; +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +@Slf4j +public class BanyanDBPprofTaskQueryDAO extends AbstractBanyanDBDAO implements IPprofTaskQueryDAO { + private static final Gson GSON = new Gson(); + private static final Set TAGS = ImmutableSet.of( + PprofTaskRecord.SERVICE_ID, + PprofTaskRecord.SERVICE_INSTANCE_IDS, + PprofTaskRecord.TASK_ID, + PprofTaskRecord.CREATE_TIME, + PprofTaskRecord.EVENT_TYPES, + PprofTaskRecord.DURATION, + PprofTaskRecord.DUMP_PERIOD + ); + + private final int queryMaxSize; + + public BanyanDBPprofTaskQueryDAO(BanyanDBStorageClient client, int queryMaxSize) { + super(client); + this.queryMaxSize = queryMaxSize; + } + + @Override + public List getTaskList(String serviceId, Long startTimeBucket, Long endTimeBucket, Integer limit) throws IOException { + long startTS = LOWER_BOUND_TIME; + long endTS = UPPER_BOUND_TIME; + if (startTimeBucket != null) { + startTS = TimeBucket.getTimestamp(startTimeBucket); + } + if (endTimeBucket != null) { + endTS = TimeBucket.getTimestamp(endTimeBucket); + } + StreamQueryResponse resp = query(false, PprofTaskRecord.INDEX_NAME, TAGS, new TimestampRange(startTS, endTS), + new QueryBuilder() { + @Override + protected void apply(StreamQuery query) { + if (StringUtil.isNotEmpty(serviceId)) { + query.and(eq(PprofTaskRecord.SERVICE_ID, serviceId)); + } + + if (limit != null) { + query.setLimit(limit); + } else { + query.setLimit(BanyanDBPprofTaskQueryDAO.this.queryMaxSize); + } + query.setOrderBy(new AbstractQuery.OrderBy(AbstractQuery.Sort.DESC)); + } + }); + + List tasks = new ArrayList<>(resp.size()); + for (final RowEntity entity : resp.getElements()) { + tasks.add(buildPprofTask(entity)); + } + return tasks; + } + + @Override + public PprofTask getById(String id) throws IOException { + StreamQueryResponse resp = query(false, PprofTaskRecord.INDEX_NAME, TAGS, + new QueryBuilder() { + @Override + protected void apply(StreamQuery query) { + if (StringUtil.isNotEmpty(id)) { + query.and(eq(PprofTaskRecord.TASK_ID, id)); + } + query.setLimit(1); + } + }); + + if (resp.size() == 0) { + return null; + } + + return buildPprofTask(resp.getElements().get(0)); + } + + private PprofTask buildPprofTask(RowEntity data) { + Type listType = new TypeToken>() { + }.getType(); + + String serviceInstanceIds = data.getTagValue(PprofTaskRecord.SERVICE_INSTANCE_IDS); + List serviceInstanceIdList = GSON.fromJson(serviceInstanceIds, listType); + + // Convert string events to PprofEventType enum + String eventsStr = data.getTagValue(PprofTaskRecord.EVENT_TYPES); + PprofEventType eventType = null; + if (StringUtil.isNotEmpty(eventsStr)) { + try { + eventType = PprofEventType.valueOfString(eventsStr); + } catch (Exception e) { + // Default to CPU if conversion fails + eventType = PprofEventType.CPU; + log.warn("Failed to parse pprof event type: {}, using CPU as default", eventsStr, e); + } + } + + return PprofTask.builder() + .id(data.getTagValue(PprofTaskRecord.TASK_ID)) + .serviceId(data.getTagValue(PprofTaskRecord.SERVICE_ID)) + .serviceInstanceIds(serviceInstanceIdList) + .createTime(((Number) data.getTagValue(PprofTaskRecord.CREATE_TIME)).longValue()) + .events(eventType) + .duration(((Number) data.getTagValue(PprofTaskRecord.DURATION)).intValue()) + .dumpPeriod(((Number) data.getTagValue(PprofTaskRecord.DUMP_PERIOD)).intValue()) + .build(); + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java index cd255d99b6cf..cfbd84c4cb13 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java @@ -39,6 +39,8 @@ import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IJFRDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.continuous.IContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingDataDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingScheduleDAO; @@ -81,6 +83,7 @@ import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.AlarmQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.AsyncProfilerTaskLogQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.AsyncProfilerTaskQueryEsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.PprofTaskLogQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.BrowserLogQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ContinuousProfilingPolicyEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.EBPFProfilingDataEsDAO; @@ -95,6 +98,7 @@ import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ProfileTaskLogEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ProfileTaskQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ProfileThreadSnapshotQueryEsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.PprofTaskQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.RecordsQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ServiceLabelEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.SpanAttachedEventEsDAO; @@ -271,10 +275,18 @@ IProfileThreadSnapshotQueryDAO.class, new ProfileThreadSnapshotQueryEsDAO(elasti IAsyncProfilerTaskLogQueryDAO.class, new AsyncProfilerTaskLogQueryEsDAO(elasticSearchClient, config.getAsyncProfilerTaskQueryMaxSize()) ); + this.registerServiceImplementation( + IPprofTaskLogQueryDAO.class, + new PprofTaskLogQueryEsDAO(elasticSearchClient, config.getAsyncProfilerTaskQueryMaxSize()) + ); this.registerServiceImplementation( IJFRDataQueryDAO.class, new JFRDataQueryEsDAO(elasticSearchClient) ); + this.registerServiceImplementation( + IPprofTaskQueryDAO.class, + new PprofTaskQueryEsDAO(elasticSearchClient, config.getAsyncProfilerTaskQueryMaxSize()) + ); this.registerServiceImplementation( StorageTTLStatusQuery.class, new DefaultStorageTTLStatusQuery() diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java new file mode 100644 index 000000000000..4d9ce9a5e51d --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import org.apache.skywalking.library.elasticsearch.requests.search.Query; +import org.apache.skywalking.library.elasticsearch.requests.search.Search; +import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Sort; +import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; +import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskLogRecord; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; + +public class PprofTaskLogQueryEsDAO extends EsDAO implements IPprofTaskLogQueryDAO { + + private final int queryMaxSize; + + public PprofTaskLogQueryEsDAO(ElasticSearchClient client, int profileTaskQueryMaxSize) { + super(client); + // query log size use pprof task query max size * per log count + this.queryMaxSize = profileTaskQueryMaxSize * 50; + } + + @Override + public List getTaskLogList() throws IOException { + final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofTaskLogRecord.INDEX_NAME); + + final SearchBuilder search = Search.builder().query(Query.bool()); + if (IndexController.LogicIndicesRegister.isMergedTable(PprofTaskLogRecord.INDEX_NAME)) { + search.query(Query.bool().must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofTaskLogRecord.INDEX_NAME))); + } + + search.size(queryMaxSize); + search.sort(PprofTaskLogRecord.OPERATION_TIME, Sort.Order.DESC); + + final SearchResponse response = getClient().search(index, search.build()); + + List tasks = new LinkedList<>(); + for (SearchHit hit : response.getHits().getHits()) { + tasks.add(parseTaskLog(hit)); + } + return tasks; + } + + private PprofTaskLog parseTaskLog(SearchHit data) { + Map source = data.getSource(); + + int operationTypeInt = ((Number) source.get(PprofTaskLogRecord.OPERATION_TYPE)).intValue(); + PprofTaskLogOperationType operationType = PprofTaskLogOperationType.parse(operationTypeInt); + + return PprofTaskLog.builder() + .id((String) source.get(PprofTaskLogRecord.TASK_ID)) + .instanceId((String) source.get(PprofTaskLogRecord.INSTANCE_ID)) + .operationType(operationType) + .operationTime(((Number) source.get(PprofTaskLogRecord.OPERATION_TIME)).longValue()) + .build(); + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java new file mode 100644 index 000000000000..e93afbfbdef1 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.lang.reflect.Type; + +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Query; +import org.apache.skywalking.library.elasticsearch.requests.search.Search; +import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Sort; +import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; +import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; + +public class PprofTaskQueryEsDAO extends EsDAO implements IPprofTaskQueryDAO { + private static final Gson GSON = new Gson(); + + private final int queryMaxSize; + + public PprofTaskQueryEsDAO(ElasticSearchClient client, int queryMaxSize) { + super(client); + this.queryMaxSize = queryMaxSize; + } + + @Override + public List getTaskList(String serviceId, Long startTimeBucket, Long endTimeBucket, Integer limit) throws IOException { + final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofTaskRecord.INDEX_NAME); + final BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isMergedTable(PprofTaskRecord.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofTaskRecord.INDEX_NAME)); + } + + if (StringUtil.isNotEmpty(serviceId)) { + query.must(Query.term(PprofTaskRecord.SERVICE_ID, serviceId)); + } + + if (startTimeBucket != null) { + query.must(Query.range(PprofTaskRecord.TIME_BUCKET).gte(startTimeBucket)); + } + + if (endTimeBucket != null) { + query.must(Query.range(PprofTaskRecord.TIME_BUCKET).lte(endTimeBucket)); + } + + final SearchBuilder search = Search.builder().query(query); + + if (limit != null) { + search.size(limit); + } else { + search.size(queryMaxSize); + } + + search.sort(PprofTaskRecord.CREATE_TIME, Sort.Order.DESC); + + final SearchResponse response = getClient().search(index, search.build()); + + List tasks = new LinkedList<>(); + for (SearchHit hit : response.getHits().getHits()) { + tasks.add(parseTask(hit)); + } + return tasks; + } + + @Override + public PprofTask getById(String id) throws IOException { + if (StringUtil.isEmpty(id)) { + return null; + } + final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofTaskRecord.INDEX_NAME); + final BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isMergedTable(PprofTaskRecord.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofTaskRecord.INDEX_NAME)); + } + query.must(Query.term(PprofTaskRecord.TASK_ID, id)); + + final SearchBuilder search = Search.builder().query(query).size(1); + final SearchResponse response = getClient().search(index, search.build()); + + if (!response.getHits().getHits().isEmpty()) { + return parseTask(response.getHits().getHits().iterator().next()); + } + return null; + } + + private PprofTask parseTask(SearchHit data) { + Map source = data.getSource(); + Type listType = new TypeToken>() { + }.getType(); + + String serviceInstanceIds = (String) source.get(PprofTaskRecord.SERVICE_INSTANCE_IDS); + + List instanceIdList = GSON.fromJson(serviceInstanceIds, listType); + + // Convert string events to PprofEventType enum + String eventsStr = (String) source.get(PprofTaskRecord.EVENT_TYPES); + PprofEventType eventType = null; + if (StringUtil.isNotEmpty(eventsStr)) { + try { + eventType = PprofEventType.valueOfString(eventsStr); + } catch (Exception e) { + // Default to CPU if conversion fails + eventType = PprofEventType.CPU; + } + } + + return PprofTask.builder() + .id((String) source.get(PprofTaskRecord.TASK_ID)) + .serviceId((String) source.get(PprofTaskRecord.SERVICE_ID)) + .serviceInstanceIds(instanceIdList) + .createTime(((Number) source.get(PprofTaskRecord.CREATE_TIME)).longValue()) + .startTime(((Number) source.get(PprofTaskRecord.START_TIME)).longValue()) + .events(eventType) + .duration(((Number) source.get(PprofTaskRecord.DURATION)).intValue()) + .dumpPeriod(((Number) source.get(PprofTaskRecord.DUMP_PERIOD)).intValue()) + .build(); + } +} From 6b4640f81cf97b5e0c34b9630d95b6a53c0ebc55 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Thu, 18 Sep 2025 15:08:22 +0800 Subject: [PATCH 02/69] fix --- .../component/command/PprofTaskCommand.java | 45 +++++++++++++------ .../pprof/module/PprofModuleConfig.java | 2 +- .../src/main/resources/bydb.yml | 3 +- .../banyandb/BanyanDBStorageConfig.java | 5 ++- 4 files changed, 39 insertions(+), 16 deletions(-) diff --git a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java index 3cbf9b5f4750..87210f8685da 100644 --- a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java +++ b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package org.apache.skywalking.oap.server.network.trace.component.command; import org.apache.skywalking.apm.network.common.v3.Command; @@ -15,11 +33,22 @@ public class PprofTaskCommand extends BaseCommand implements Serializable, Deser private String taskId; // Type of profiling (CPU/Heap/Block/Mutex/Goroutine/Threadcreate/Allocs) private String events; - // unit is minute + /** + * run profiling for duration (minute) + */ private long duration; - // Unix timestamp in milliseconds when the task was created + /** + * task create time + */ private long createTime; - // + /** + * pprof dump period parameters. There are different dumpperiod configurations for different events. + * Here is a table of parameters. + * + *

For Block - sample an average of one blocking event per rate nanoseconds spent blocked. (default: 0)

+ *

For Mutex - sample an average of 1/rate events are reported. (default: 0)

+ * details @see pprof argument + */ private int dumpPeriod; public PprofTaskCommand(String serialNumber, String taskId, String events, @@ -32,16 +61,6 @@ public PprofTaskCommand(String serialNumber, String taskId, String events, this.events = events; } - // public PprofTaskCommand(String serialNumber, String taskId, - // long duration, long startTime, long createTime, int dumpPeriod) { - // super(NAME, serialNumber); - // this.taskId = taskId; - // this.duration = duration; - // this.startTime = startTime; - // this.createTime = createTime; - // this.dumpPeriod = dumpPeriod; - // } - @Override public PprofTaskCommand deserialize(Command command) { final List argsList = command.getArgsList(); diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java index 99f5b70fc078..acb92227d020 100644 --- a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java @@ -29,7 +29,7 @@ public class PprofModuleConfig extends ModuleConfig { * Used to manage the maximum size of the pprof file that can be received, the unit is Byte * default is 30M */ - private int pprofMaxSize = 4 * 1024; + private int pprofMaxSize = 30 * 1024 * 1024; /** * default is true *

diff --git a/oap-server/server-starter/src/main/resources/bydb.yml b/oap-server/server-starter/src/main/resources/bydb.yml index 75899fee8aca..48bc7d3e1b98 100644 --- a/oap-server/server-starter/src/main/resources/bydb.yml +++ b/oap-server/server-starter/src/main/resources/bydb.yml @@ -18,7 +18,7 @@ global: # Each target is a BanyanDB server in the format of `host:port`. # If BanyanDB is deployed as a standalone server, the target should be the IP address or domain name and port of the BanyanDB server. # If BanyanDB is deployed in a cluster, the targets should be the IP address or domain name and port of the `liaison` nodes, separated by commas. - targets: ${SW_STORAGE_BANYANDB_TARGETS:127.0.0.1:17912} + targets: ${SW_STORAGE_BANYANDB_TARGETS:192.168.50.128:17912} # The maximum number of records in a bulk write request. # A larger value can improve write performance but also increases OAP and BanyanDB Server memory usage. maxBulkSize: ${SW_STORAGE_BANYANDB_MAX_BULK_SIZE:10000} @@ -43,6 +43,7 @@ global: # The batch size for querying profile data. profileDataQueryBatchSize: ${SW_STORAGE_BANYANDB_QUERY_PROFILE_DATA_BATCH_SIZE:100} asyncProfilerTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_ASYNC_PROFILER_TASK_QUERY_MAX_SIZE:200} + pprofTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_PPROF_TASK_QUERY_MAX_SIZE:200} # If the BanyanDB server is configured with TLS, configure the TLS cert file path and enable TLS connection. sslTrustCAPath: ${SW_STORAGE_BANYANDB_SSL_TRUST_CA_PATH:""} # Cleanup TopN rules in BanyanDB server that are not configured in the bydb-topn.yml config. diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java index 0f533990ea02..e21039359dae 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java @@ -96,7 +96,10 @@ public static class Global { * single request. */ private int asyncProfilerTaskQueryMaxSize; - + /** + * Max size of {@link org.apache.skywalking.oap.server.core.query.type.PprofTask} to be fetched in a + * single request. + */ private int pprofTaskQueryMaxSize; private int resultWindowMaxSize = 10000; From a54d8dc8b1bf7a48a242fbe23aae80a51bb50b20 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Wed, 24 Sep 2025 20:50:37 +0800 Subject: [PATCH 03/69] fix --- .../profiling/pprof/PprofMutationService.java | 23 +++++++--- .../pprof/parser/PprofMergeBuilder.java | 36 ++++++--------- .../library/pprof/parser/PprofParser.java | 30 +++++++++++-- .../oap/server/library/pprof/type/Frame.java | 45 +++++++++++++++++++ .../oap/server/library/pprof/type/Index.java | 34 ++++++++++++++ .../StorageModuleElasticsearchConfig.java | 5 +++ .../StorageModuleElasticsearchProvider.java | 4 +- .../query/PprofTaskQueryEsDAO.java | 16 +------ 8 files changed, 143 insertions(+), 50 deletions(-) create mode 100644 oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java create mode 100644 oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Index.java diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java index c909790df570..ddfb45d5667a 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java @@ -62,7 +62,7 @@ public PprofTaskCreationResult createTask(String serviceId, long createTime = System.currentTimeMillis(); // check data PprofTaskCreationResult checkResult = checkDataSuccess( - serviceId, serviceInstanceIds, duration, createTime, events + serviceId, serviceInstanceIds, duration, createTime, events, dumpPeriod ); if (checkResult != null) { return checkResult; @@ -90,8 +90,9 @@ private PprofTaskCreationResult checkDataSuccess(String serviceId, List serviceInstanceIds, int duration, long createTime, - PprofEventType events) throws IOException { - String checkArgumentMessage = checkArgumentError(serviceId, serviceInstanceIds, duration, events); + PprofEventType events, + int dumpPeriod) throws IOException { + String checkArgumentMessage = checkArgumentError(serviceId, serviceInstanceIds, duration, events, dumpPeriod); if (checkArgumentMessage != null) { return PprofTaskCreationResult.builder() .code(PprofTaskCreationType.ARGUMENT_ERROR) @@ -111,16 +112,24 @@ private PprofTaskCreationResult checkDataSuccess(String serviceId, private String checkArgumentError(String serviceId, List serviceInstanceIds, int duration, - PprofEventType events) { + PprofEventType events, + int dumpPeriod) { if (serviceId == null) { return "service cannot be null"; } - if (duration <= 0) { - return "duration cannot be negative"; - } if (events == null) { return "events cannot be empty"; } + if (events == PprofEventType.CPU || events == PprofEventType.BLOCK || events == PprofEventType.MUTEX) { + if (duration <= 0) { + return "duration cannot be negative"; + } + } + if (events == PprofEventType.BLOCK || events == PprofEventType.MUTEX) { + if (dumpPeriod <= 0) { + return "dumpPeriod cannot be negative"; + } + } if (CollectionUtils.isEmpty(serviceInstanceIds)) { return "serviceInstanceIds cannot be empty"; } diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java index 20b2cf57570b..6937c3416eb9 100644 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java @@ -19,12 +19,13 @@ package org.apache.skywalking.oap.server.library.pprof.parser; import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; -import java.util.HashMap; +import org.apache.skywalking.oap.server.library.pprof.type.Frame; +import org.apache.skywalking.oap.server.library.pprof.type.Index; import java.util.List; -import java.util.Map; public class PprofMergeBuilder { - private final Node root = new Node("root"); + private final Index cpool = new Index<>(String.class, ""); + private final Frame root = new Frame("root"); public PprofMergeBuilder merge(List trees) { if (trees == null || trees.isEmpty()) { @@ -41,44 +42,35 @@ public PprofMergeBuilder merge(FrameTree tree) { return this; } - private void merge0(Node node, FrameTree tree) { + private void merge0(Frame frame, FrameTree tree) { if (tree == null) { return; } if (tree.getChildren() != null) { for (FrameTree childTree : tree.getChildren()) { - Node child = getOrAddChild(node, childTree.getSignature()); + Frame child = addChild(frame, childTree.getSignature()); merge0(child, childTree); } } - node.total += tree.getTotal(); - node.self += tree.getSelf(); + frame.setTotal(frame.getTotal() + tree.getTotal()); + frame.setSelf(frame.getSelf() + tree.getSelf()); } - private Node getOrAddChild(Node parent, String signature) { - return parent.children.computeIfAbsent(signature, Node::new); + private Frame addChild(Frame parent, String signature) { + int titleIndex = cpool.index(signature); + return parent.getChild(titleIndex, signature); } public FrameTree build() { return toFrameTree(root); } - private FrameTree toFrameTree(Node node) { - FrameTree tree = new FrameTree(node.signature, node.total, node.self); - for (Node child : node.children.values()) { + private FrameTree toFrameTree(Frame node) { + FrameTree tree = new FrameTree(node.getSignature(), node.getTotal(), node.getSelf()); + for (Frame child : node.values()) { tree.getChildren().add(toFrameTree(child)); } return tree; } - private static class Node { - final String signature; - long total; - long self; - final Map children = new HashMap<>(); - - Node(String signature) { - this.signature = signature; - } - } } diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java index eacbf2d55f18..8e8acbaa2391 100644 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java @@ -19,11 +19,13 @@ package org.apache.skywalking.oap.server.library.pprof.parser; import com.google.perftools.profiles.ProfileProto; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; +import java.util.zip.GZIPInputStream; import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; import org.apache.skywalking.oap.server.library.pprof.type.FrameTreeBuilder; @@ -33,9 +35,14 @@ public class PprofParser { public static FrameTree dumpTree(ByteBuffer buf) throws IOException { - ProfileProto.Profile profile = ProfileProto.Profile.parseFrom(buf); + byte[] bytes = new byte[buf.remaining()]; + buf.get(bytes); + InputStream stream = new java.io.ByteArrayInputStream(bytes); + InputStream inputStream = isGzippedBytes(bytes) ? new GZIPInputStream(stream) : stream; + ProfileProto.Profile profile = ProfileProto.Profile.parseFrom(inputStream); FrameTree tree = new FrameTreeBuilder(profile).build(); return tree; + } public static FrameTree dumpTree(String filePath) throws IOException { @@ -43,11 +50,26 @@ public static FrameTree dumpTree(String filePath) throws IOException { if (!file.exists()) { throw new IOException("Pprof file not found: " + filePath); } - InputStream fileStream = new FileInputStream(file); + InputStream stream = filePath.endsWith(".gz") || isGzipped(file) ? + new GZIPInputStream(fileStream) : fileStream; ProfileProto.Profile profile = ProfileProto.Profile.parseFrom(fileStream); FrameTree tree = new FrameTreeBuilder(profile).build(); return tree; - } -} \ No newline at end of file + + private static boolean isGzipped(File file) throws IOException { + try (FileInputStream fis = new FileInputStream(file)) { + byte[] magic = new byte[2]; + if (fis.read(magic) == 2) { + return (magic[0] == (byte) 0x1f) && (magic[1] == (byte) 0x8b); + } + } + return false; + } + + private static boolean isGzippedBytes(byte[] bytes) { + return bytes.length >= 2 && + (bytes[0] == (byte) 0x1f) && (bytes[1] == (byte) 0x8b); + } +} diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java new file mode 100644 index 000000000000..65993079526d --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.pprof.type; + +import java.util.HashMap; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class Frame extends HashMap { + final int key; + final String signature; + long total; + long self; + + private Frame(int key, String signature) { + this.key = key; + this.signature = signature; + } + + public Frame(String signature) { + this(signature.hashCode(), signature); + } + + public Frame getChild(int titleIndex, String signature) { + return super.computeIfAbsent(titleIndex, k -> new Frame(k, signature)); + } +} diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Index.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Index.java new file mode 100644 index 000000000000..30582ff5022c --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Index.java @@ -0,0 +1,34 @@ +package org.apache.skywalking.oap.server.library.pprof.type; + +import java.lang.reflect.Array; +import java.util.HashMap; + +public class Index extends HashMap { + private final Class cls; + + public Index(Class cls, T empty) { + this.cls = cls; + super.put(empty, 0); + } + + public int index(T key) { + Integer index = super.get(key); + if (index != null) { + return index; + } else { + int newIndex = super.size(); + super.put(key, newIndex); + return newIndex; + } + } + + @SuppressWarnings("unchecked") + public T[] keys() { + T[] result = (T[]) Array.newInstance(cls, size()); + for (Entry entry : entrySet()) { + result[entry.getValue()] = entry.getKey(); + } + return result; + } +} + diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java index b275acd65014..d8dc848be5c3 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java @@ -167,4 +167,9 @@ public class StorageModuleElasticsearchConfig extends ModuleConfig { * in a single request. */ private int asyncProfilerTaskQueryMaxSize; + /** + * Max size of {@link org.apache.skywalking.oap.server.core.query.type.PprofTask} to be fetched + * in a single request. + */ + private int pprofTaskQueryMaxSize; } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java index cfbd84c4cb13..343b4d354f09 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java @@ -277,7 +277,7 @@ IProfileThreadSnapshotQueryDAO.class, new ProfileThreadSnapshotQueryEsDAO(elasti ); this.registerServiceImplementation( IPprofTaskLogQueryDAO.class, - new PprofTaskLogQueryEsDAO(elasticSearchClient, config.getAsyncProfilerTaskQueryMaxSize()) + new PprofTaskLogQueryEsDAO(elasticSearchClient, config.getPprofTaskQueryMaxSize()) ); this.registerServiceImplementation( IJFRDataQueryDAO.class, @@ -285,7 +285,7 @@ IProfileThreadSnapshotQueryDAO.class, new ProfileThreadSnapshotQueryEsDAO(elasti ); this.registerServiceImplementation( IPprofTaskQueryDAO.class, - new PprofTaskQueryEsDAO(elasticSearchClient, config.getAsyncProfilerTaskQueryMaxSize()) + new PprofTaskQueryEsDAO(elasticSearchClient, config.getPprofTaskQueryMaxSize()) ); this.registerServiceImplementation( StorageTTLStatusQuery.class, diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java index e93afbfbdef1..eb41c21fb7a9 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java @@ -120,26 +120,12 @@ private PprofTask parseTask(SearchHit data) { String serviceInstanceIds = (String) source.get(PprofTaskRecord.SERVICE_INSTANCE_IDS); List instanceIdList = GSON.fromJson(serviceInstanceIds, listType); - - // Convert string events to PprofEventType enum - String eventsStr = (String) source.get(PprofTaskRecord.EVENT_TYPES); - PprofEventType eventType = null; - if (StringUtil.isNotEmpty(eventsStr)) { - try { - eventType = PprofEventType.valueOfString(eventsStr); - } catch (Exception e) { - // Default to CPU if conversion fails - eventType = PprofEventType.CPU; - } - } - return PprofTask.builder() .id((String) source.get(PprofTaskRecord.TASK_ID)) .serviceId((String) source.get(PprofTaskRecord.SERVICE_ID)) .serviceInstanceIds(instanceIdList) .createTime(((Number) source.get(PprofTaskRecord.CREATE_TIME)).longValue()) - .startTime(((Number) source.get(PprofTaskRecord.START_TIME)).longValue()) - .events(eventType) + .events((PprofEventType) source.get(PprofTaskRecord.EVENT_TYPES)) .duration(((Number) source.get(PprofTaskRecord.DURATION)).intValue()) .dumpPeriod(((Number) source.get(PprofTaskRecord.DUMP_PERIOD)).intValue()) .build(); From d069e68c2bd7458d04c4909fc5c582b5067469f5 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Wed, 24 Sep 2025 20:53:31 +0800 Subject: [PATCH 04/69] fix: rollback yaml --- oap-server/server-starter/src/main/resources/application.yml | 2 +- oap-server/server-starter/src/main/resources/bydb.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index 50b66c9c8de9..a92a4604325b 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -151,7 +151,7 @@ storage: # Since 10.2.0, the banyandb configuration is separated to an independent configuration file: `bydb.yaml`. elasticsearch: namespace: ${SW_NAMESPACE:""} - clusterNodes: 192.168.50.242:9200 + clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:localhost:9200} protocol: ${SW_STORAGE_ES_HTTP_PROTOCOL:"http"} connectTimeout: ${SW_STORAGE_ES_CONNECT_TIMEOUT:3000} socketTimeout: ${SW_STORAGE_ES_SOCKET_TIMEOUT:30000} diff --git a/oap-server/server-starter/src/main/resources/bydb.yml b/oap-server/server-starter/src/main/resources/bydb.yml index 48bc7d3e1b98..2f83d0a26147 100644 --- a/oap-server/server-starter/src/main/resources/bydb.yml +++ b/oap-server/server-starter/src/main/resources/bydb.yml @@ -18,7 +18,7 @@ global: # Each target is a BanyanDB server in the format of `host:port`. # If BanyanDB is deployed as a standalone server, the target should be the IP address or domain name and port of the BanyanDB server. # If BanyanDB is deployed in a cluster, the targets should be the IP address or domain name and port of the `liaison` nodes, separated by commas. - targets: ${SW_STORAGE_BANYANDB_TARGETS:192.168.50.128:17912} + targets: ${SW_STORAGE_BANYANDB_TARGETS:127.0.0.1:17912} # The maximum number of records in a bulk write request. # A larger value can improve write performance but also increases OAP and BanyanDB Server memory usage. maxBulkSize: ${SW_STORAGE_BANYANDB_MAX_BULK_SIZE:10000} From 1afb571ad46776e6d58685d60363079d224c868c Mon Sep 17 00:00:00 2001 From: JophieQu Date: Mon, 29 Sep 2025 17:43:44 +0800 Subject: [PATCH 05/69] fix --- .../oap/server/core/CoreModuleProvider.java | 2 +- .../oap/server/core/cache/PprofTaskCache.java | 28 ++---------- .../profiling/pprof/PprofMutationService.java | 1 - .../storage/PprofProfilingDataDispatcher.java | 1 - .../storage/PprofProfilingDataRecord.java | 9 ---- .../pprof/parser/PprofMergeBuilder.java | 15 +------ .../library/pprof/parser/PprofParser.java | 22 ++-------- .../oap/server/library/pprof/type/Frame.java | 13 ++---- .../server/library/pprof/type/FrameTree.java | 43 ++++++++++++++----- .../library/pprof/type/FrameTreeBuilder.java | 3 +- .../oap/server/library/pprof/type/Index.java | 34 --------------- .../PprofByteBufCollectionObserver.java | 8 ---- .../stream/BanyanDBPprofDataQueryDAO.java | 19 +++++++- .../query/PprofTaskLogQueryEsDAO.java | 23 ++++++++++ 14 files changed, 88 insertions(+), 133 deletions(-) delete mode 100644 oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Index.java diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java index c8d88d8fdf47..9d8f15ad56a2 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java @@ -339,7 +339,7 @@ TTLStatusQuery.class, new TTLStatusQuery( this.registerServiceImplementation( PprofQueryService.class, new PprofQueryService(getManager())); this.registerServiceImplementation( - PprofTaskCache.class, new PprofTaskCache(getManager(), moduleConfig)); + PprofTaskCache.class, new PprofTaskCache(moduleConfig)); this.registerServiceImplementation( EBPFProfilingMutationService.class, new EBPFProfilingMutationService(getManager())); this.registerServiceImplementation( diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java index 9ce7e60defc2..304826f5efd3 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java @@ -19,50 +19,28 @@ package org.apache.skywalking.oap.server.core.cache; import org.apache.skywalking.oap.server.library.module.Service; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import org.apache.skywalking.oap.server.core.CoreModuleConfig; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.query.type.PprofTask; -import org.apache.skywalking.oap.server.core.storage.StorageModule; -import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; -import org.apache.skywalking.oap.server.library.module.ModuleManager; import java.time.Duration; -import java.util.Objects; import java.util.concurrent.TimeUnit; public class PprofTaskCache implements Service { - private static final Logger LOGGER = LoggerFactory.getLogger(PprofTaskCache.class); - private final Cache serviceId2taskCache; - private final ModuleManager moduleManager; - - private IPprofTaskQueryDAO taskQueryDAO; - - public PprofTaskCache(ModuleManager moduleManager, CoreModuleConfig moduleConfig) { - this.moduleManager = moduleManager; + public PprofTaskCache(CoreModuleConfig moduleConfig) { long initialSize = moduleConfig.getMaxSizeOfProfileTask() / 10L; int initialCapacitySize = (int) (initialSize > Integer.MAX_VALUE ? Integer.MAX_VALUE : initialSize); serviceId2taskCache = CacheBuilder.newBuilder() .initialCapacity(initialCapacitySize) .maximumSize(moduleConfig.getMaxSizeOfProfileTask()) - // remove old profile task data - extend to 10 minutes to ensure data availability - .expireAfterWrite(Duration.ofMinutes(10)) + // remove old profile task data + .expireAfterWrite(Duration.ofMinutes(1)) .build(); } - private IPprofTaskQueryDAO getTaskQueryDAO() { - if (Objects.isNull(taskQueryDAO)) { - taskQueryDAO = moduleManager.find(StorageModule.NAME) - .provider() - .getService(IPprofTaskQueryDAO.class); - } - return taskQueryDAO; - } - public PprofTask getPprofTask(String serviceId) { PprofTask task = serviceId2taskCache.getIfPresent(serviceId); return task; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java index ddfb45d5667a..3d8264e90644 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java @@ -67,7 +67,6 @@ public PprofTaskCreationResult createTask(String serviceId, if (checkResult != null) { return checkResult; } - // create task PprofTaskRecord task = new PprofTaskRecord(); String taskId = createTime + Const.ID_CONNECTOR + serviceId; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java index 5cdf1dd2b1ed..21df992f1d54 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java @@ -32,7 +32,6 @@ public void dispatch(PprofProfilingData source) { PprofProfilingDataRecord record = new PprofProfilingDataRecord(); record.setTaskId(source.getTaskId()); record.setInstanceId(source.getInstanceId()); - record.setEventType(source.getEventType().toString()); record.setDataBinary(GSON.toJson(source.getFrameTree()).getBytes()); record.setUploadTime(source.getUploadTime()); record.setTimeBucket(TimeBucket.getRecordTimeBucket(source.getUploadTime())); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java index 17164411a257..c88704b75491 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java @@ -38,9 +38,7 @@ @BanyanDB.TimestampColumn(PprofProfilingDataRecord.UPLOAD_TIME) public class PprofProfilingDataRecord extends Record { public static final String INDEX_NAME = "pprof_profiling_data"; - public static final String TASK_ID = "task_id"; - public static final String EVENT_TYPE = "event_type"; public static final String INSTANCE_ID = "instance_id"; public static final String DATA_BINARY = "data_binary"; public static final String UPLOAD_TIME = "upload_time"; @@ -51,9 +49,6 @@ public class PprofProfilingDataRecord extends Record { @Column(name = INSTANCE_ID) @BanyanDB.SeriesID(index = 0) private String instanceId; - - @Column(name = EVENT_TYPE) - private String eventType; @Column(name = UPLOAD_TIME) private long uploadTime; @@ -67,13 +62,11 @@ public StorageID id() { new String[]{ TASK_ID, INSTANCE_ID, - EVENT_TYPE, UPLOAD_TIME }, Hashing.sha256().newHasher() .putString(taskId, StandardCharsets.UTF_8) .putString(instanceId, StandardCharsets.UTF_8) - .putString(eventType, StandardCharsets.UTF_8) .putLong(uploadTime) .hash().toString() ); @@ -87,7 +80,6 @@ public PprofProfilingDataRecord storage2Entity(final Convert2Entity converter) { dataTraffic.setTaskId((String) converter.get(TASK_ID)); dataTraffic.setInstanceId((String) converter.get(INSTANCE_ID)); dataTraffic.setUploadTime(((Number) converter.get(UPLOAD_TIME)).longValue()); - dataTraffic.setEventType((String) converter.get(EVENT_TYPE)); dataTraffic.setDataBinary(converter.getBytes(DATA_BINARY)); return dataTraffic; } @@ -98,7 +90,6 @@ public void entity2Storage(final PprofProfilingDataRecord storageData, final Con converter.accept(TASK_ID, storageData.getTaskId()); converter.accept(INSTANCE_ID, storageData.getInstanceId()); converter.accept(UPLOAD_TIME, storageData.getUploadTime()); - converter.accept(EVENT_TYPE, storageData.getEventType()); converter.accept(DATA_BINARY, storageData.getDataBinary()); } } diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java index 6937c3416eb9..3e4e0f9f0082 100644 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java @@ -20,11 +20,9 @@ import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; import org.apache.skywalking.oap.server.library.pprof.type.Frame; -import org.apache.skywalking.oap.server.library.pprof.type.Index; import java.util.List; public class PprofMergeBuilder { - private final Index cpool = new Index<>(String.class, ""); private final Frame root = new Frame("root"); public PprofMergeBuilder merge(List trees) { @@ -57,20 +55,11 @@ private void merge0(Frame frame, FrameTree tree) { } private Frame addChild(Frame parent, String signature) { - int titleIndex = cpool.index(signature); - return parent.getChild(titleIndex, signature); + return parent.getChild(signature); } public FrameTree build() { - return toFrameTree(root); - } - - private FrameTree toFrameTree(Frame node) { - FrameTree tree = new FrameTree(node.getSignature(), node.getTotal(), node.getSelf()); - for (Frame child : node.values()) { - tree.getChildren().add(toFrameTree(child)); - } - return tree; + return FrameTree.buildTree(root); } } diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java index 8e8acbaa2391..9fe8ab776b3f 100644 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java @@ -38,7 +38,7 @@ public static FrameTree dumpTree(ByteBuffer buf) throws IOException { byte[] bytes = new byte[buf.remaining()]; buf.get(bytes); InputStream stream = new java.io.ByteArrayInputStream(bytes); - InputStream inputStream = isGzippedBytes(bytes) ? new GZIPInputStream(stream) : stream; + InputStream inputStream = new GZIPInputStream(stream); ProfileProto.Profile profile = ProfileProto.Profile.parseFrom(inputStream); FrameTree tree = new FrameTreeBuilder(profile).build(); return tree; @@ -51,25 +51,9 @@ public static FrameTree dumpTree(String filePath) throws IOException { throw new IOException("Pprof file not found: " + filePath); } InputStream fileStream = new FileInputStream(file); - InputStream stream = filePath.endsWith(".gz") || isGzipped(file) ? - new GZIPInputStream(fileStream) : fileStream; - ProfileProto.Profile profile = ProfileProto.Profile.parseFrom(fileStream); + InputStream stream = new GZIPInputStream(fileStream); + ProfileProto.Profile profile = ProfileProto.Profile.parseFrom(stream); FrameTree tree = new FrameTreeBuilder(profile).build(); return tree; } - - private static boolean isGzipped(File file) throws IOException { - try (FileInputStream fis = new FileInputStream(file)) { - byte[] magic = new byte[2]; - if (fis.read(magic) == 2) { - return (magic[0] == (byte) 0x1f) && (magic[1] == (byte) 0x8b); - } - } - return false; - } - - private static boolean isGzippedBytes(byte[] bytes) { - return bytes.length >= 2 && - (bytes[0] == (byte) 0x1f) && (bytes[1] == (byte) 0x8b); - } } diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java index 65993079526d..194829d9262b 100644 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java @@ -25,21 +25,16 @@ @Getter @Setter public class Frame extends HashMap { - final int key; final String signature; long total; long self; - private Frame(int key, String signature) { - this.key = key; - this.signature = signature; - } public Frame(String signature) { - this(signature.hashCode(), signature); + this.signature = signature; } - - public Frame getChild(int titleIndex, String signature) { - return super.computeIfAbsent(titleIndex, k -> new Frame(k, signature)); + + public Frame getChild(String signature) { + return super.computeIfAbsent(signature.hashCode(), k -> new Frame(signature)); } } diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java index 2b1587a8ec53..9a2f240ccdd3 100755 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java @@ -18,21 +18,42 @@ package org.apache.skywalking.oap.server.library.pprof.type; -import com.google.gson.annotations.SerializedName; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import java.util.ArrayList; import java.util.List; - -@Data -@NoArgsConstructor -@AllArgsConstructor +import lombok.Getter; +@Getter public class FrameTree { - @SerializedName("name") private String signature; - @SerializedName("value") private long total; private long self; - private final List children = new ArrayList<>(); + private List children; + + public FrameTree(Frame frame) { + this.signature = frame.getSignature(); + this.total = frame.getTotal(); + this.self = frame.getSelf(); + this.children = new ArrayList<>(frame.size()); + } + public FrameTree(String signature, long total, long self) { + this.signature = signature; + this.total = total; + this.self = self; + this.children = new ArrayList<>(); + } + + public static FrameTree buildTree(Frame frame) { + if (frame == null) return null; + + FrameTree frameTree = new FrameTree(frame); + // has children? + if (!frame.isEmpty()) { + frameTree.children = new ArrayList<>(frame.size()); + // build tree + for (Frame childFrame : frame.values()) { + FrameTree childFrameTree = buildTree(childFrame); + frameTree.children.add(childFrameTree); + } + } + return frameTree; + } } diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java index 3fe4a3b04089..c42f504ad8a2 100644 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java @@ -56,7 +56,6 @@ private FrameTree parseTree(RawFrameTree rawTree) { } return tree; } - private String getSignature(long locationId) { if (locationId == 0) { return "root"; @@ -87,11 +86,13 @@ private void mergeSample(ProfileProto.Sample sample) { boolean isEnd = i == size - 1; long locationId = locationIdList.get(i); if (children.containsKey(locationId)) { + // if the child exists, merge the sample data RawFrameTree child = children.get(locationId); child.setTotal(child.getTotal() + 1); child.setSelf(child.getSelf() + (isEnd ? 1 : 0)); children = child.getChildren(); } else { + // if the child does not exist, create a new child RawFrameTree child = new RawFrameTree(locationId, 1, (isEnd ? 1 : 0)); children.put(locationId, child); children = child.getChildren(); diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Index.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Index.java deleted file mode 100644 index 30582ff5022c..000000000000 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Index.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.apache.skywalking.oap.server.library.pprof.type; - -import java.lang.reflect.Array; -import java.util.HashMap; - -public class Index extends HashMap { - private final Class cls; - - public Index(Class cls, T empty) { - this.cls = cls; - super.put(empty, 0); - } - - public int index(T key) { - Integer index = super.get(key); - if (index != null) { - return index; - } else { - int newIndex = super.size(); - super.put(key, newIndex); - return newIndex; - } - } - - @SuppressWarnings("unchecked") - public T[] keys() { - T[] result = (T[]) Array.newInstance(cls, size()); - for (Entry entry : entrySet()) { - result[entry.getValue()] = entry.getKey(); - } - return result; - } -} - diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java index 5b9f10abba0e..833305e8f0b9 100644 --- a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java @@ -63,31 +63,23 @@ public void onNext(PprofData pprofData) { taskMetaData = parseMetaData(pprofData.getMetadata(), taskDAO); if (PprofProfilingStatus.PPROF_PROFILING_SUCCESS.equals(taskMetaData.getType())) { int size = taskMetaData.getContentSize(); - log.info("pprofMaxSize: {}, Pprof data size: {}", pprofMaxSize, size); if (pprofMaxSize >= size) { buf = ByteBuffer.allocate(size); // Send success response to allow client to continue uploading responseObserver.onNext(PprofCollectionResponse.newBuilder() .setStatus(PprofProfilingStatus.PPROF_PROFILING_SUCCESS) .build()); - - log.info("Started collecting pprof data in memory - service: {}, serviceInstance: {}, size: {} bytes", - pprofData.getMetadata().getService(), pprofData.getMetadata().getServiceInstance(), size); } else { responseObserver.onNext(PprofCollectionResponse.newBuilder() .setStatus(PprofProfilingStatus.PPROF_TERMINATED_BY_OVERSIZE) .build()); recordPprofTaskLog(taskMetaData.getTask(), taskMetaData.getInstanceId(), PprofTaskLogOperationType.PPROF_UPLOAD_FILE_TOO_LARGE_ERROR); - log.warn("Pprof file size {} exceeds maximum allowed size {} for service: {}, serviceInstance: {}", - size, pprofMaxSize, pprofData.getMetadata().getService(), pprofData.getMetadata().getServiceInstance()); } } else { responseObserver.onNext(PprofCollectionResponse.newBuilder() .setStatus(PprofProfilingStatus.PPROF_EXECUTION_TASK_ERROR) .build()); recordPprofTaskLog(taskMetaData.getTask(), taskMetaData.getInstanceId(), PprofTaskLogOperationType.EXECUTION_TASK_ERROR); - log.error("Received execution error from agent - service: {}, serviceInstance: {}, status: {}", - pprofData.getMetadata().getService(), pprofData.getMetadata().getServiceInstance(), taskMetaData.getType()); } } else if (pprofData.hasContent()) { if (buf != null) { diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java index 7d03db2f5aac..cdb573a9a8ef 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream; import com.google.common.collect.ImmutableSet; @@ -19,7 +37,6 @@ public class BanyanDBPprofDataQueryDAO extends AbstractBanyanDBDAO implements IP private static final Set TAGS = ImmutableSet.of( PprofProfilingDataRecord.TASK_ID, PprofProfilingDataRecord.INSTANCE_ID, - PprofProfilingDataRecord.EVENT_TYPE, PprofProfilingDataRecord.UPLOAD_TIME, PprofProfilingDataRecord.DATA_BINARY ); diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java index 4d9ce9a5e51d..6f9f16606e9e 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java @@ -82,3 +82,26 @@ private PprofTaskLog parseTaskLog(SearchHit data) { .build(); } } + + + + + + + + + + + + + + + + + + + + + + + From 2578bd168b53ac8f6fa8303741046b957f11e727 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Mon, 29 Sep 2025 22:33:39 +0800 Subject: [PATCH 06/69] fix --- apm-protocol/apm-network/src/main/proto | 2 +- docs/en/changes/changes.md | 1 + .../query/input/PprofTaskListRequest.java | 18 ++++++++++++++++++ .../query/type/PprofTaskCreationResult.java | 18 ++++++++++++++++++ .../query/type/PprofTaskCreationType.java | 18 ++++++++++++++++++ .../pprof/IPprofTaskLogQueryDAO.java | 19 ++++++++++++++++++- .../src/main/resources/query-protocol | 2 +- 7 files changed, 75 insertions(+), 3 deletions(-) diff --git a/apm-protocol/apm-network/src/main/proto b/apm-protocol/apm-network/src/main/proto index 157baf3e7a97..d5882f167677 160000 --- a/apm-protocol/apm-network/src/main/proto +++ b/apm-protocol/apm-network/src/main/proto @@ -1 +1 @@ -Subproject commit 157baf3e7a9710d3041e9caec39eecab7e0c5a82 +Subproject commit d5882f167677626479d7e1df3933f1024eecd3bc diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index bc3886219b8b..603b34bec16d 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -46,6 +46,7 @@ * Add UI dashboard for Ruby runtime metrics. * Tracing Query Execution HTTP APIs: make the argument `service layer` optional. * GraphQL API: metadata, topology, log and trace support query by name. +* Support pprof profiling feature #### UI diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java index db789a56228a..a2304a04ba95 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package org.apache.skywalking.oap.server.core.query.input; import lombok.Getter; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java index 1e9ca4d8596c..10f7653e9aa6 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package org.apache.skywalking.oap.server.core.query.type; import lombok.AllArgsConstructor; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java index ddf62db10a05..14519f62d3c0 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package org.apache.skywalking.oap.server.core.query.type; public enum PprofTaskCreationType { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java index 78571479204a..ba6b652dc396 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java @@ -1,8 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package org.apache.skywalking.oap.server.core.storage.profiling.pprof; import org.apache.skywalking.oap.server.core.query.PprofTaskLog; import org.apache.skywalking.oap.server.core.storage.DAO; - import java.io.IOException; import java.util.List; diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol index abf4c4d1588d..ed68593ec32e 160000 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol @@ -1 +1 @@ -Subproject commit abf4c4d1588d16facae4a696032d5f8b68a4ccaf +Subproject commit ed68593ec32ecd2cb56326bede0cd7b9f85b7bec From 4c03a81fb9380f26e7020d030e59688cbc2e738c Mon Sep 17 00:00:00 2001 From: JophieQu Date: Fri, 10 Oct 2025 17:55:57 +0800 Subject: [PATCH 07/69] add jdbc & es storage --- .../oap/server/core/cache/PprofTaskCache.java | 2 +- .../src/main/resources/application.yml | 1 + .../StorageModuleElasticsearchProvider.java | 14 +- .../query/PprofDataQueryEsDAO.java | 72 ++++++++ .../query/PprofTaskLogQueryEsDAO.java | 21 ++- .../query/PprofTaskQueryEsDAO.java | 19 +-- .../storage-jdbc-hikaricp-plugin/pom.xml | 5 + .../jdbc/common/JDBCStorageProvider.java | 18 ++ .../common/dao/JDBCPprofDataQueryDAO.java | 92 +++++++++++ .../common/dao/JDBCPprofTaskLogQueryDAO.java | 83 ++++++++++ .../common/dao/JDBCPprofTaskQueryDAO.java | 155 ++++++++++++++++++ 11 files changed, 454 insertions(+), 28 deletions(-) create mode 100644 oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java create mode 100644 oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java create mode 100644 oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskLogQueryDAO.java create mode 100644 oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskQueryDAO.java diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java index 304826f5efd3..73ac6dcfa347 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java @@ -36,7 +36,7 @@ public PprofTaskCache(CoreModuleConfig moduleConfig) { serviceId2taskCache = CacheBuilder.newBuilder() .initialCapacity(initialCapacitySize) .maximumSize(moduleConfig.getMaxSizeOfProfileTask()) - // remove old profile task data + // remove old pprof task data .expireAfterWrite(Duration.ofMinutes(1)) .build(); } diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index a92a4604325b..e6b084cc3891 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -184,6 +184,7 @@ storage: segmentQueryMaxSize: ${SW_STORAGE_ES_QUERY_SEGMENT_SIZE:200} profileTaskQueryMaxSize: ${SW_STORAGE_ES_QUERY_PROFILE_TASK_SIZE:200} asyncProfilerTaskQueryMaxSize: ${SW_STORAGE_ES_QUERY_ASYNC_PROFILER_TASK_SIZE:200} + pprofTaskQueryMaxSize: ${SW_STORAGE_ES_QUERY_PPROF_TASK_SIZE:200} profileDataQueryBatchSize: ${SW_STORAGE_ES_QUERY_PROFILE_DATA_BATCH_SIZE:100} oapAnalyzer: ${SW_STORAGE_ES_OAP_ANALYZER:"{\"analyzer\":{\"oap_analyzer\":{\"type\":\"stop\"}}}"} # the oap analyzer. oapLogAnalyzer: ${SW_STORAGE_ES_OAP_LOG_ANALYZER:"{\"analyzer\":{\"oap_log_analyzer\":{\"type\":\"standard\"}}}"} # the oap log analyzer. It could be customized by the ES analyzer configuration to support more language log formats, such as Chinese log, Japanese log and etc. diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java index 343b4d354f09..3cea61495b1d 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java @@ -41,6 +41,7 @@ import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IJFRDataQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.continuous.IContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingDataDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingScheduleDAO; @@ -92,6 +93,7 @@ import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ESEventQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.HierarchyQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.JFRDataQueryEsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.PprofDataQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.LogQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.MetadataQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.MetricsQueryEsDAO; @@ -275,10 +277,6 @@ IProfileThreadSnapshotQueryDAO.class, new ProfileThreadSnapshotQueryEsDAO(elasti IAsyncProfilerTaskLogQueryDAO.class, new AsyncProfilerTaskLogQueryEsDAO(elasticSearchClient, config.getAsyncProfilerTaskQueryMaxSize()) ); - this.registerServiceImplementation( - IPprofTaskLogQueryDAO.class, - new PprofTaskLogQueryEsDAO(elasticSearchClient, config.getPprofTaskQueryMaxSize()) - ); this.registerServiceImplementation( IJFRDataQueryDAO.class, new JFRDataQueryEsDAO(elasticSearchClient) @@ -287,6 +285,14 @@ IProfileThreadSnapshotQueryDAO.class, new ProfileThreadSnapshotQueryEsDAO(elasti IPprofTaskQueryDAO.class, new PprofTaskQueryEsDAO(elasticSearchClient, config.getPprofTaskQueryMaxSize()) ); + this.registerServiceImplementation( + IPprofTaskLogQueryDAO.class, + new PprofTaskLogQueryEsDAO(elasticSearchClient, config.getPprofTaskQueryMaxSize()) + ); + this.registerServiceImplementation( + IPprofDataQueryDAO.class, + new PprofDataQueryEsDAO(elasticSearchClient) + ); this.registerServiceImplementation( StorageTTLStatusQuery.class, new DefaultStorageTTLStatusQuery() diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java new file mode 100644 index 000000000000..2dc2b03d7be1 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; +import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Query; +import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Search; +import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; +import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; +import com.google.common.collect.Lists; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.ElasticSearchConverter; +import java.util.Map; +import java.util.ArrayList; +import java.util.List; + +public class PprofDataQueryEsDAO extends EsDAO implements IPprofDataQueryDAO { + public PprofDataQueryEsDAO(ElasticSearchClient client) { + super(client); + } + + @Override + public List getByTaskIdAndInstances(String taskId, List instanceIds) { + if (StringUtil.isBlank(taskId) || CollectionUtils.isEmpty(instanceIds)) { + return new ArrayList<>(); + } + final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofProfilingDataRecord.INDEX_NAME); + final BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isMergedTable(PprofProfilingDataRecord.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofProfilingDataRecord.INDEX_NAME)); + } + query.must(Query.term(PprofProfilingDataRecord.TASK_ID, taskId)); + query.must(Query.terms(PprofProfilingDataRecord.INSTANCE_ID, instanceIds)); + final SearchBuilder search = Search.builder().query(query); + final SearchResponse response = getClient().search(index, search.build()); + List dataRecords = Lists.newArrayList(); + for (SearchHit searchHit : response.getHits().getHits()) { + dataRecords.add(parseData(searchHit)); + } + return dataRecords; + } + + private PprofProfilingDataRecord parseData(SearchHit data) { + final Map sourceAsMap = data.getSource(); + final PprofProfilingDataRecord.Builder builder = new PprofProfilingDataRecord.Builder(); + return builder.storage2Entity(new ElasticSearchConverter.ToEntity(PprofProfilingDataRecord.INDEX_NAME, sourceAsMap)); + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java index 6f9f16606e9e..87f20d4498bf 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; +import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; import org.apache.skywalking.library.elasticsearch.requests.search.Query; import org.apache.skywalking.library.elasticsearch.requests.search.Search; import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; @@ -50,30 +51,28 @@ public PprofTaskLogQueryEsDAO(ElasticSearchClient client, int profileTaskQueryMa @Override public List getTaskLogList() throws IOException { final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofTaskLogRecord.INDEX_NAME); - - final SearchBuilder search = Search.builder().query(Query.bool()); + final BoolQueryBuilder query = Query.bool(); if (IndexController.LogicIndicesRegister.isMergedTable(PprofTaskLogRecord.INDEX_NAME)) { - search.query(Query.bool().must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofTaskLogRecord.INDEX_NAME))); + query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofTaskLogRecord.INDEX_NAME)); } - search.size(queryMaxSize); - search.sort(PprofTaskLogRecord.OPERATION_TIME, Sort.Order.DESC); - + final SearchBuilder search = + Search.builder().query(query) + .sort(PprofTaskLogRecord.OPERATION_TIME, Sort.Order.DESC) + .size(queryMaxSize); final SearchResponse response = getClient().search(index, search.build()); List tasks = new LinkedList<>(); - for (SearchHit hit : response.getHits().getHits()) { - tasks.add(parseTaskLog(hit)); + for (SearchHit searchHit : response.getHits().getHits()) { + tasks.add(buildPprofTaskLog(searchHit)); } return tasks; } - private PprofTaskLog parseTaskLog(SearchHit data) { + private PprofTaskLog buildPprofTaskLog(SearchHit data) { Map source = data.getSource(); - int operationTypeInt = ((Number) source.get(PprofTaskLogRecord.OPERATION_TYPE)).intValue(); PprofTaskLogOperationType operationType = PprofTaskLogOperationType.parse(operationTypeInt); - return PprofTaskLog.builder() .id((String) source.get(PprofTaskLogRecord.TASK_ID)) .instanceId((String) source.get(PprofTaskLogRecord.INSTANCE_ID)) diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java index eb41c21fb7a9..555e39cabe4e 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; import com.google.gson.Gson; +import java.util.Objects; import com.google.gson.reflect.TypeToken; import java.lang.reflect.Type; @@ -54,8 +55,8 @@ public PprofTaskQueryEsDAO(ElasticSearchClient client, int queryMaxSize) { @Override public List getTaskList(String serviceId, Long startTimeBucket, Long endTimeBucket, Integer limit) throws IOException { - final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofTaskRecord.INDEX_NAME); - final BoolQueryBuilder query = Query.bool(); + String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofTaskRecord.INDEX_NAME); + BoolQueryBuilder query = Query.bool(); if (IndexController.LogicIndicesRegister.isMergedTable(PprofTaskRecord.INDEX_NAME)) { query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofTaskRecord.INDEX_NAME)); } @@ -71,14 +72,8 @@ public List getTaskList(String serviceId, Long startTimeBucket, Long if (endTimeBucket != null) { query.must(Query.range(PprofTaskRecord.TIME_BUCKET).lte(endTimeBucket)); } - - final SearchBuilder search = Search.builder().query(query); - - if (limit != null) { - search.size(limit); - } else { - search.size(queryMaxSize); - } + SearchBuilder search = Search.builder().query(query); + search.size(Objects.requireNonNullElse(limit, queryMaxSize)); search.sort(PprofTaskRecord.CREATE_TIME, Sort.Order.DESC); @@ -92,7 +87,7 @@ public List getTaskList(String serviceId, Long startTimeBucket, Long } @Override - public PprofTask getById(String id) throws IOException { + public PprofTask getById(String id) { if (StringUtil.isEmpty(id)) { return null; } @@ -125,7 +120,7 @@ private PprofTask parseTask(SearchHit data) { .serviceId((String) source.get(PprofTaskRecord.SERVICE_ID)) .serviceInstanceIds(instanceIdList) .createTime(((Number) source.get(PprofTaskRecord.CREATE_TIME)).longValue()) - .events((PprofEventType) source.get(PprofTaskRecord.EVENT_TYPES)) + .events(PprofEventType.valueOfString((String) source.get(PprofTaskRecord.EVENT_TYPES))) .duration(((Number) source.get(PprofTaskRecord.DURATION)).intValue()) .dumpPeriod(((Number) source.get(PprofTaskRecord.DUMP_PERIOD)).intValue()) .build(); diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml index 3285d110a4e1..c56a24e86a19 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml @@ -48,6 +48,11 @@ org.postgresql postgresql + + mysql + mysql-connector-java + 8.0.13 + org.testcontainers diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCStorageProvider.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCStorageProvider.java index 2fd5cbcb4927..7891c1c7b122 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCStorageProvider.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCStorageProvider.java @@ -33,6 +33,9 @@ import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IJFRDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.continuous.IContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingDataDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingScheduleDAO; @@ -66,6 +69,9 @@ import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCAlarmQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCAsyncProfilerTaskQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCPprofDataQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCPprofTaskQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCBatchDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCBrowserLogQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCContinuousProfilingPolicyDAO; @@ -255,6 +261,18 @@ public void prepare() throws ServiceNotProvidedException, ModuleStartException { IJFRDataQueryDAO.class, new JDBCJFRDataQueryDAO(jdbcClient, tableHelper) ); + this.registerServiceImplementation( + IPprofTaskQueryDAO.class, + new JDBCPprofTaskQueryDAO(jdbcClient, tableHelper) + ); + this.registerServiceImplementation( + IPprofTaskLogQueryDAO.class, + new JDBCPprofTaskLogQueryDAO(jdbcClient, tableHelper) + ); + this.registerServiceImplementation( + IPprofDataQueryDAO.class, + new JDBCPprofDataQueryDAO(jdbcClient, tableHelper) + ); this.registerServiceImplementation( StorageTTLStatusQuery.class, new DefaultStorageTTLStatusQuery() diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java new file mode 100644 index 000000000000..6a0aa9055f4a --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + + import lombok.RequiredArgsConstructor; + import lombok.SneakyThrows; + import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; + import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; + import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCClient; + import org.apache.skywalking.oap.server.library.util.CollectionUtils; + import org.apache.skywalking.oap.server.library.util.StringUtil; + import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCEntityConverters; + import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCTableInstaller; + import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.TableHelper; + + import java.io.IOException; + import java.sql.ResultSet; + import java.util.ArrayList; + import java.util.Base64; + import java.util.List; + + @RequiredArgsConstructor + public class JDBCPprofDataQueryDAO implements IPprofDataQueryDAO { + private final JDBCClient jdbcClient; + private final TableHelper tableHelper; + + @Override + @SneakyThrows + public List getByTaskIdAndInstances(String taskId, List instanceIds) throws IOException { + if (StringUtil.isBlank(taskId)) { + return new ArrayList<>(); + } + List tables = tableHelper.getTablesWithinTTL(PprofProfilingDataRecord.INDEX_NAME); + List results = new ArrayList<>(); + for (final var table : tables) { + List condition = new ArrayList<>(4); + StringBuilder sql = new StringBuilder() + .append("select * from ").append(table) + .append(" where ").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); + condition.add(PprofProfilingDataRecord.INDEX_NAME); + + if (CollectionUtils.isNotEmpty(instanceIds)) { + sql.append(" and ").append(PprofProfilingDataRecord.INSTANCE_ID).append(" in (?) "); + String joinedInstanceIds = String.join(",", instanceIds); + condition.add(joinedInstanceIds); + } + + results.addAll( + jdbcClient.executeQuery( + sql.toString(), + resultSet -> { + final var result = new ArrayList(); + while (resultSet.next()) { + result.add(parseData(resultSet)); + } + return result; + }, + condition.toArray(new Object[0])) + ); + } + return results; + } + + private PprofProfilingDataRecord parseData(ResultSet data) { + final PprofProfilingDataRecord.Builder builder = new PprofProfilingDataRecord.Builder(); + PprofProfilingDataRecord PprofProfilingDataRecord = builder.storage2Entity(JDBCEntityConverters.toEntity(data)); + byte[] dataBinary = PprofProfilingDataRecord.getDataBinary(); + if (dataBinary != null) { + byte[] decodeResult = Base64.getDecoder().decode(dataBinary); + PprofProfilingDataRecord.setDataBinary(decodeResult); + } + return PprofProfilingDataRecord; + } + } + \ No newline at end of file diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskLogQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskLogQueryDAO.java new file mode 100644 index 000000000000..ea6da7dd81d1 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskLogQueryDAO.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskLogRecord; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCClient; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCTableInstaller; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.SQLAndParameters; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.TableHelper; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +@RequiredArgsConstructor +public class JDBCPprofTaskLogQueryDAO implements IPprofTaskLogQueryDAO { + private final JDBCClient jdbcClient; + private final TableHelper tableHelper; + + @Override + @SneakyThrows + public List getTaskLogList() { + List tables = tableHelper.getTablesWithinTTL(PprofTaskLogRecord.INDEX_NAME); + final List results = new ArrayList(); + for (String table : tables) { + SQLAndParameters sqlAndParameters = buildSQL(table); + List logs = jdbcClient.executeQuery( + sqlAndParameters.sql(), + resultSet -> { + final List tasks = new ArrayList<>(); + while (resultSet.next()) { + tasks.add(parseLog(resultSet)); + } + return tasks; + }, + sqlAndParameters.parameters()); + results.addAll(logs); + } + return results; + } + + private SQLAndParameters buildSQL(String table) { + StringBuilder sql = new StringBuilder(); + List parameters = new ArrayList<>(2); + sql.append("select * from ").append(table) + .append(" where ").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); + parameters.add(PprofTaskLogRecord.INDEX_NAME); + sql.append(" order by ").append(PprofTaskLogRecord.OPERATION_TIME).append(" desc"); + return new SQLAndParameters(sql.toString(), parameters); + } + + private PprofTaskLog parseLog(ResultSet data) throws SQLException { + return PprofTaskLog.builder() + .id(data.getString(PprofTaskLogRecord.TASK_ID)) + .instanceId(data.getString(PprofTaskLogRecord.INSTANCE_ID)) + .operationType(PprofTaskLogOperationType.parse(data.getInt(PprofTaskLogRecord.OPERATION_TYPE))) + .operationTime(data.getLong(PprofTaskLogRecord.OPERATION_TIME)) + .build(); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskQueryDAO.java new file mode 100644 index 000000000000..645accbb4d42 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskQueryDAO.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCClient; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCTableInstaller; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.TableHelper; + +import java.io.IOException; +import java.lang.reflect.Type; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import static java.util.stream.Collectors.toList; + +@RequiredArgsConstructor +public class JDBCPprofTaskQueryDAO implements IPprofTaskQueryDAO { + private static final Gson GSON = new Gson(); + + private final JDBCClient jdbcClient; + private final TableHelper tableHelper; + + @Override + @SneakyThrows + public List getTaskList(String serviceId, Long startTimeBucket, Long endTimeBucket, Integer limit) throws IOException { + final var results = new ArrayList(); + final var tables = startTimeBucket == null || endTimeBucket == null ? + tableHelper.getTablesWithinTTL(PprofTaskRecord.INDEX_NAME) : + tableHelper.getTablesForRead(PprofTaskRecord.INDEX_NAME, startTimeBucket, endTimeBucket); + for (final var table : tables) { + List condition = new ArrayList<>(4); + StringBuilder sql = new StringBuilder() + .append("select * from ").append(table) + .append(" where ").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); + condition.add(PprofTaskRecord.INDEX_NAME); + + if (StringUtil.isNotEmpty(serviceId)) { + sql.append(" and ").append(PprofTaskRecord.SERVICE_ID).append("=? "); + condition.add(serviceId); + } + + if (startTimeBucket != null) { + sql.append(" and ").append(PprofTaskRecord.TIME_BUCKET).append(" >= ? "); + condition.add(startTimeBucket); + } + + if (endTimeBucket != null) { + sql.append(" and ").append(PprofTaskRecord.TIME_BUCKET).append(" <= ? "); + condition.add(endTimeBucket); + } + + sql.append(" ORDER BY ").append(PprofTaskRecord.CREATE_TIME).append(" DESC "); + + if (limit != null) { + sql.append(" LIMIT ").append(limit); + } + + results.addAll( + jdbcClient.executeQuery( + sql.toString(), + resultSet -> { + final var tasks = new ArrayList(); + while (resultSet.next()) { + tasks.add(buildPprofTask(resultSet)); + } + return tasks; + }, + condition.toArray(new Object[0])) + ); + } + return limit == null ? + results : + results + .stream() + .limit(limit) + .collect(toList()); + } + + @Override + @SneakyThrows + public PprofTask getById(String id) throws IOException { + final var tables = tableHelper.getTablesWithinTTL(PprofTaskRecord.INDEX_NAME); + for (String table : tables) { + final StringBuilder sql = new StringBuilder(); + final List condition = new ArrayList<>(1); + sql.append("select * from ").append(table) + .append(" where ") + .append(JDBCTableInstaller.TABLE_COLUMN).append(" = ? ") + .append(" and ") + .append(PprofTaskRecord.TASK_ID + "=? LIMIT 1"); + condition.add(PprofTaskRecord.INDEX_NAME); + condition.add(id); + + final var r = jdbcClient.executeQuery( + sql.toString(), + resultSet -> { + if (resultSet.next()) { + return buildPprofTask(resultSet); + } + return null; + }, + condition.toArray(new Object[0])); + if (r != null) { + return r; + } + } + return null; + } + + private PprofTask buildPprofTask(ResultSet data) throws SQLException { + Type listType = new TypeToken>() { + }.getType(); + String events = data.getString(PprofTaskRecord.EVENT_TYPES); + String serviceInstanceIds = data.getString(PprofTaskRecord.SERVICE_INSTANCE_IDS); + List serviceInstanceIdList = GSON.fromJson(serviceInstanceIds, listType); + return PprofTask.builder() + .id(data.getString(PprofTaskRecord.TASK_ID)) + .serviceId(data.getString(PprofTaskRecord.SERVICE_ID)) + .serviceInstanceIds(serviceInstanceIdList) + .createTime(data.getLong(PprofTaskRecord.CREATE_TIME)) + .duration(data.getInt(PprofTaskRecord.DURATION)) + .events(PprofEventType.valueOfString(events)) + .dumpPeriod(data.getInt(PprofTaskRecord.DUMP_PERIOD)) + .build(); + } + +} From 5520d019ddfc8a43096a83bb29a5800048f7c775 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Fri, 10 Oct 2025 22:31:12 +0800 Subject: [PATCH 08/69] test ci --- .github/workflows/codeql.yaml | 3 ++- .github/workflows/dead-link-checker.yaml | 3 +++ .github/workflows/publish-docker-e2e-service.yaml | 1 + .github/workflows/publish-docker.yaml | 1 + .github/workflows/skywalking.yaml | 3 +++ 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 59f762f808b6..038e92bcced6 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -18,7 +18,8 @@ name: "CodeQL" on: push: - branches: ["master"] + branches: + - ci pull_request: branches: ["master"] paths: diff --git a/.github/workflows/dead-link-checker.yaml b/.github/workflows/dead-link-checker.yaml index b134daf30021..b05d292ce732 100644 --- a/.github/workflows/dead-link-checker.yaml +++ b/.github/workflows/dead-link-checker.yaml @@ -17,6 +17,9 @@ name: Dead Link Checker on: + push: + branches: + - ci pull_request: paths: - 'docs/**' diff --git a/.github/workflows/publish-docker-e2e-service.yaml b/.github/workflows/publish-docker-e2e-service.yaml index 661053ec40ae..0b8420f67579 100644 --- a/.github/workflows/publish-docker-e2e-service.yaml +++ b/.github/workflows/publish-docker-e2e-service.yaml @@ -20,6 +20,7 @@ on: push: branches: - master + - ci paths: - 'test/e2e-v2/java-test-service/**' - 'test/Makefile' diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml index c5bf734e54ab..a3367b2465f2 100644 --- a/.github/workflows/publish-docker.yaml +++ b/.github/workflows/publish-docker.yaml @@ -20,6 +20,7 @@ on: push: branches: - master + - ci release: types: - released diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index 5b3b3a6c93c0..ebc085074dbb 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -17,6 +17,9 @@ name: CI on: + push: + branches: + - ci pull_request: schedule: - cron: "0 18 * * *" # TimeZone: UTC 0 From c96548200f4dd36747e70dbbaf306f30c1626049 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Fri, 10 Oct 2025 23:26:23 +0800 Subject: [PATCH 09/69] update proto --- apm-protocol/apm-network/src/main/proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apm-protocol/apm-network/src/main/proto b/apm-protocol/apm-network/src/main/proto index d5882f167677..fb3fb005650e 160000 --- a/apm-protocol/apm-network/src/main/proto +++ b/apm-protocol/apm-network/src/main/proto @@ -1 +1 @@ -Subproject commit d5882f167677626479d7e1df3933f1024eecd3bc +Subproject commit fb3fb005650e2489164978b7804117c7ade1529a From c467076074ef4a5c777107686004255b10b30e45 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Sat, 11 Oct 2025 11:24:54 +0800 Subject: [PATCH 10/69] fix query submules --- .../query-graphql-plugin/src/main/resources/query-protocol | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol index ed68593ec32e..4fc10625ba72 160000 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol @@ -1 +1 @@ -Subproject commit ed68593ec32ecd2cb56326bede0cd7b9f85b7bec +Subproject commit 4fc10625ba72ef4788972b4f7991a535065d609b From 5000e7180263da06320b8eb526b98d8f4283a1a8 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Sat, 11 Oct 2025 13:15:43 +0800 Subject: [PATCH 11/69] fix ci --- .../oap/server/core/cache/PprofTaskCache.java | 1 + .../library-pprof-parser/pom.xml | 22 +++++++++++++++++-- .../src/main/proto/profile.proto | 17 ++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java index 73ac6dcfa347..6a06227a5001 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java @@ -29,6 +29,7 @@ public class PprofTaskCache implements Service { private final Cache serviceId2taskCache; + public PprofTaskCache(CoreModuleConfig moduleConfig) { long initialSize = moduleConfig.getMaxSizeOfProfileTask() / 10L; int initialCapacitySize = (int) (initialSize > Integer.MAX_VALUE ? Integer.MAX_VALUE : initialSize); diff --git a/oap-server/server-library/library-pprof-parser/pom.xml b/oap-server/server-library/library-pprof-parser/pom.xml index 196e38a441bd..25a549747ea2 100755 --- a/oap-server/server-library/library-pprof-parser/pom.xml +++ b/oap-server/server-library/library-pprof-parser/pom.xml @@ -1,4 +1,22 @@ + + @@ -13,8 +31,8 @@ library-pprof-parser - 17 - 17 + 11 + 11 UTF-8 true diff --git a/oap-server/server-library/library-pprof-parser/src/main/proto/profile.proto b/oap-server/server-library/library-pprof-parser/src/main/proto/profile.proto index 60216a3cfc8f..d1cb0a5a4040 100755 --- a/oap-server/server-library/library-pprof-parser/src/main/proto/profile.proto +++ b/oap-server/server-library/library-pprof-parser/src/main/proto/profile.proto @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); From cce039df6b8df0df3473a83c35f30f85e629476e Mon Sep 17 00:00:00 2001 From: JophieQu Date: Sat, 11 Oct 2025 15:04:46 +0800 Subject: [PATCH 12/69] fix ci --- .../apache/skywalking/oap/server/core/CoreModuleTest.java | 2 +- .../plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/CoreModuleTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/CoreModuleTest.java index a8e0098a72e2..cd1edcc1cac9 100644 --- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/CoreModuleTest.java +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/CoreModuleTest.java @@ -26,6 +26,6 @@ public class CoreModuleTest { public void testOpenServiceList() { CoreModule coreModule = new CoreModule(); - Assertions.assertEquals(49, coreModule.services().length); + Assertions.assertEquals(52, coreModule.services().length); } } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java index 6a0aa9055f4a..86899febe75b 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java @@ -80,13 +80,13 @@ public List getByTaskIdAndInstances(String taskId, Lis private PprofProfilingDataRecord parseData(ResultSet data) { final PprofProfilingDataRecord.Builder builder = new PprofProfilingDataRecord.Builder(); - PprofProfilingDataRecord PprofProfilingDataRecord = builder.storage2Entity(JDBCEntityConverters.toEntity(data)); - byte[] dataBinary = PprofProfilingDataRecord.getDataBinary(); + PprofProfilingDataRecord pprofProfilingDataRecord = builder.storage2Entity(JDBCEntityConverters.toEntity(data)); + byte[] dataBinary = pprofProfilingDataRecord.getDataBinary(); if (dataBinary != null) { byte[] decodeResult = Base64.getDecoder().decode(dataBinary); - PprofProfilingDataRecord.setDataBinary(decodeResult); + pprofProfilingDataRecord.setDataBinary(decodeResult); } - return PprofProfilingDataRecord; + return pprofProfilingDataRecord; } } \ No newline at end of file From 39894942bf7a2582b9f8b21f4d9cfa23b3e731df Mon Sep 17 00:00:00 2001 From: JophieQu Date: Sat, 11 Oct 2025 15:05:13 +0800 Subject: [PATCH 13/69] ci test (to rollback) --- .github/workflows/skywalking.yaml | 28 +++++++++---------- .../setup/backend/backend-redis-monitoring.md | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index ebc085074dbb..e92bc03bcf41 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -35,7 +35,7 @@ env: jobs: license-header: - if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || (github.event_name != 'schedule') name: License header runs-on: ubuntu-latest timeout-minutes: 10 @@ -48,7 +48,7 @@ jobs: uses: apache/skywalking-eyes@5b7ee1731d036b5aac68f8bd3fc9e6f98ada082e code-style: - if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || (github.event_name != 'schedule') name: Code style runs-on: ubuntu-latest timeout-minutes: 10 @@ -63,7 +63,7 @@ jobs: dependency-license: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.pom == 'true' || needs.changes.outputs.ui == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.pom == 'true' || needs.changes.outputs.ui == 'true') name: Dependency licenses needs: [changes] runs-on: ubuntu-latest @@ -93,7 +93,7 @@ jobs: fi sanity-check: - if: ( always() && ! cancelled() ) && (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') + if: ( always() && ! cancelled() ) && (github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || (github.event_name != 'schedule') name: Sanity check results needs: [license-header, code-style, dependency-license] runs-on: ubuntu-latest @@ -162,7 +162,7 @@ jobs: dist-tar: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: Build dist tar needs: [changes] runs-on: ubuntu-latest @@ -194,7 +194,7 @@ jobs: docker: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: Docker images needs: [sanity-check, dist-tar, changes] runs-on: ubuntu-latest @@ -233,7 +233,7 @@ jobs: unit-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: Unit test needs: [sanity-check, changes] runs-on: ${{ matrix.os }} @@ -268,7 +268,7 @@ jobs: integration-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: Integration test needs: [sanity-check, changes] runs-on: ubuntu-latest @@ -301,7 +301,7 @@ jobs: slow-integration-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: Slow Integration Tests needs: [sanity-check, changes] runs-on: ubuntu-latest @@ -334,7 +334,7 @@ jobs: e2e-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker, dist-tar] runs-on: ${{ matrix.test.runs-on || 'ubuntu-latest' }} @@ -789,7 +789,7 @@ jobs: e2e-test-istio: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-24.04 @@ -857,7 +857,7 @@ jobs: e2e-test-istio-ambient: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-24.04 @@ -918,7 +918,7 @@ jobs: e2e-test-java-versions: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-latest @@ -968,7 +968,7 @@ jobs: e2e-test-banyandb-stages: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker, dist-tar] runs-on: ${{ matrix.test.runs-on || 'ubuntu-latest' }} diff --git a/docs/en/setup/backend/backend-redis-monitoring.md b/docs/en/setup/backend/backend-redis-monitoring.md index bd20a9712025..07b1ccd4cbe0 100644 --- a/docs/en/setup/backend/backend-redis-monitoring.md +++ b/docs/en/setup/backend/backend-redis-monitoring.md @@ -46,7 +46,7 @@ SkyWalking leverages [fluentbit](https://fluentbit.io/) or other log agents for 4. The SkyWalking OAP Server parses the expression with [LAL](../../concepts-and-designs/lal.md) to parse/extract and store the results. ### Set up -1. Set up [fluentbit](https://docs.fluentbit.io/manual/installation/docker). +1. Set up [fluentbit](https://docs.fluentbit.io/manual/installation/downloads/docker). 2. Config fluentbit from [here](../../../../test/e2e-v2/cases/redis/redis-exporter/fluent-bit.conf) for Redis. 3. Config slow log from [here](../../../../test/e2e-v2/cases/redis/redis-exporter/redis.conf) for Redis. 4. Periodically execute the [commands](../../../../test/e2e-v2/cases/redis/redis-exporter/scripts/slowlog.sh). From 232cdc402ac7369695d683fe6eadaa4edc09c88f Mon Sep 17 00:00:00 2001 From: JophieQu Date: Sun, 12 Oct 2025 14:54:09 +0800 Subject: [PATCH 14/69] fix --- .../storage-jdbc-hikaricp-plugin/pom.xml | 41 +++++++++++++++ .../profiling/pprof/profiling-cases.yaml | 50 +++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml index c56a24e86a19..025c702d12a8 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml @@ -52,6 +52,7 @@ mysql mysql-connector-java 8.0.13 + provided @@ -60,4 +61,44 @@ test + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + default-test + test + + test + + + + **/PreventRedistributionMySQLDriverTest.java + + + + + + test-without-mysql + test + + test + + + + **/PreventRedistributionMySQLDriverTest.java + + + mysql:mysql-connector-java + + + + + + + diff --git a/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml b/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml new file mode 100644 index 000000000000..7fd17aab6e34 --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +cases: + # service list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls + expected: expected/service.yml + # service instance list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider + expected: expected/service-instance.yml + # create task + - query: | + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ + profiling pprof create --service-name=e2e-service-provider \ + --duration=20 --events=CPU \ + --instance-name-list=provider1 + expected: expected/create.yml + # list task + - query: | + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ + profiling async list --service-name=e2e-service-provider \ + expected: expected/list.yml + # get task progress + - query: | + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ + profiling async progress --task-id=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling async list --service-name=e2e-service-provider | yq e '.tasks[0].id') + expected: expected/progress.yml + # get task analysis + - query: | + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ + profiling async analysis --service-name=e2e-service-provider \ + --event=execution_sample \ + --instance-name-list=provider1 \ + --task-id=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling async list --service-name=e2e-service-provider | yq e '.tasks[0].id') + expected: expected/analysis.yml + From e8f44ad456279ccb9c412b5b210b5cdfb2e48d02 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Tue, 14 Oct 2025 13:19:27 +0800 Subject: [PATCH 15/69] fix --- .github/workflows/dead-link-checker.yaml | 2 - .github/workflows/skywalking.yaml | 7 +++ .../storage/PprofProfilingDataRecord.java | 7 +-- .../server/core/query/type/AlarmMessage.java | 1 + .../pprof/banyandb/docker-compose.yml | 50 ++++++++++++++++ .../cases/profiling/pprof/banyandb/e2e.yaml | 37 ++++++++++++ .../profiling/pprof/es/docker-compose.yml | 58 ++++++++++++++++++ test/e2e-v2/cases/profiling/pprof/es/e2e.yaml | 37 ++++++++++++ .../profiling/pprof/expected/analysis.yml | 25 ++++++++ .../cases/profiling/pprof/expected/create.yml | 18 ++++++ .../cases/profiling/pprof/expected/list.yml | 29 +++++++++ .../profiling/pprof/expected/progress.yml | 26 ++++++++ .../pprof/expected/service-instance.yml | 40 +++++++++++++ .../profiling/pprof/expected/service.yml | 24 ++++++++ .../profiling/pprof/mysql/docker-compose.yml | 60 +++++++++++++++++++ .../cases/profiling/pprof/mysql/e2e.yml | 37 ++++++++++++ .../profiling/pprof/profiling-cases.yaml | 8 +-- test/e2e-v2/script/env | 2 +- 18 files changed, 455 insertions(+), 13 deletions(-) create mode 100644 test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml create mode 100644 test/e2e-v2/cases/profiling/pprof/banyandb/e2e.yaml create mode 100644 test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml create mode 100644 test/e2e-v2/cases/profiling/pprof/es/e2e.yaml create mode 100644 test/e2e-v2/cases/profiling/pprof/expected/analysis.yml create mode 100644 test/e2e-v2/cases/profiling/pprof/expected/create.yml create mode 100644 test/e2e-v2/cases/profiling/pprof/expected/list.yml create mode 100644 test/e2e-v2/cases/profiling/pprof/expected/progress.yml create mode 100644 test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml create mode 100644 test/e2e-v2/cases/profiling/pprof/expected/service.yml create mode 100644 test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml create mode 100644 test/e2e-v2/cases/profiling/pprof/mysql/e2e.yml diff --git a/.github/workflows/dead-link-checker.yaml b/.github/workflows/dead-link-checker.yaml index b05d292ce732..659518c1365c 100644 --- a/.github/workflows/dead-link-checker.yaml +++ b/.github/workflows/dead-link-checker.yaml @@ -18,8 +18,6 @@ name: Dead Link Checker on: push: - branches: - - ci pull_request: paths: - 'docs/**' diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index 6fb991a24f82..c2136b1feb53 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -726,6 +726,13 @@ jobs: config: test/e2e-v2/cases/profiling/async-profiler/banyandb/e2e.yaml - name: Async Profiler MySQL config: test/e2e-v2/cases/profiling/async-profiler/mysql/e2e.yaml + + - name: Pprof ES + config: test/e2e-v2/cases/profiling/pprof/es/e2e.yaml + - name: Pprof BanyanDB + config: test/e2e-v2/cases/profiling/pprof/banyandb/e2e.yaml + - name: Pprof MySQL + config: test/e2e-v2/cases/profiling/pprof/mysql/e2e.yaml steps: - uses: actions/checkout@v4 with: diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java index c88704b75491..71492a1bf5ee 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java @@ -58,12 +58,7 @@ public class PprofProfilingDataRecord extends Record { @Override public StorageID id() { - return new StorageID().appendMutant( - new String[]{ - TASK_ID, - INSTANCE_ID, - UPLOAD_TIME - }, + return new StorageID().append( Hashing.sha256().newHasher() .putString(taskId, StandardCharsets.UTF_8) .putString(instanceId, StandardCharsets.UTF_8) diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/AlarmMessage.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/AlarmMessage.java index 7ff8e5306ccd..67dd14f234a8 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/AlarmMessage.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/AlarmMessage.java @@ -35,6 +35,7 @@ public class AlarmMessage { private String name; private String message; private Long startTime; + private Long recoveryTime; private transient String id1; private final List tags; private List events = new ArrayList<>(2); diff --git a/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml new file mode 100644 index 000000000000..99aff4e41d90 --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '3.8' + +services: + banyandb: + extends: + file: ../../../../script/docker-compose/base-compose.yml + service: banyandb + networks: + - e2e + + provider: + extends: + file: ../../../../script/docker-compose/base-compose.yml + service: provider + privileged: true + depends_on: + oap: + condition: service_healthy + ports: + - 9090 + + oap: + extends: + file: ../../../../script/docker-compose/base-compose.yml + service: oap + environment: + SW_STORAGE: banyandb + depends_on: + banyandb: + condition: service_healthy + ports: + - 12800 + +networks: + e2e: diff --git a/test/e2e-v2/cases/profiling/pprof/banyandb/e2e.yaml b/test/e2e-v2/cases/profiling/pprof/banyandb/e2e.yaml new file mode 100644 index 000000000000..956e87c2820e --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/banyandb/e2e.yaml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 20m + init-system-environment: ../../../../script/env + steps: + - name: set PATH + command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + +verify: + retry: + count: 20 + interval: 10s + cases: + - includes: + - ../profiling-cases.yaml diff --git a/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml new file mode 100644 index 000000000000..916ad2903f2e --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '3.8' + +services: + es: + image: elastic/elasticsearch:7.15.0 + expose: + - 9200 + networks: + - e2e + environment: + - discovery.type=single-node + - cluster.routing.allocation.disk.threshold_enabled=false + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"] + interval: 5s + timeout: 60s + retries: 120 + + provider: + extends: + file: ../../../../script/docker-compose/base-compose.yml + service: provider + privileged: true + depends_on: + oap: + condition: service_healthy + ports: + - 9090 + + oap: + extends: + file: ../../../../script/docker-compose/base-compose.yml + service: oap + environment: + SW_STORAGE: elasticsearch + depends_on: + es: + condition: service_healthy + ports: + - 12800 + +networks: + e2e: diff --git a/test/e2e-v2/cases/profiling/pprof/es/e2e.yaml b/test/e2e-v2/cases/profiling/pprof/es/e2e.yaml new file mode 100644 index 000000000000..956e87c2820e --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/es/e2e.yaml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 20m + init-system-environment: ../../../../script/env + steps: + - name: set PATH + command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + +verify: + retry: + count: 20 + interval: 10s + cases: + - includes: + - ../profiling-cases.yaml diff --git a/test/e2e-v2/cases/profiling/pprof/expected/analysis.yml b/test/e2e-v2/cases/profiling/pprof/expected/analysis.yml new file mode 100644 index 000000000000..265c20c55490 --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/expected/analysis.yml @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tree: + type: EXECUTION_SAMPLE + elements: + {{- contains .tree.elements }} + - id: {{ notEmpty .id }} + parentid: {{ notEmpty .parentid }} + codesignature: {{ notEmpty .codesignature }} + total: {{ ge .total -1 }} + self: {{ ge .self -1 }} + {{- end }} \ No newline at end of file diff --git a/test/e2e-v2/cases/profiling/pprof/expected/create.yml b/test/e2e-v2/cases/profiling/pprof/expected/create.yml new file mode 100644 index 000000000000..85bd8870434e --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/expected/create.yml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +errorreason: null +code: SUCCESS +id: {{ notEmpty .id }} \ No newline at end of file diff --git a/test/e2e-v2/cases/profiling/pprof/expected/list.yml b/test/e2e-v2/cases/profiling/pprof/expected/list.yml new file mode 100644 index 000000000000..645ece7fc066 --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/expected/list.yml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +errorreason: null +tasks: + {{- contains .tasks }} +- id: {{ notEmpty .id }} + serviceid: {{ b64enc "e2e-service-provider" }}.1 + serviceinstanceids: + - {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} + createtime: {{ gt .createtime 0 }} + events: + - CPU + - ALLOC + duration: {{ ge .duration 0 }} + execargs: null + {{- end }} \ No newline at end of file diff --git a/test/e2e-v2/cases/profiling/pprof/expected/progress.yml b/test/e2e-v2/cases/profiling/pprof/expected/progress.yml new file mode 100644 index 000000000000..257376e0f047 --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/expected/progress.yml @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +logs: + {{- contains .logs }} +- id: {{ notEmpty .id}} + instanceid: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} + instancename: {{ notEmpty .instancename}} + operationtype: {{ notEmpty .operationtype}} + operationtime: {{ ge .operationtime 0 }} + {{- end }} +errorinstanceids: [] +successinstanceids: + - {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} diff --git a/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml b/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml new file mode 100644 index 000000000000..f32cc2b0f1e4 --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml @@ -0,0 +1,40 @@ +# Licensed to Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Apache Software Foundation (ASF) licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{{- contains . }} +- id: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} + name: provider1 + attributes: + {{- contains .attributes }} + - name: OS Name + value: Linux + - name: hostname + value: {{ notEmpty .value }} + - name: Process No. + value: {{ notEmpty .value }} + - name: Start Time + value: {{ notEmpty .value }} + - name: JVM Arguments + value: '{{ notEmpty .value }}' + - name: Jar Dependencies + value: '{{ notEmpty .value }}' + - name: ipv4s + value: {{ notEmpty .value }} + {{- end}} + language: JAVA + instanceuuid: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} +{{- end}} diff --git a/test/e2e-v2/cases/profiling/pprof/expected/service.yml b/test/e2e-v2/cases/profiling/pprof/expected/service.yml new file mode 100644 index 000000000000..75d0916f00b4 --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/expected/service.yml @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- contains . }} +- id: {{ b64enc "e2e-service-provider" }}.1 + name: e2e-service-provider + group: "" + shortname: e2e-service-provider + normal: true + layers: + - GENERAL +{{- end }} diff --git a/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml new file mode 100644 index 000000000000..8b98f4d64bb4 --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + mysql: + image: mysql/mysql-server:8.0.13 + networks: + - e2e + expose: + - 3306 + environment: + - MYSQL_ROOT_PASSWORD=root@1234 + - MYSQL_DATABASE=swtest + - MYSQL_ROOT_HOST=% + healthcheck: + test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/3306" ] + interval: 5s + timeout: 60s + retries: 120 + + provider: + extends: + file: ../../../../script/docker-compose/base-compose.yml + service: provider + privileged: true + depends_on: + oap: + condition: service_healthy + ports: + - 9090 + + oap: + extends: + file: ../../../../script/docker-compose/base-compose.yml + service: oap + environment: + SW_STORAGE: mysql + depends_on: + mysql: + condition: service_healthy + entrypoint: ['sh', '-c', '/download-mysql.sh /skywalking/oap-libs && /skywalking/docker-entrypoint.sh'] + ports: + - 12800 + +networks: + e2e: diff --git a/test/e2e-v2/cases/profiling/pprof/mysql/e2e.yml b/test/e2e-v2/cases/profiling/pprof/mysql/e2e.yml new file mode 100644 index 000000000000..956e87c2820e --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/mysql/e2e.yml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 20m + init-system-environment: ../../../../script/env + steps: + - name: set PATH + command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + +verify: + retry: + count: 20 + interval: 10s + cases: + - includes: + - ../profiling-cases.yaml diff --git a/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml b/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml index 7fd17aab6e34..8a9fff6c15fb 100644 --- a/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml +++ b/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml @@ -32,19 +32,19 @@ cases: # list task - query: | swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ - profiling async list --service-name=e2e-service-provider \ + profiling pprof list --service-name=e2e-service-provider \ expected: expected/list.yml # get task progress - query: | swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ - profiling async progress --task-id=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling async list --service-name=e2e-service-provider | yq e '.tasks[0].id') + profiling pprof progress --task-id=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling pprof list --service-name=e2e-service-provider | yq e '.tasks[0].id') expected: expected/progress.yml # get task analysis - query: | swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ - profiling async analysis --service-name=e2e-service-provider \ + profiling pprof analysis --service-name=e2e-service-provider \ --event=execution_sample \ --instance-name-list=provider1 \ - --task-id=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling async list --service-name=e2e-service-provider | yq e '.tasks[0].id') + --task-id=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling pprof list --service-name=e2e-service-provider | yq e '.tasks[0].id') expected: expected/analysis.yml diff --git a/test/e2e-v2/script/env b/test/e2e-v2/script/env index 480d3081fef1..3cc8ef7583a7 100644 --- a/test/e2e-v2/script/env +++ b/test/e2e-v2/script/env @@ -27,4 +27,4 @@ SW_BANYANDB_COMMIT=7466afd2a2120f7fa311983be5a3077cb15d07e7 SW_AGENT_PHP_COMMIT=d1114e7be5d89881eec76e5b56e69ff844691e35 SW_PREDICTOR_COMMIT=54a0197654a3781a6f73ce35146c712af297c994 -SW_CTL_COMMIT=3b675df73824bbb80e6aabf6a95d110feb37b6b1 +SW_CTL_COMMIT=9a1beab08413ce415a00a8547a238a14691c5655 From f46887fa3d5687cfab3ee03507dab7d0b5c8ef1b Mon Sep 17 00:00:00 2001 From: JophieQu Date: Tue, 14 Oct 2025 16:30:02 +0800 Subject: [PATCH 16/69] fix e2e --- dist-material/release-docs/LICENSE | 31 ++++++++++--------- .../pprof/banyandb/docker-compose.yml | 7 ++--- .../cases/profiling/pprof/expected/list.yml | 2 -- .../pprof/expected/service-instance.yml | 8 +---- .../pprof/mysql/{e2e.yml => e2e.yaml} | 0 5 files changed, 19 insertions(+), 29 deletions(-) rename test/e2e-v2/cases/profiling/pprof/mysql/{e2e.yml => e2e.yaml} (100%) diff --git a/dist-material/release-docs/LICENSE b/dist-material/release-docs/LICENSE index 35777513907e..df015b932865 100644 --- a/dist-material/release-docs/LICENSE +++ b/dist-material/release-docs/LICENSE @@ -477,20 +477,21 @@ The text of each license is also included in licenses/LICENSE-[project].txt. https://npmjs.com/package/@floating-ui/core/v/1.6.9 1.6.9 MIT https://npmjs.com/package/@floating-ui/dom/v/1.6.13 1.6.13 MIT https://npmjs.com/package/@floating-ui/utils/v/0.2.9 0.2.9 MIT - https://npmjs.com/package/@interactjs/actions/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/auto-scroll/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/auto-start/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/core/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/dev-tools/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/inertia/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/interact/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/interactjs/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/modifiers/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/offset/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/pointer-events/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/reflow/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/snappers/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/utils/v/1.10.27 1.10.27 MIT + https://npmjs.com/package/@interactjs/actions/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/auto-scroll/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/auto-start/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/core/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/dev-tools/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/inertia/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/interact/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/interactjs/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/modifiers/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/offset/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/pointer-events/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/reflow/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/snappers/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/types/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/utils/v/1.10.17 1.10.17 MIT https://npmjs.com/package/@intlify/core-base/v/9.14.5 9.14.5 MIT https://npmjs.com/package/@intlify/message-compiler/v/9.14.5 9.14.5 MIT https://npmjs.com/package/@intlify/shared/v/9.14.5 9.14.5 MIT @@ -528,7 +529,7 @@ The text of each license is also included in licenses/LICENSE-[project].txt. https://npmjs.com/package/d3-dsv/node_modules/commander/v/7.2.0 7.2.0 MIT https://npmjs.com/package/d3-tip/v/0.9.1 0.9.1 MIT https://npmjs.com/package/dayjs/v/1.11.13 1.11.13 MIT - https://npmjs.com/package/element-plus/v/2.11.0 2.11.0 MIT + https://npmjs.com/package/element-plus/v/2.9.4 2.9.4 MIT https://npmjs.com/package/element-resize-detector/v/1.2.4 1.2.4 MIT https://npmjs.com/package/escape-html/v/1.0.3 1.0.3 MIT https://npmjs.com/package/estree-walker/v/2.0.2 2.0.2 MIT diff --git a/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml index 99aff4e41d90..7502cc718d2f 100644 --- a/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml @@ -23,16 +23,13 @@ services: networks: - e2e - provider: + go-service: extends: file: ../../../../script/docker-compose/base-compose.yml - service: provider - privileged: true + service: go-service depends_on: oap: condition: service_healthy - ports: - - 9090 oap: extends: diff --git a/test/e2e-v2/cases/profiling/pprof/expected/list.yml b/test/e2e-v2/cases/profiling/pprof/expected/list.yml index 645ece7fc066..47a327cede43 100644 --- a/test/e2e-v2/cases/profiling/pprof/expected/list.yml +++ b/test/e2e-v2/cases/profiling/pprof/expected/list.yml @@ -23,7 +23,5 @@ tasks: createtime: {{ gt .createtime 0 }} events: - CPU - - ALLOC duration: {{ ge .duration 0 }} - execargs: null {{- end }} \ No newline at end of file diff --git a/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml b/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml index f32cc2b0f1e4..50b90d8fe50a 100644 --- a/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml +++ b/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml @@ -26,15 +26,9 @@ value: {{ notEmpty .value }} - name: Process No. value: {{ notEmpty .value }} - - name: Start Time - value: {{ notEmpty .value }} - - name: JVM Arguments - value: '{{ notEmpty .value }}' - - name: Jar Dependencies - value: '{{ notEmpty .value }}' - name: ipv4s value: {{ notEmpty .value }} {{- end}} - language: JAVA + language: GO instanceuuid: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} {{- end}} diff --git a/test/e2e-v2/cases/profiling/pprof/mysql/e2e.yml b/test/e2e-v2/cases/profiling/pprof/mysql/e2e.yaml similarity index 100% rename from test/e2e-v2/cases/profiling/pprof/mysql/e2e.yml rename to test/e2e-v2/cases/profiling/pprof/mysql/e2e.yaml From f49e7a36f0a24007be0fffcc46bc8793f1266c69 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Tue, 14 Oct 2025 21:37:40 +0800 Subject: [PATCH 17/69] fix e2e --- .github/workflows/skywalking.yaml | 375 ------------------ test/e2e-v2/cases/go/docker-compose.yml | 1 + test/e2e-v2/cases/go/service/go.mod | 2 +- .../pprof/banyandb/docker-compose.yml | 5 +- .../profiling/pprof/expected/analysis.yml | 1 - .../cases/profiling/pprof/expected/list.yml | 4 +- .../profiling/pprof/expected/progress.yml | 4 +- .../pprof/expected/service-instance.yml | 6 +- .../profiling/pprof/expected/service.yml | 7 +- .../profiling/pprof/mysql/docker-compose.yml | 7 +- .../profiling/pprof/profiling-cases.yaml | 15 +- test/e2e-v2/script/env | 2 +- 12 files changed, 28 insertions(+), 401 deletions(-) diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index c2136b1feb53..53f9a87d4e28 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -345,381 +345,6 @@ jobs: fail-fast: false matrix: test: - - name: Cluster ZK/ES - config: test/e2e-v2/cases/cluster/zk/es/e2e.yaml - - - name: Agent NodeJS Backend - config: test/e2e-v2/cases/nodejs/e2e.yaml - - name: Agent Golang - config: test/e2e-v2/cases/go/e2e.yaml - - name: Agent NodeJS Frontend - config: test/e2e-v2/cases/browser/e2e.yaml - - name: Agent NodeJS Frontend ES - config: test/e2e-v2/cases/browser/es/e2e.yaml - - name: Agent NodeJS Frontend ES Sharding - config: test/e2e-v2/cases/browser/es/es-sharding/e2e.yaml - - name: Agent PHP - config: test/e2e-v2/cases/php/e2e.yaml - - name: Agent Python - config: test/e2e-v2/cases/python/e2e.yaml - - name: Agent Lua - config: test/e2e-v2/cases/lua/e2e.yaml - - - name: BanyanDB - config: test/e2e-v2/cases/storage/banyandb/e2e.yaml - - name: BanyanDB TLS - config: test/e2e-v2/cases/storage/banyandb/tls/e2e.yaml - - name: Storage MySQL - config: test/e2e-v2/cases/storage/mysql/e2e.yaml - - name: Storage PostgreSQL - config: test/e2e-v2/cases/storage/postgres/e2e.yaml - - name: Storage ES 7.16.3 - config: test/e2e-v2/cases/storage/es/e2e.yaml - env: ES_VERSION=7.16.3 - - name: Storage ES 7.17.10 - config: test/e2e-v2/cases/storage/es/e2e.yaml - env: ES_VERSION=7.17.10 - - name: Storage ES 8.1.0 - config: test/e2e-v2/cases/storage/es/e2e.yaml - env: ES_VERSION=8.1.0 - - name: Storage ES 8.9.0 - config: test/e2e-v2/cases/storage/es/e2e.yaml - env: ES_VERSION=8.9.0 - - name: Storage ES 8.9.0 - config: test/e2e-v2/cases/storage/es/e2e.yaml - env: ES_VERSION=8.18.1 - - name: Storage OpenSearch 1.1.0 - config: test/e2e-v2/cases/storage/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=1.1.0 - - name: Storage OpenSearch 1.3.10 - config: test/e2e-v2/cases/storage/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=1.3.10 - - name: Storage OpenSearch 2.4.0 - config: test/e2e-v2/cases/storage/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=2.4.0 - - name: Storage OpenSearch 2.8.0 - config: test/e2e-v2/cases/storage/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=2.8.0 - - name: Storage OpenSearch 3.0.0 - config: test/e2e-v2/cases/storage/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=3.0.0 - - name: Storage ES Sharding - config: test/e2e-v2/cases/storage/es/es-sharding/e2e.yaml - - - name: Alarm ES - config: test/e2e-v2/cases/alarm/es/e2e.yaml - - name: Alarm ES Sharding - config: test/e2e-v2/cases/alarm/es/es-sharding/e2e.yaml - - name: Alarm MySQL - config: test/e2e-v2/cases/alarm/mysql/e2e.yaml - - name: Alarm PostgreSQL - config: test/e2e-v2/cases/alarm/postgres/e2e.yaml - - name: Alarm BanyanDB - config: test/e2e-v2/cases/alarm/banyandb/e2e.yaml - - - name: Baseline-driven Alarm ES - config: test/e2e-v2/cases/baseline/es/e2e.yaml - - name: Baseline-driven Alarm ES Sharding - config: test/e2e-v2/cases/baseline/es/es-sharding/e2e.yaml - - name: Baseline-driven Alarm BanyanDB - config: test/e2e-v2/cases/baseline/banyandb/e2e.yaml - - - name: TTL ES 7.16.3 - config: test/e2e-v2/cases/ttl/es/e2e.yaml - env: ES_VERSION=7.16.3 - - name: TTL ES 8.8.1 - config: test/e2e-v2/cases/ttl/es/e2e.yaml - env: ES_VERSION=8.8.1 - - name: TTL ES 8.18.1 - config: test/e2e-v2/cases/ttl/es/e2e.yaml - env: ES_VERSION=8.18.1 - - - name: Event BanyanDB - config: test/e2e-v2/cases/event/banyandb/e2e.yaml - - name: Event ES - config: test/e2e-v2/cases/event/es/e2e.yaml - - name: Event MySQL - config: test/e2e-v2/cases/event/mysql/e2e.yaml - - - name: Log MySQL - config: test/e2e-v2/cases/log/mysql/e2e.yaml - - name: Log PostgreSQL - config: test/e2e-v2/cases/log/postgres/e2e.yaml - - name: Log ES 7.16.3 - config: test/e2e-v2/cases/log/es/e2e.yaml - env: ES_VERSION=7.16.3 - - name: Log ES 7.17.10 - config: test/e2e-v2/cases/log/es/e2e.yaml - env: ES_VERSION=7.17.10 - - name: Log ES 8.8.1 Sharding - config: test/e2e-v2/cases/log/es/es-sharding/e2e.yaml - env: ES_VERSION=8.8.1 - - name: Log ES 8.18.1 Sharding - config: test/e2e-v2/cases/log/es/es-sharding/e2e.yaml - env: ES_VERSION=8.18.1 - - name: Log BanyanDB - config: test/e2e-v2/cases/log/banyandb/e2e.yaml - - - name: Log FluentBit ES 7.16.3 - config: test/e2e-v2/cases/log/fluent-bit/e2e.yaml - env: ES_VERSION=7.16.3 - - name: Log FluentBit ES 7.17.10 - config: test/e2e-v2/cases/log/fluent-bit/e2e.yaml - env: ES_VERSION=7.17.10 - - name: Log FluentBit ES 8.8.1 - config: test/e2e-v2/cases/log/fluent-bit/e2e.yaml - env: ES_VERSION=8.8.1 - - name: Log FluentBit ES 8.18.1 - config: test/e2e-v2/cases/log/fluent-bit/e2e.yaml - env: ES_VERSION=8.18.1 - - - name: Trace Profiling BanyanDB - config: test/e2e-v2/cases/profiling/trace/banyandb/e2e.yaml - - name: Trace Profiling ES - config: test/e2e-v2/cases/profiling/trace/es/e2e.yaml - - name: Trace Profiling ES Sharding - config: test/e2e-v2/cases/profiling/trace/es/es-sharding/e2e.yaml - - name: Trace Profiling MySQL - config: test/e2e-v2/cases/profiling/trace/mysql/e2e.yaml - - name: Trace Profiling Postgres - config: test/e2e-v2/cases/profiling/trace/postgres/e2e.yaml - - name: Trace Profiling OpenSearch 1.1.0 - config: test/e2e-v2/cases/profiling/trace/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=1.1.0 - - name: Trace Profiling OpenSearch 1.3.6 - config: test/e2e-v2/cases/profiling/trace/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=1.3.6 - - name: Trace Profiling OpenSearch 2.4.0 - config: test/e2e-v2/cases/profiling/trace/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=2.4.0 - - - name: eBPF Profiling On CPU BanyanDB - config: test/e2e-v2/cases/profiling/ebpf/oncpu/banyandb/e2e.yaml - docker: - base: test/e2e-v2/cases/profiling/ebpf/oncpu/ - file: Dockerfile.sqrt - name: test/oncpu:test - - name: eBPF Profiling On CPU ES - config: test/e2e-v2/cases/profiling/ebpf/oncpu/es/e2e.yaml - docker: - base: test/e2e-v2/cases/profiling/ebpf/oncpu/ - file: Dockerfile.sqrt - name: test/oncpu:test - - name: eBPF Profiling On CPU ES Sharding - config: test/e2e-v2/cases/profiling/ebpf/oncpu/es/es-sharding/e2e.yaml - docker: - base: test/e2e-v2/cases/profiling/ebpf/oncpu/ - file: Dockerfile.sqrt - name: test/oncpu:test - - name: eBPF Profiling Off CPU - config: test/e2e-v2/cases/profiling/ebpf/offcpu/e2e.yaml - docker: - base: test/e2e-v2/cases/profiling/ebpf/offcpu/ - file: Dockerfile.file - name: test/offcpu:test - runs-on: ubuntu-24.04 - - - name: eBPF Profiling Network BanyanDB - config: test/e2e-v2/cases/profiling/ebpf/network/banyandb/e2e.yaml - docker: - base: test/e2e-v2/cases/profiling/ebpf/network/ - file: Dockerfile.service - name: test/network:test - - name: eBPF Profiling Network ES - config: test/e2e-v2/cases/profiling/ebpf/network/es/e2e.yaml - docker: - base: test/e2e-v2/cases/profiling/ebpf/network/ - file: Dockerfile.service - name: test/network:test - - name: eBPF Profiling Network ES Sharding - config: test/e2e-v2/cases/profiling/ebpf/network/es/es-sharding/e2e.yaml - docker: - base: test/e2e-v2/cases/profiling/ebpf/network/ - file: Dockerfile.service - name: test/network:test - - - name: Continuous Profiling BanyanDB - config: test/e2e-v2/cases/profiling/ebpf/continuous/banyandb/e2e.yaml - docker: - base: test/e2e-v2/cases/profiling/ebpf/continuous/ - file: Dockerfile.sqrt - name: test/continuous:test - - name: Continuous Profiling ES - config: test/e2e-v2/cases/profiling/ebpf/continuous/es/e2e.yaml - docker: - base: test/e2e-v2/cases/profiling/ebpf/continuous/ - file: Dockerfile.sqrt - name: test/continuous:test - - name: Continuous Profiling Sharding ES - config: test/e2e-v2/cases/profiling/ebpf/continuous/es/es-sharding/e2e.yaml - docker: - base: test/e2e-v2/cases/profiling/ebpf/continuous/ - file: Dockerfile.sqrt - name: test/continuous:test - - # eBPF Access Log - - name: eBPF Access Log BanyanDB - config: test/e2e-v2/cases/profiling/ebpf/access_log/banyandb/e2e.yaml - - name: eBPF Access Log ES - config: test/e2e-v2/cases/profiling/ebpf/access_log/es/e2e.yaml - - name: eBPF Access Log ES Sharding - config: test/e2e-v2/cases/profiling/ebpf/access_log/es/es-sharding/e2e.yaml - - - name: Kafka Basic - config: test/e2e-v2/cases/kafka/simple-so11y/e2e.yaml - - name: Kafka Profiling - config: test/e2e-v2/cases/kafka/profile/e2e.yaml - - name: Kafka Meter - config: test/e2e-v2/cases/kafka/meter/e2e.yaml - - name: Kafka Log - config: test/e2e-v2/cases/kafka/log/e2e.yaml - - - name: Istio Metrics Service 1.20.0 - config: test/e2e-v2/cases/istio/metrics/e2e.yaml - env: | - ISTIO_VERSION=1.20.0 - KUBERNETES_VERSION=28 - - name: Istio Metrics Service 1.21.0 - config: test/e2e-v2/cases/istio/metrics/e2e.yaml - env: | - ISTIO_VERSION=1.21.0 - KUBERNETES_VERSION=28 - - name: Istio Metrics Service 1.22.0 - config: test/e2e-v2/cases/istio/metrics/e2e.yaml - env: | - ISTIO_VERSION=1.22.0 - KUBERNETES_VERSION=28 - - name: Istio Metrics Service 1.23.0 - config: test/e2e-v2/cases/istio/metrics/e2e.yaml - env: | - ISTIO_VERSION=1.23.0 - KUBERNETES_VERSION=28 - - name: Istio Metrics Service 1.24.0 - config: test/e2e-v2/cases/istio/metrics/e2e.yaml - env: | - ISTIO_VERSION=1.24.0 - KUBERNETES_VERSION=28 - - - name: Rover with Istio Process 1.15.0 - config: test/e2e-v2/cases/rover/process/istio/e2e.yaml - env: ISTIO_VERSION=1.15.0 - runs-on: ubuntu-24.04 - - - name: Satellite - config: test/e2e-v2/cases/satellite/native-protocols/e2e.yaml - - name: Auth - config: test/e2e-v2/cases/simple/auth/e2e.yaml - - name: SSL - config: test/e2e-v2/cases/simple/ssl/e2e.yaml - - name: mTLS - config: test/e2e-v2/cases/simple/mtls/e2e.yaml - - name: Virtual Gateway - config: test/e2e-v2/cases/gateway/e2e.yaml - - name: Meter - config: test/e2e-v2/cases/meter/e2e.yaml - - name: VM Zabbix - config: test/e2e-v2/cases/vm/zabbix/e2e.yaml - - name: VM Prometheus - config: test/e2e-v2/cases/vm/prometheus-node-exporter/e2e.yaml - - name: VM Telegraf - config: test/e2e-v2/cases/vm/telegraf/e2e.yaml - - name: So11y - config: test/e2e-v2/cases/so11y/e2e.yaml - - name: MySQL Prometheus and slowsql - config: test/e2e-v2/cases/mysql/mysql-slowsql/e2e.yaml - - name: PostgreSQL Prometheus - config: test/e2e-v2/cases/postgresql/postgres-exporter/e2e.yaml - - name: MariaDB Prometheus and slowsql - config: test/e2e-v2/cases/mariadb/mariadb-slowsql/e2e.yaml - - - name: Zipkin ES - config: test/e2e-v2/cases/zipkin/es/e2e.yaml - - name: Zipkin ES Sharding - config: test/e2e-v2/cases/zipkin/es/es-sharding/e2e.yaml - - name: Zipkin MySQL - config: test/e2e-v2/cases/zipkin/mysql/e2e.yaml - - name: Zipkin Opensearch - config: test/e2e-v2/cases/zipkin/opensearch/e2e.yaml - - name: Zipkin Postgres - config: test/e2e-v2/cases/zipkin/postgres/e2e.yaml - - name: Zipkin Kafka - config: test/e2e-v2/cases/zipkin/kafka/e2e.yaml - - name: Zipkin BanyanDB - config: test/e2e-v2/cases/zipkin/banyandb/e2e.yaml - - - name: Nginx - config: test/e2e-v2/cases/nginx/e2e.yaml - - name: APISIX metrics - config: test/e2e-v2/cases/apisix/otel-collector/e2e.yaml - - name: Exporter Kafka - config: test/e2e-v2/cases/exporter/kafka/e2e.yaml - - name: Virtual MQ - config: test/e2e-v2/cases/virtual-mq/e2e.yaml - - name: AWS Cloud EKS - config: test/e2e-v2/cases/aws/eks/e2e.yaml - - name: Windows - config: test/e2e-v2/cases/win/e2e.yaml - - name: AWS Cloud S3 - config: test/e2e-v2/cases/aws/s3/e2e.yaml - - name: AWS Cloud DynamoDB - config: test/e2e-v2/cases/aws/dynamodb/e2e.yaml - - name: PromQL Service - config: test/e2e-v2/cases/promql/e2e.yaml - - name: LogQL Service - config: test/e2e-v2/cases/logql/e2e.yaml - - name: AWS API Gateway - config: test/e2e-v2/cases/aws/api-gateway/e2e.yaml - - name: Redis Prometheus and Log Collecting - config: test/e2e-v2/cases/redis/redis-exporter/e2e.yaml - - name: Elasticsearch - config: test/e2e-v2/cases/elasticsearch/e2e.yaml - - name: MongoDB - config: test/e2e-v2/cases/mongodb/e2e.yaml - - name: RabbitMQ - config: test/e2e-v2/cases/rabbitmq/e2e.yaml - - name: Kafka - config: test/e2e-v2/cases/kafka/kafka-monitoring/e2e.yaml - - name: MQE Service - config: test/e2e-v2/cases/mqe/e2e.yaml - - name: Pulsar and BookKeeper - config: test/e2e-v2/cases/pulsar/e2e.yaml - - name: RocketMQ - config: test/e2e-v2/cases/rocketmq/e2e.yaml - - name: ClickHouse - config: test/e2e-v2/cases/clickhouse/clickhouse-prometheus-endpoint/e2e.yaml - - name: ActiveMQ - config: test/e2e-v2/cases/activemq/e2e.yaml - - name: Kong - config: test/e2e-v2/cases/kong/e2e.yaml - - name: Flink - config: test/e2e-v2/cases/flink/e2e.yaml - - - name: UI Menu BanyanDB - config: test/e2e-v2/cases/menu/banyandb/e2e.yaml - - name: UI Menu ES - config: test/e2e-v2/cases/menu/es/e2e.yaml - - name: UI Menu Sharding ES - config: test/e2e-v2/cases/menu/es/es-sharding/e2e.yaml - - name: UI Menu MySQL - config: test/e2e-v2/cases/menu/mysql/e2e.yaml - - name: UI Menu Postgres - config: test/e2e-v2/cases/menu/postgres/e2e.yaml - - name: UI Menu OpenSearch 1.1.0 - config: test/e2e-v2/cases/menu/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=1.1.0 - - name: UI Menu OpenSearch 1.3.6 - config: test/e2e-v2/cases/menu/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=1.3.6 - - name: UI Menu OpenSearch 2.4.0 - config: test/e2e-v2/cases/menu/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=2.4.0 - - - name: OTLP Trace - config: test/e2e-v2/cases/otlp-traces/e2e.yaml - - - name: Cilium Service - config: test/e2e-v2/cases/cilium/e2e.yaml - - name: Async Profiler ES config: test/e2e-v2/cases/profiling/async-profiler/es/e2e.yaml - name: Async Profiler BanyanDB diff --git a/test/e2e-v2/cases/go/docker-compose.yml b/test/e2e-v2/cases/go/docker-compose.yml index 8dc12b67604d..cfae30d2df11 100644 --- a/test/e2e-v2/cases/go/docker-compose.yml +++ b/test/e2e-v2/cases/go/docker-compose.yml @@ -50,6 +50,7 @@ services: - 8080 environment: SW_AGENT_NAME: go-service + SW_AGENT_INSTANCE_NAME: provider1 SW_AGENT_REPORTER_GRPC_BACKEND_SERVICE: oap:11800 UPSTREAM_URL: http://provider:9090/correlation depends_on: diff --git a/test/e2e-v2/cases/go/service/go.mod b/test/e2e-v2/cases/go/service/go.mod index 2c412645299f..783387b19bf9 100644 --- a/test/e2e-v2/cases/go/service/go.mod +++ b/test/e2e-v2/cases/go/service/go.mod @@ -20,7 +20,7 @@ module sw-e2e go 1.19 require ( - github.com/apache/skywalking-go v0.5.1-0.20250301084827-154de50628e8 + github.com/apache/skywalking-go v0.6.0 github.com/apache/skywalking-go/toolkit v0.5.1-0.20250301084827-154de50628e8 github.com/gin-gonic/gin v1.10.0 ) diff --git a/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml index 7502cc718d2f..b765e537ce5c 100644 --- a/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml @@ -25,11 +25,14 @@ services: go-service: extends: - file: ../../../../script/docker-compose/base-compose.yml + file: ../../../../go/docker-compose.yml service: go-service + privileged: true depends_on: oap: condition: service_healthy + ports: + - 9090 oap: extends: diff --git a/test/e2e-v2/cases/profiling/pprof/expected/analysis.yml b/test/e2e-v2/cases/profiling/pprof/expected/analysis.yml index 265c20c55490..7f04a1d43f79 100644 --- a/test/e2e-v2/cases/profiling/pprof/expected/analysis.yml +++ b/test/e2e-v2/cases/profiling/pprof/expected/analysis.yml @@ -14,7 +14,6 @@ # limitations under the License. tree: - type: EXECUTION_SAMPLE elements: {{- contains .tree.elements }} - id: {{ notEmpty .id }} diff --git a/test/e2e-v2/cases/profiling/pprof/expected/list.yml b/test/e2e-v2/cases/profiling/pprof/expected/list.yml index 47a327cede43..963f50e9ea96 100644 --- a/test/e2e-v2/cases/profiling/pprof/expected/list.yml +++ b/test/e2e-v2/cases/profiling/pprof/expected/list.yml @@ -17,9 +17,9 @@ errorreason: null tasks: {{- contains .tasks }} - id: {{ notEmpty .id }} - serviceid: {{ b64enc "e2e-service-provider" }}.1 + serviceid: {{ b64enc "go-service" }}.1 serviceinstanceids: - - {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} + - {{ b64enc "go-service" }}.1_{{ b64enc "provider1" }} createtime: {{ gt .createtime 0 }} events: - CPU diff --git a/test/e2e-v2/cases/profiling/pprof/expected/progress.yml b/test/e2e-v2/cases/profiling/pprof/expected/progress.yml index 257376e0f047..9c285f08c530 100644 --- a/test/e2e-v2/cases/profiling/pprof/expected/progress.yml +++ b/test/e2e-v2/cases/profiling/pprof/expected/progress.yml @@ -16,11 +16,11 @@ logs: {{- contains .logs }} - id: {{ notEmpty .id}} - instanceid: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} + instanceid: {{ b64enc "go-service" }}.1_{{ b64enc "provider1" }} instancename: {{ notEmpty .instancename}} operationtype: {{ notEmpty .operationtype}} operationtime: {{ ge .operationtime 0 }} {{- end }} errorinstanceids: [] successinstanceids: - - {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} + - {{ b64enc "go-service" }}.1_{{ b64enc "provider1" }} diff --git a/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml b/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml index 50b90d8fe50a..f5f861672506 100644 --- a/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml +++ b/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml @@ -16,8 +16,8 @@ # under the License. {{- contains . }} -- id: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} - name: provider1 +- id: {{ notEmpty .id }} + name: {{ notEmpty .name }} attributes: {{- contains .attributes }} - name: OS Name @@ -30,5 +30,5 @@ value: {{ notEmpty .value }} {{- end}} language: GO - instanceuuid: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} + instanceuuid: {{ notEmpty .instanceuuid }} {{- end}} diff --git a/test/e2e-v2/cases/profiling/pprof/expected/service.yml b/test/e2e-v2/cases/profiling/pprof/expected/service.yml index 75d0916f00b4..5744ee6c74b4 100644 --- a/test/e2e-v2/cases/profiling/pprof/expected/service.yml +++ b/test/e2e-v2/cases/profiling/pprof/expected/service.yml @@ -14,11 +14,12 @@ # limitations under the License. {{- contains . }} -- id: {{ b64enc "e2e-service-provider" }}.1 - name: e2e-service-provider +- id: {{ b64enc "go-service" }}.1 + name: go-service group: "" - shortname: e2e-service-provider + shortname: go-service normal: true layers: - GENERAL + - SO11Y_GO_AGENT {{- end }} diff --git a/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml index 8b98f4d64bb4..dbaad2a02c97 100644 --- a/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml @@ -31,11 +31,10 @@ services: interval: 5s timeout: 60s retries: 120 - - provider: + go-service: extends: - file: ../../../../script/docker-compose/base-compose.yml - service: provider + file: ../../../../go/docker-compose.yml + service: go-service privileged: true depends_on: oap: diff --git a/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml b/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml index 8a9fff6c15fb..666a0cc88c47 100644 --- a/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml +++ b/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml @@ -20,31 +20,30 @@ cases: - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls expected: expected/service.yml # service instance list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=go-service expected: expected/service-instance.yml # create task - query: | swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ - profiling pprof create --service-name=e2e-service-provider \ - --duration=20 --events=CPU \ + profiling pprof create --service-name=go-service \ + --duration=2 --events=CPU \ --instance-name-list=provider1 expected: expected/create.yml # list task - query: | swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ - profiling pprof list --service-name=e2e-service-provider \ + profiling pprof list --service-name=go-service \ expected: expected/list.yml # get task progress - query: | swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ - profiling pprof progress --task-id=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling pprof list --service-name=e2e-service-provider | yq e '.tasks[0].id') + profiling pprof progress --task-id=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling pprof list --service-name=go-service | yq e '.tasks[0].id') expected: expected/progress.yml # get task analysis - query: | swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ - profiling pprof analysis --service-name=e2e-service-provider \ - --event=execution_sample \ + profiling pprof analysis --service-name=go-service \ --instance-name-list=provider1 \ - --task-id=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling pprof list --service-name=e2e-service-provider | yq e '.tasks[0].id') + --task-id=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling pprof list --service-name=go-service | yq e '.tasks[0].id') expected: expected/analysis.yml diff --git a/test/e2e-v2/script/env b/test/e2e-v2/script/env index 3cc8ef7583a7..a04eee45ffd3 100644 --- a/test/e2e-v2/script/env +++ b/test/e2e-v2/script/env @@ -17,7 +17,7 @@ SW_AGENT_JAVA_COMMIT=f0245864e4388a388fe7445b56b6ce7cedc94aaf SW_AGENT_SATELLITE_COMMIT=ea27a3f4e126a24775fe12e2aa2695bcb23d99c3 SW_AGENT_NGINX_LUA_COMMIT=c3cee4841798a147d83b96a10914d4ac0e11d0aa SW_AGENT_NODEJS_COMMIT=4f9a91dad3dfd8cfe5ba8f7bd06b39e11eb5e65e -SW_AGENT_GO_COMMIT=154de50628e82e590941585411299459e352317d +SW_AGENT_GO_COMMIT=aa948377ecdb4724fad1cc365c13a1188021316f SW_AGENT_PYTHON_COMMIT=c76a6ec51a478ac91abb20ec8f22a99b8d4d6a58 SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449 SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016 From 3449d895f6cced23288ecfe3c80e46cc003d33a5 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Wed, 15 Oct 2025 00:30:01 +0800 Subject: [PATCH 18/69] fix e2e docker path --- .../cases/profiling/pprof/banyandb/docker-compose.yml | 2 +- test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml | 6 +++--- .../cases/profiling/pprof/expected/service-instance.yml | 6 +++--- test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml index b765e537ce5c..9b7ad7735e09 100644 --- a/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml @@ -25,7 +25,7 @@ services: go-service: extends: - file: ../../../../go/docker-compose.yml + file: ../../../go/docker-compose.yml service: go-service privileged: true depends_on: diff --git a/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml index 916ad2903f2e..31952cd3463e 100644 --- a/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml @@ -31,10 +31,10 @@ services: timeout: 60s retries: 120 - provider: + go-service: extends: - file: ../../../../script/docker-compose/base-compose.yml - service: provider + file: ../../../go/docker-compose.yml + service: go-service privileged: true depends_on: oap: diff --git a/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml b/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml index f5f861672506..83f07be75715 100644 --- a/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml +++ b/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml @@ -16,8 +16,8 @@ # under the License. {{- contains . }} -- id: {{ notEmpty .id }} - name: {{ notEmpty .name }} +- id: {{ b64enc "go-service" }}.1_{{ b64enc "provider1" }} + name: provider1 attributes: {{- contains .attributes }} - name: OS Name @@ -30,5 +30,5 @@ value: {{ notEmpty .value }} {{- end}} language: GO - instanceuuid: {{ notEmpty .instanceuuid }} + instanceuuid: {{ b64enc "go-service" }}.1_{{ b64enc "provider1" }} {{- end}} diff --git a/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml index dbaad2a02c97..061877ad4c6f 100644 --- a/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml @@ -33,7 +33,7 @@ services: retries: 120 go-service: extends: - file: ../../../../go/docker-compose.yml + file: ../../../go/docker-compose.yml service: go-service privileged: true depends_on: From 26956b06c18de949090ee2efa9ce45437a1ae42b Mon Sep 17 00:00:00 2001 From: JophieQu Date: Wed, 15 Oct 2025 01:07:34 +0800 Subject: [PATCH 19/69] fix --- .../pprof/banyandb/docker-compose.yml | 21 ++++++++++++++++--- .../profiling/pprof/es/docker-compose.yml | 21 ++++++++++++++++--- .../profiling/pprof/mysql/docker-compose.yml | 21 ++++++++++++++++--- 3 files changed, 54 insertions(+), 9 deletions(-) diff --git a/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml index 9b7ad7735e09..17df60f2b753 100644 --- a/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml @@ -24,13 +24,28 @@ services: - e2e go-service: - extends: - file: ../../../go/docker-compose.yml - service: go-service + build: + context: ../../../go/service + dockerfile: Dockerfile + args: + - SW_AGENT_GO_COMMIT=${SW_AGENT_GO_COMMIT} + networks: + - e2e + expose: + - 8080 + environment: + SW_AGENT_NAME: go-service + SW_AGENT_INSTANCE_NAME: provider1 + SW_AGENT_REPORTER_GRPC_BACKEND_SERVICE: oap:11800 privileged: true depends_on: oap: condition: service_healthy + healthcheck: + test: ["CMD", "sh", "-c", "nc -z 127.0.0.1 8080"] + interval: 5s + timeout: 60s + retries: 120 ports: - 9090 diff --git a/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml index 31952cd3463e..7cf39dfc2d6d 100644 --- a/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml @@ -32,13 +32,28 @@ services: retries: 120 go-service: - extends: - file: ../../../go/docker-compose.yml - service: go-service + build: + context: ../../../go/service + dockerfile: Dockerfile + args: + - SW_AGENT_GO_COMMIT=${SW_AGENT_GO_COMMIT} + networks: + - e2e + expose: + - 8080 + environment: + SW_AGENT_NAME: go-service + SW_AGENT_INSTANCE_NAME: provider1 + SW_AGENT_REPORTER_GRPC_BACKEND_SERVICE: oap:11800 privileged: true depends_on: oap: condition: service_healthy + healthcheck: + test: ["CMD", "sh", "-c", "nc -z 127.0.0.1 8080"] + interval: 5s + timeout: 60s + retries: 120 ports: - 9090 diff --git a/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml index 061877ad4c6f..b9d61783e35c 100644 --- a/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml @@ -32,13 +32,28 @@ services: timeout: 60s retries: 120 go-service: - extends: - file: ../../../go/docker-compose.yml - service: go-service + build: + context: ../../../go/service + dockerfile: Dockerfile + args: + - SW_AGENT_GO_COMMIT=${SW_AGENT_GO_COMMIT} + networks: + - e2e + expose: + - 8080 + environment: + SW_AGENT_NAME: go-service + SW_AGENT_INSTANCE_NAME: provider1 + SW_AGENT_REPORTER_GRPC_BACKEND_SERVICE: oap:11800 privileged: true depends_on: oap: condition: service_healthy + healthcheck: + test: ["CMD", "sh", "-c", "nc -z 127.0.0.1 8080"] + interval: 5s + timeout: 60s + retries: 120 ports: - 9090 From baaf886fbee3a1619aff700d78d8ab2bcab9757b Mon Sep 17 00:00:00 2001 From: JophieQu Date: Wed, 15 Oct 2025 07:55:39 +0800 Subject: [PATCH 20/69] fix --- test/e2e-v2/cases/go/service/go.mod | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/e2e-v2/cases/go/service/go.mod b/test/e2e-v2/cases/go/service/go.mod index 783387b19bf9..bc338fccb1f4 100644 --- a/test/e2e-v2/cases/go/service/go.mod +++ b/test/e2e-v2/cases/go/service/go.mod @@ -20,7 +20,7 @@ module sw-e2e go 1.19 require ( - github.com/apache/skywalking-go v0.6.0 + github.com/apache/skywalking-go v0.6.1-0.20250924145416-aa948377ecdb github.com/apache/skywalking-go/toolkit v0.5.1-0.20250301084827-154de50628e8 github.com/gin-gonic/gin v1.10.0 ) @@ -40,13 +40,16 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/uuid v1.3.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.15.9 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/segmentio/kafka-go v0.4.43 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect golang.org/x/arch v0.8.0 // indirect @@ -58,5 +61,4 @@ require ( google.golang.org/grpc v1.55.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - skywalking.apache.org/repo/goapi v0.0.0-20230314034821-0c5a44bb767a // indirect ) From 62fd3fb1eb0006ff2cb2fee50581c3627122f5c9 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Wed, 15 Oct 2025 08:54:21 +0800 Subject: [PATCH 21/69] fix --- test/e2e-v2/cases/go/service/Dockerfile | 2 +- test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml | 1 + test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml | 1 + test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/test/e2e-v2/cases/go/service/Dockerfile b/test/e2e-v2/cases/go/service/Dockerfile index 79dc8b1a8366..3eb7c791093f 100644 --- a/test/e2e-v2/cases/go/service/Dockerfile +++ b/test/e2e-v2/cases/go/service/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. ARG SW_AGENT_GO_COMMIT= -FROM ghcr.io/apache/skywalking-go/skywalking-go:${SW_AGENT_GO_COMMIT}-go1.19 as base +FROM ghcr.io/apache/skywalking-go/skywalking-go:${SW_AGENT_GO_COMMIT}-go1.19 AS base ENV CGO_ENABLED=0 ENV GO111MODULE=on diff --git a/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml index 17df60f2b753..801c97b05caf 100644 --- a/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml @@ -37,6 +37,7 @@ services: SW_AGENT_NAME: go-service SW_AGENT_INSTANCE_NAME: provider1 SW_AGENT_REPORTER_GRPC_BACKEND_SERVICE: oap:11800 + UPSTREAM_URL: http://localhost:8080/ignored.html privileged: true depends_on: oap: diff --git a/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml index 7cf39dfc2d6d..4003e947060a 100644 --- a/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml @@ -45,6 +45,7 @@ services: SW_AGENT_NAME: go-service SW_AGENT_INSTANCE_NAME: provider1 SW_AGENT_REPORTER_GRPC_BACKEND_SERVICE: oap:11800 + UPSTREAM_URL: http://localhost:8080/ignored.html privileged: true depends_on: oap: diff --git a/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml index b9d61783e35c..757b06261657 100644 --- a/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml @@ -45,6 +45,7 @@ services: SW_AGENT_NAME: go-service SW_AGENT_INSTANCE_NAME: provider1 SW_AGENT_REPORTER_GRPC_BACKEND_SERVICE: oap:11800 + UPSTREAM_URL: http://localhost:8080/ignored.html privileged: true depends_on: oap: From 88692d1a431ad8b46dfd98e5713c250a75eb0d76 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Wed, 15 Oct 2025 08:59:56 +0800 Subject: [PATCH 22/69] fix duration --- test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml b/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml index 666a0cc88c47..dd0da0216728 100644 --- a/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml +++ b/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml @@ -26,7 +26,7 @@ cases: - query: | swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ profiling pprof create --service-name=go-service \ - --duration=2 --events=CPU \ + --duration=1 --events=CPU \ --instance-name-list=provider1 expected: expected/create.yml # list task From 843585fac28599a286a89aae0d1f236492d05b26 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Wed, 15 Oct 2025 13:50:59 +0800 Subject: [PATCH 23/69] fix --- .github/workflows/codeql.yaml | 2 -- test/e2e-v2/cases/go/service/Dockerfile | 5 +++-- .../e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml | 1 + test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml | 1 + test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml | 1 + 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 038e92bcced6..5be152e675fc 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -18,8 +18,6 @@ name: "CodeQL" on: push: - branches: - - ci pull_request: branches: ["master"] paths: diff --git a/test/e2e-v2/cases/go/service/Dockerfile b/test/e2e-v2/cases/go/service/Dockerfile index 3eb7c791093f..2f16f90a1cda 100644 --- a/test/e2e-v2/cases/go/service/Dockerfile +++ b/test/e2e-v2/cases/go/service/Dockerfile @@ -13,8 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG SW_AGENT_GO_COMMIT= -FROM ghcr.io/apache/skywalking-go/skywalking-go:${SW_AGENT_GO_COMMIT}-go1.19 AS base +ARG SW_AGENT_GO_COMMIT=aa948377ecdb4724fad1cc365c13a1188021316f +ARG GO_VERSION=go1.19 +FROM ghcr.io/apache/skywalking-go/skywalking-go:${SW_AGENT_GO_COMMIT}-${GO_VERSION} AS base ENV CGO_ENABLED=0 ENV GO111MODULE=on diff --git a/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml index 801c97b05caf..c2570cf22705 100644 --- a/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml @@ -39,6 +39,7 @@ services: SW_AGENT_REPORTER_GRPC_BACKEND_SERVICE: oap:11800 UPSTREAM_URL: http://localhost:8080/ignored.html privileged: true + stop_grace_period: 30s depends_on: oap: condition: service_healthy diff --git a/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml index 4003e947060a..0fcde6818c4d 100644 --- a/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml @@ -47,6 +47,7 @@ services: SW_AGENT_REPORTER_GRPC_BACKEND_SERVICE: oap:11800 UPSTREAM_URL: http://localhost:8080/ignored.html privileged: true + stop_grace_period: 30s depends_on: oap: condition: service_healthy diff --git a/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml index 757b06261657..db3d3ebc9746 100644 --- a/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml @@ -47,6 +47,7 @@ services: SW_AGENT_REPORTER_GRPC_BACKEND_SERVICE: oap:11800 UPSTREAM_URL: http://localhost:8080/ignored.html privileged: true + stop_grace_period: 30s depends_on: oap: condition: service_healthy From 07fe14cda3ed22fa9c962a1e793f5f06f2afc1ca Mon Sep 17 00:00:00 2001 From: JophieQu Date: Wed, 15 Oct 2025 14:24:18 +0800 Subject: [PATCH 24/69] fix port --- test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml | 2 +- test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml | 2 +- test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml index c2570cf22705..f8964fe7a8d3 100644 --- a/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml @@ -49,7 +49,7 @@ services: timeout: 60s retries: 120 ports: - - 9090 + - 8080 oap: extends: diff --git a/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml index 0fcde6818c4d..c7d2e9519de4 100644 --- a/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml @@ -57,7 +57,7 @@ services: timeout: 60s retries: 120 ports: - - 9090 + - 8080 oap: extends: diff --git a/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml index db3d3ebc9746..828163d3eeda 100644 --- a/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml @@ -57,7 +57,7 @@ services: timeout: 60s retries: 120 ports: - - 9090 + - 8080 oap: extends: From 2e628762ef888d2c1ef2f14280ca21b688f0cc0d Mon Sep 17 00:00:00 2001 From: JophieQu Date: Wed, 15 Oct 2025 17:27:04 +0800 Subject: [PATCH 25/69] fix e2e --- .../cases/profiling/pprof/expected/analysis.yml | 4 ++-- test/e2e-v2/cases/profiling/pprof/expected/list.yml | 4 ++-- .../cases/profiling/pprof/expected/progress.yml | 9 +-------- .../profiling/pprof/expected/service-instance.yml | 12 +++--------- 4 files changed, 8 insertions(+), 21 deletions(-) diff --git a/test/e2e-v2/cases/profiling/pprof/expected/analysis.yml b/test/e2e-v2/cases/profiling/pprof/expected/analysis.yml index 7f04a1d43f79..31d6b8056290 100644 --- a/test/e2e-v2/cases/profiling/pprof/expected/analysis.yml +++ b/test/e2e-v2/cases/profiling/pprof/expected/analysis.yml @@ -19,6 +19,6 @@ tree: - id: {{ notEmpty .id }} parentid: {{ notEmpty .parentid }} codesignature: {{ notEmpty .codesignature }} - total: {{ ge .total -1 }} - self: {{ ge .self -1 }} + total: {{ gt .total -1 }} + self: {{ gt .self -1 }} {{- end }} \ No newline at end of file diff --git a/test/e2e-v2/cases/profiling/pprof/expected/list.yml b/test/e2e-v2/cases/profiling/pprof/expected/list.yml index 963f50e9ea96..823d93eebdbe 100644 --- a/test/e2e-v2/cases/profiling/pprof/expected/list.yml +++ b/test/e2e-v2/cases/profiling/pprof/expected/list.yml @@ -21,7 +21,7 @@ tasks: serviceinstanceids: - {{ b64enc "go-service" }}.1_{{ b64enc "provider1" }} createtime: {{ gt .createtime 0 }} - events: - - CPU + dumpperiod: {{ ge .dumpperiod 0 }} + events: HEAP duration: {{ ge .duration 0 }} {{- end }} \ No newline at end of file diff --git a/test/e2e-v2/cases/profiling/pprof/expected/progress.yml b/test/e2e-v2/cases/profiling/pprof/expected/progress.yml index 9c285f08c530..790d0624ea34 100644 --- a/test/e2e-v2/cases/profiling/pprof/expected/progress.yml +++ b/test/e2e-v2/cases/profiling/pprof/expected/progress.yml @@ -13,14 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -logs: - {{- contains .logs }} -- id: {{ notEmpty .id}} - instanceid: {{ b64enc "go-service" }}.1_{{ b64enc "provider1" }} - instancename: {{ notEmpty .instancename}} - operationtype: {{ notEmpty .operationtype}} - operationtime: {{ ge .operationtime 0 }} - {{- end }} +logs: [] errorinstanceids: [] successinstanceids: - {{ b64enc "go-service" }}.1_{{ b64enc "provider1" }} diff --git a/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml b/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml index 83f07be75715..630d58ce03c4 100644 --- a/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml +++ b/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml @@ -16,19 +16,13 @@ # under the License. {{- contains . }} -- id: {{ b64enc "go-service" }}.1_{{ b64enc "provider1" }} - name: provider1 +- id: {{ notEmpty .id }} + name: {{ notEmpty .name }} attributes: {{- contains .attributes }} - - name: OS Name - value: Linux - - name: hostname - value: {{ notEmpty .value }} - - name: Process No. - value: {{ notEmpty .value }} - name: ipv4s value: {{ notEmpty .value }} {{- end}} language: GO - instanceuuid: {{ b64enc "go-service" }}.1_{{ b64enc "provider1" }} + instanceuuid: {{ notEmpty .instanceuuid }} {{- end}} From 02a5b55ced35d7287a8ac37cd3d515ed1eb5ba1f Mon Sep 17 00:00:00 2001 From: JophieQu Date: Wed, 15 Oct 2025 18:01:39 +0800 Subject: [PATCH 26/69] fix e2e --- test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml b/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml index dd0da0216728..8256a642241a 100644 --- a/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml +++ b/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml @@ -26,17 +26,17 @@ cases: - query: | swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ profiling pprof create --service-name=go-service \ - --duration=1 --events=CPU \ + --dump-period=1 --events=HEAP \ --instance-name-list=provider1 expected: expected/create.yml # list task - query: | swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ - profiling pprof list --service-name=go-service \ + profiling pprof list --service-name=go-service expected: expected/list.yml - # get task progress + # get task progress finished - query: | - swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ + sleep 15 && swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ profiling pprof progress --task-id=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling pprof list --service-name=go-service | yq e '.tasks[0].id') expected: expected/progress.yml # get task analysis From ed0403a43dcf0698776803e6b948a213be3e03c9 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Wed, 15 Oct 2025 18:40:43 +0800 Subject: [PATCH 27/69] fix e2e --- test/e2e-v2/cases/profiling/pprof/expected/progress.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/e2e-v2/cases/profiling/pprof/expected/progress.yml b/test/e2e-v2/cases/profiling/pprof/expected/progress.yml index 790d0624ea34..9c285f08c530 100644 --- a/test/e2e-v2/cases/profiling/pprof/expected/progress.yml +++ b/test/e2e-v2/cases/profiling/pprof/expected/progress.yml @@ -13,7 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -logs: [] +logs: + {{- contains .logs }} +- id: {{ notEmpty .id}} + instanceid: {{ b64enc "go-service" }}.1_{{ b64enc "provider1" }} + instancename: {{ notEmpty .instancename}} + operationtype: {{ notEmpty .operationtype}} + operationtime: {{ ge .operationtime 0 }} + {{- end }} errorinstanceids: [] successinstanceids: - {{ b64enc "go-service" }}.1_{{ b64enc "provider1" }} From fc0ae02c631728e8aa4c80068e654a5e59a87999 Mon Sep 17 00:00:00 2001 From: Wan Kai Date: Fri, 8 Aug 2025 18:00:19 +0800 Subject: [PATCH 28/69] Add self obs metrics for L1/L2 queue and persistent cache. (#13405) * 1.[Break Change] MQE function `sort_values` sorts according to the aggregation result and labels rather than the simple time series values. 2. Fix `MetricsPersistentWorker`, remove DataCarrier queue from `Hour/Day` dimensions metrics persistent process. 3. Self Observability: add `metrics_aggregation_queue_used_percentage` and `metrics_persistent_collection_cached_size` metrics for the OAP server. --- apm-protocol/apm-network/src/main/proto | 2 +- docs/en/api/metrics-query-expression.md | 37 ++++- docs/en/changes/changes.md | 5 + .../skywalking/mqe/rt/grammar/MQEParser.g4 | 2 +- .../skywalking/mqe/rt/MQEVisitorBase.java | 8 +- .../mqe/rt/operation/SortValuesOp.java | 106 ++++++++++--- .../skywalking/mqe/rt/SortValuesOpTest.java | 70 ++++++--- .../worker/MetricsAggregateWorker.java | 20 ++- .../worker/MetricsPersistentMinWorker.java | 142 ++++++++++++++++++ .../worker/MetricsPersistentWorker.java | 98 ++++-------- .../worker/MetricsStreamProcessor.java | 2 +- .../oap/server/core/query/mqe/Metadata.java | 2 + .../library/datacarrier/DataCarrier.java | 2 + .../library/datacarrier/buffer/Channels.java | 2 + .../src/main/resources/query-protocol | 2 +- .../src/main/resources/otel-rules/oap.yaml | 4 + .../so11y_oap/so11y-instance.json | 134 ++++++++++++++++- test/e2e-v2/cases/mqe/mqe-cases.yaml | 4 +- 18 files changed, 510 insertions(+), 132 deletions(-) create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentMinWorker.java diff --git a/apm-protocol/apm-network/src/main/proto b/apm-protocol/apm-network/src/main/proto index d5882f167677..fb3fb005650e 160000 --- a/apm-protocol/apm-network/src/main/proto +++ b/apm-protocol/apm-network/src/main/proto @@ -1 +1 @@ -Subproject commit d5882f167677626479d7e1df3933f1024eecd3bc +Subproject commit fb3fb005650e2489164978b7804117c7ade1529a diff --git a/docs/en/api/metrics-query-expression.md b/docs/en/api/metrics-query-expression.md index e96e0f2c1477..46de8e51249a 100644 --- a/docs/en/api/metrics-query-expression.md +++ b/docs/en/api/metrics-query-expression.md @@ -440,19 +440,42 @@ TIME_SERIES_VALUES. ## Sort Operation ### SortValues Operation -SortValues Operation takes an expression and sorts the values of the input expression result. - +SortValues Operation takes an expression used to sort and pick the top N label value groups, which according to +the values of a given ExpressionResult and based on the specified order, limit and aggregation type. +If the input expression is not a labeled result, it will retrurn the original expression result. Expression: ```text -sort_values(Expression, , ) +sort_values(Expression, , , ) ``` -- `limit` is the number of the sort results, should be a positive integer, if not specified, will return all results. Optional. +- `limit` is the number of the sort results, should be a positive integer. - `order` is the order of the sort results. The value of `order` can be `asc` or `des`. +- `aggregation_type` is the type of the aggregation operation. The type can be `avg`, `sum`, `max`, `min`. -For example: -If we want to sort the `service_resp_time` metric values in descending order and get the top 10 values, we can use the following expression: +For example, the following metrics in time series T1 and T2: +```text +T1: +http_requests_total{service="api"} 160 +http_requests_total{service="web"} 120 +http_requests_total{service="auth"} 80 + +T2: +http_requests_total{service="api"} 100 +http_requests_total{service="web"} 180 +http_requests_total{service="auth"} 10 +``` +We can use SortValuesOp to pick the top 2 services with the most avg requests in descending order: +```text +sort_values(http_requests_total, 2, desc, avg) +``` +The result will be: ```text -sort_values(service_resp_time, 10, des) +T1: +http_requests_total{service="web"} 120 +http_requests_total{service="api"} 160 + +T2: +http_requests_total{service="web"} 180 +http_requests_total{service="api"} 100 ``` #### Result Type diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 603b34bec16d..8a168f98a19c 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -4,6 +4,8 @@ * Bump up BanyanDB dependency version(server and java-client) to 0.9.0. * Fix CVE-2025-54057, restrict and validate url for widgets. +* Fix `MetricsPersistentWorker`, remove DataCarrier queue from `Hour/Day` dimensions metrics persistent process. + This is important to reduce memory cost and `Hour/Day` dimensions metrics persistent latency. #### OAP Server @@ -46,6 +48,8 @@ * Add UI dashboard for Ruby runtime metrics. * Tracing Query Execution HTTP APIs: make the argument `service layer` optional. * GraphQL API: metadata, topology, log and trace support query by name. +* [Break Change] MQE function `sort_values` sorts according to the aggregation result and labels rather than the simple time series values. +* Self Observability: add `metrics_aggregation_queue_used_percentage` and `metrics_persistent_collection_cached_size` metrics for the OAP server. * Support pprof profiling feature #### UI @@ -64,6 +68,7 @@ * Fix the snapshot charts unable to display. * Bump vue-i18n from 9.14.3 to 9.14.5. * Fix split queries for topology to avoid page crash. +* Self Observability ui-template: Add new panels for monitor `metrics aggregation queue used percentage` and `metrics persistent collection cached size`. #### Documentation diff --git a/oap-server/mqe-grammar/src/main/antlr4/org/apache/skywalking/mqe/rt/grammar/MQEParser.g4 b/oap-server/mqe-grammar/src/main/antlr4/org/apache/skywalking/mqe/rt/grammar/MQEParser.g4 index ccb0a6c5c840..5131cbfd19d3 100644 --- a/oap-server/mqe-grammar/src/main/antlr4/org/apache/skywalking/mqe/rt/grammar/MQEParser.g4 +++ b/oap-server/mqe-grammar/src/main/antlr4/org/apache/skywalking/mqe/rt/grammar/MQEParser.g4 @@ -38,7 +38,7 @@ expression | topNOf L_PAREN topN (COMMA topN)* COMMA INTEGER COMMA order R_PAREN #topNOfOP | relabels L_PAREN expression COMMA label COMMA replaceLabel R_PAREN #relablesOP | aggregateLabels L_PAREN expression COMMA aggregateLabelsFunc R_PAREN #aggregateLabelsOp - | sort_values L_PAREN expression (COMMA INTEGER)? COMMA order R_PAREN #sortValuesOP + | sort_values L_PAREN expression COMMA INTEGER COMMA order COMMA aggregation R_PAREN #sortValuesOP | sort_label_values L_PAREN expression COMMA order COMMA labelNameList R_PAREN #sortLabelValuesOP | baseline L_PAREN metric COMMA baseline_type R_PAREN #baselineOP ; diff --git a/oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/MQEVisitorBase.java b/oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/MQEVisitorBase.java index a1691e9dec01..68856a6b372d 100644 --- a/oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/MQEVisitorBase.java +++ b/oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/MQEVisitorBase.java @@ -23,7 +23,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.function.BiFunction; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -437,12 +436,9 @@ public ExpressionResult visitSortValuesOP(MQEParser.SortValuesOPContext ctx) { try { ExpressionResult result = visit(ctx.expression()); int order = ctx.order().getStart().getType(); - Optional limit = Optional.empty(); - if (ctx.INTEGER() != null) { - limit = Optional.of(Integer.valueOf(ctx.INTEGER().getText())); - } + int limit = Integer.parseInt(ctx.INTEGER().getText()); try { - return SortValuesOp.doSortValuesOp(result, limit, order); + return SortValuesOp.doSortValuesOp(result, limit, order, MQEParser.AVG); } catch (IllegalExpressionException e) { return getErrorResult(e.getMessage()); } diff --git a/oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/operation/SortValuesOp.java b/oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/operation/SortValuesOp.java index e6dc6baecf11..14d9823e3bc4 100644 --- a/oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/operation/SortValuesOp.java +++ b/oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/operation/SortValuesOp.java @@ -18,40 +18,98 @@ package org.apache.skywalking.mqe.rt.operation; +import java.util.ArrayList; import java.util.Comparator; import java.util.List; -import java.util.Optional; +import java.util.Map; import java.util.stream.Collectors; import org.apache.skywalking.mqe.rt.exception.IllegalExpressionException; import org.apache.skywalking.mqe.rt.grammar.MQEParser; import org.apache.skywalking.oap.server.core.query.mqe.ExpressionResult; -import org.apache.skywalking.oap.server.core.query.mqe.MQEValue; +import org.apache.skywalking.oap.server.core.query.mqe.ExpressionResultType; +import org.apache.skywalking.oap.server.core.query.mqe.MQEValues; +import org.apache.skywalking.oap.server.core.query.mqe.Metadata; +/** + * When a result has multiple label value groups, it is often required to sort the values of these groups. + * SortValuesOp is used to sort and pick the top N label value groups, which according to + * the values of a given ExpressionResult and based on the specified order, limit and aggregation type. + * + * It first performs aggregation on the results, then sorts them according to the specified order (ascending or descending), + * and finally limits the number of results if a limit is provided. + * + * for example, the following metrics in time series T1 and T2: + * T1: + * http_requests_total{service="api"} 160 + * http_requests_total{service="web"} 120 + * http_requests_total{service="auth"} 80 + * + * T2: + * http_requests_total{service="api"} 100 + * http_requests_total{service="web"} 180 + * http_requests_total{service="auth"} 10 + * + * We can use SortValuesOp to pick the top 2 services with the most avg requests in descending order: + * `sort_values(http_requests_total, 2, desc, avg)` + * The result will be: + * T1: + * http_requests_total{service="web"} 120 + * http_requests_total{service="api"} 160 + * + * T2: + * http_requests_total{service="web"} 180 + * http_requests_total{service="api"} 100 + */ public class SortValuesOp { public static ExpressionResult doSortValuesOp(ExpressionResult expResult, - Optional limit, - int order) throws IllegalExpressionException { - if (MQEParser.ASC == order || MQEParser.DES == order) { - expResult.getResults().forEach(mqeValues -> { - List values = mqeValues.getValues() - .stream() - // Filter out empty values - .filter(mqeValue -> !mqeValue.isEmptyValue()) - .sorted(MQEParser.ASC == order ? Comparator.comparingDouble( - MQEValue::getDoubleValue) : - Comparator.comparingDouble(MQEValue::getDoubleValue) - .reversed()) - .collect( - Collectors.toList()); - if (limit.isPresent() && limit.get() < values.size()) { - mqeValues.setValues(values.subList(0, limit.get())); - } else { - mqeValues.setValues(values); - } - }); - } else { - throw new IllegalExpressionException("Unsupported sort order."); + int limit, + int order, + int aggregationType) throws IllegalExpressionException { + // no label result, no need to sort + if (!expResult.isLabeledResult()) { + return expResult; } + // store the original results in a map to avoid losing data during aggregation + Map resultMap = expResult.getResults() + .stream() + .collect(Collectors.toMap( + MQEValues::getMetric, v -> { + MQEValues newValues = new MQEValues(); + newValues.setMetric(v.getMetric()); + newValues.setValues(v.getValues()); + return newValues; + } + )); + // do aggregation first + ExpressionResult aggResult = AggregationOp.doAggregationOp(expResult, aggregationType); + + List sorted = + aggResult.getResults().stream() + .sorted(getComparator(order)) + .collect(Collectors.toList()); + if (limit < sorted.size()) { + sorted = sorted.subList(0, limit); + } + List results = new ArrayList<>(); + sorted.forEach(v -> { + MQEValues mqeValues = resultMap.get(v.getMetric()); + if (mqeValues != null) { + results.add(mqeValues); + } + } + ); + + expResult.setResults(results); + expResult.setType(ExpressionResultType.TIME_SERIES_VALUES); return expResult; } + + private static Comparator getComparator(int order) { + Comparator comparator = Comparator.comparingDouble( + mqeValues -> mqeValues.getValues().isEmpty() + ? Double.NaN + : mqeValues.getValues().get(0).getDoubleValue() + ); + return order == MQEParser.ASC ? comparator : comparator.reversed(); + } } diff --git a/oap-server/mqe-rt/src/test/java/org/apache/skywalking/mqe/rt/SortValuesOpTest.java b/oap-server/mqe-rt/src/test/java/org/apache/skywalking/mqe/rt/SortValuesOpTest.java index 0360f09fbe66..3f26155a97c0 100644 --- a/oap-server/mqe-rt/src/test/java/org/apache/skywalking/mqe/rt/SortValuesOpTest.java +++ b/oap-server/mqe-rt/src/test/java/org/apache/skywalking/mqe/rt/SortValuesOpTest.java @@ -18,7 +18,6 @@ package org.apache.skywalking.mqe.rt; -import java.util.Optional; import org.apache.skywalking.mqe.rt.exception.IllegalExpressionException; import org.apache.skywalking.mqe.rt.grammar.MQEParser; import org.apache.skywalking.mqe.rt.operation.SortValuesOp; @@ -32,35 +31,64 @@ public class SortValuesOpTest { @Test public void sortValueTest() throws IllegalExpressionException { //no label - ExpressionResult des = SortValuesOp.doSortValuesOp(mockData.newSeriesNoLabeledResult(), Optional.of(3), - MQEParser.DES); - Assertions.assertEquals(300, des.getResults().get(0).getValues().get(0).getDoubleValue()); - Assertions.assertEquals(100, des.getResults().get(0).getValues().get(1).getDoubleValue()); - ExpressionResult asc = SortValuesOp.doSortValuesOp(mockData.newSeriesNoLabeledResult(), Optional.of(3), - MQEParser.ASC); + ExpressionResult des = SortValuesOp.doSortValuesOp( + mockData.newSeriesNoLabeledResult(), 3, + MQEParser.DES, MQEParser.AVG + ); + Assertions.assertEquals(100, des.getResults().get(0).getValues().get(0).getDoubleValue()); + Assertions.assertEquals(300, des.getResults().get(0).getValues().get(1).getDoubleValue()); + ExpressionResult asc = SortValuesOp.doSortValuesOp( + mockData.newSeriesNoLabeledResult(), 3, + MQEParser.ASC, MQEParser.AVG + ); Assertions.assertEquals(100, asc.getResults().get(0).getValues().get(0).getDoubleValue()); Assertions.assertEquals(300, asc.getResults().get(0).getValues().get(1).getDoubleValue()); //labeled - ExpressionResult desLabeled = SortValuesOp.doSortValuesOp(mockData.newSeriesLabeledResult(), Optional.of(3), - MQEParser.DES); - Assertions.assertEquals(300, desLabeled.getResults().get(0).getValues().get(0).getDoubleValue()); - Assertions.assertEquals(100, desLabeled.getResults().get(0).getValues().get(1).getDoubleValue()); - Assertions.assertEquals(301, desLabeled.getResults().get(1).getValues().get(0).getDoubleValue()); - Assertions.assertEquals(101, desLabeled.getResults().get(1).getValues().get(1).getDoubleValue()); - ExpressionResult ascLabeled = SortValuesOp.doSortValuesOp(mockData.newSeriesLabeledResult(), Optional.of(2), - MQEParser.ASC); + ExpressionResult desLabeled = SortValuesOp.doSortValuesOp( + mockData.newSeriesLabeledResult(), 3, + MQEParser.DES, MQEParser.AVG + ); + Assertions.assertEquals(101, desLabeled.getResults().get(0).getValues().get(0).getDoubleValue()); + Assertions.assertEquals(301, desLabeled.getResults().get(0).getValues().get(1).getDoubleValue()); + Assertions.assertEquals("label", desLabeled.getResults().get(0).getMetric().getLabels().get(0).getKey()); + Assertions.assertEquals("2", desLabeled.getResults().get(0).getMetric().getLabels().get(0).getValue()); + Assertions.assertEquals("label2", desLabeled.getResults().get(0).getMetric().getLabels().get(1).getKey()); + Assertions.assertEquals("21", desLabeled.getResults().get(0).getMetric().getLabels().get(1).getValue()); + Assertions.assertEquals(100, desLabeled.getResults().get(1).getValues().get(0).getDoubleValue()); + Assertions.assertEquals(300, desLabeled.getResults().get(1).getValues().get(1).getDoubleValue()); + Assertions.assertEquals("label", desLabeled.getResults().get(1).getMetric().getLabels().get(0).getKey()); + Assertions.assertEquals("1", desLabeled.getResults().get(1).getMetric().getLabels().get(0).getValue()); + Assertions.assertEquals("label2", desLabeled.getResults().get(1).getMetric().getLabels().get(1).getKey()); + Assertions.assertEquals("21", desLabeled.getResults().get(1).getMetric().getLabels().get(1).getValue()); + + ExpressionResult ascLabeled = SortValuesOp.doSortValuesOp( + mockData.newSeriesLabeledResult(), 3, + MQEParser.ASC, MQEParser.AVG + ); Assertions.assertEquals(100, ascLabeled.getResults().get(0).getValues().get(0).getDoubleValue()); Assertions.assertEquals(300, ascLabeled.getResults().get(0).getValues().get(1).getDoubleValue()); + Assertions.assertEquals("label", ascLabeled.getResults().get(0).getMetric().getLabels().get(0).getKey()); + Assertions.assertEquals("1", ascLabeled.getResults().get(0).getMetric().getLabels().get(0).getValue()); + Assertions.assertEquals("label2", ascLabeled.getResults().get(0).getMetric().getLabels().get(1).getKey()); + Assertions.assertEquals("21", ascLabeled.getResults().get(0).getMetric().getLabels().get(1).getValue()); Assertions.assertEquals(101, ascLabeled.getResults().get(1).getValues().get(0).getDoubleValue()); Assertions.assertEquals(301, ascLabeled.getResults().get(1).getValues().get(1).getDoubleValue()); + Assertions.assertEquals("label", ascLabeled.getResults().get(1).getMetric().getLabels().get(0).getKey()); + Assertions.assertEquals("2", ascLabeled.getResults().get(1).getMetric().getLabels().get(0).getValue()); + Assertions.assertEquals("label2", ascLabeled.getResults().get(1).getMetric().getLabels().get(1).getKey()); + Assertions.assertEquals("21", ascLabeled.getResults().get(1).getMetric().getLabels().get(1).getValue()); //limit - ExpressionResult desLabeledLimit = SortValuesOp.doSortValuesOp(mockData.newSeriesLabeledResult(), Optional.of(1), - MQEParser.DES); - Assertions.assertEquals(1, desLabeledLimit.getResults().get(0).getValues().size()); - Assertions.assertEquals(1, desLabeledLimit.getResults().get(1).getValues().size()); - Assertions.assertEquals(300, desLabeledLimit.getResults().get(0).getValues().get(0).getDoubleValue()); - Assertions.assertEquals(301, desLabeledLimit.getResults().get(1).getValues().get(0).getDoubleValue()); + ExpressionResult desLabeledLimit = SortValuesOp.doSortValuesOp( + mockData.newSeriesLabeledResult(), 1, + MQEParser.DES, MQEParser.AVG + ); + Assertions.assertEquals(101, desLabeledLimit.getResults().get(0).getValues().get(0).getDoubleValue()); + Assertions.assertEquals(301, desLabeledLimit.getResults().get(0).getValues().get(1).getDoubleValue()); + Assertions.assertEquals("label", desLabeledLimit.getResults().get(0).getMetric().getLabels().get(0).getKey()); + Assertions.assertEquals("2", desLabeledLimit.getResults().get(0).getMetric().getLabels().get(0).getValue()); + Assertions.assertEquals("label2", desLabeledLimit.getResults().get(0).getMetric().getLabels().get(1).getKey()); + Assertions.assertEquals("21", desLabeledLimit.getResults().get(0).getMetric().getLabels().get(1).getValue()); } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsAggregateWorker.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsAggregateWorker.java index d9a64d94dd2d..46decdf43fc1 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsAggregateWorker.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsAggregateWorker.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oap.server.core.analysis.worker; +import java.util.Arrays; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.oap.server.core.UnexpectedException; @@ -26,12 +27,14 @@ import org.apache.skywalking.oap.server.core.worker.AbstractWorker; import org.apache.skywalking.oap.server.library.datacarrier.DataCarrier; import org.apache.skywalking.oap.server.library.datacarrier.buffer.BufferStrategy; +import org.apache.skywalking.oap.server.library.datacarrier.buffer.QueueBuffer; import org.apache.skywalking.oap.server.library.datacarrier.consumer.BulkConsumePool; import org.apache.skywalking.oap.server.library.datacarrier.consumer.ConsumerPoolFactory; import org.apache.skywalking.oap.server.library.datacarrier.consumer.IConsumer; import org.apache.skywalking.oap.server.library.module.ModuleDefineHolder; import org.apache.skywalking.oap.server.telemetry.TelemetryModule; import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics; +import org.apache.skywalking.oap.server.telemetry.api.GaugeMetrics; import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator; import org.apache.skywalking.oap.server.telemetry.api.MetricsTag; @@ -49,7 +52,10 @@ public class MetricsAggregateWorker extends AbstractWorker { private final MergableBufferedData mergeDataCache; private CounterMetrics abandonCounter; private CounterMetrics aggregationCounter; + private GaugeMetrics queuePercentageGauge; private long lastSendTime = 0; + private final MetricStreamKind kind; + private final int queueTotalSize; MetricsAggregateWorker(ModuleDefineHolder moduleDefineHolder, AbstractWorker nextWorker, @@ -59,6 +65,7 @@ public class MetricsAggregateWorker extends AbstractWorker { super(moduleDefineHolder); this.nextWorker = nextWorker; this.mergeDataCache = new MergableBufferedData(); + this.kind = kind; String name = "METRICS_L1_AGGREGATION"; int queueChannelSize = 2; int queueBufferSize = 10_000; @@ -85,16 +92,24 @@ public class MetricsAggregateWorker extends AbstractWorker { .provider() .getService(MetricsCreator.class); abandonCounter = metricsCreator.createCounter( - "metrics_aggregator_abandon", "The abandon number of rows received in aggregation", + "metrics_aggregator_abandon", "The abandon number of rows received in aggregation.", new MetricsTag.Keys("metricName", "level", "dimensionality"), new MetricsTag.Values(modelName, "1", "minute") ); aggregationCounter = metricsCreator.createCounter( - "metrics_aggregation", "The number of rows in aggregation", + "metrics_aggregation", "The number of rows in aggregation.", new MetricsTag.Keys("metricName", "level", "dimensionality"), new MetricsTag.Values(modelName, "1", "minute") ); + queuePercentageGauge = metricsCreator.createGauge( + "metrics_aggregation_queue_used_percentage", "The percentage of queue used in aggregation.", + new MetricsTag.Keys("metricName", "level", "kind"), + new MetricsTag.Values(modelName, "1", kind.name()) + ); this.l1FlushPeriod = l1FlushPeriod; + queueTotalSize = Arrays.stream(dataCarrier.getChannels().getBufferChannels()) + .mapToInt(QueueBuffer::getBufferSize) + .sum(); } /** @@ -137,6 +152,7 @@ private void flush() { private class AggregatorConsumer implements IConsumer { @Override public void consume(List data) { + queuePercentageGauge.setValue(Math.round(100 * (double) data.size() / queueTotalSize)); MetricsAggregateWorker.this.onWork(data); } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentMinWorker.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentMinWorker.java new file mode 100644 index 000000000000..b44988e23fa2 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentMinWorker.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.analysis.worker; + +import java.util.Arrays; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.UnexpectedException; +import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; +import org.apache.skywalking.oap.server.core.exporter.ExportEvent; +import org.apache.skywalking.oap.server.core.status.ServerStatusService; +import org.apache.skywalking.oap.server.core.status.ServerStatusWatcher; +import org.apache.skywalking.oap.server.core.storage.IMetricsDAO; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.worker.AbstractWorker; +import org.apache.skywalking.oap.server.library.datacarrier.DataCarrier; +import org.apache.skywalking.oap.server.library.datacarrier.buffer.QueueBuffer; +import org.apache.skywalking.oap.server.library.datacarrier.consumer.BulkConsumePool; +import org.apache.skywalking.oap.server.library.datacarrier.consumer.ConsumerPoolFactory; +import org.apache.skywalking.oap.server.library.datacarrier.consumer.IConsumer; +import org.apache.skywalking.oap.server.library.module.ModuleDefineHolder; +import org.apache.skywalking.oap.server.telemetry.TelemetryModule; +import org.apache.skywalking.oap.server.telemetry.api.GaugeMetrics; +import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator; +import org.apache.skywalking.oap.server.telemetry.api.MetricsTag; + +/** + * MetricsPersistentMinWorker is an extension of {@link MetricsPersistentWorker} and focuses on the Minute Metrics data persistent. + */ +@Slf4j +public class MetricsPersistentMinWorker extends MetricsPersistentWorker implements ServerStatusWatcher { + private final DataCarrier dataCarrier; + + /** + * The percentage of queue used in aggregation + */ + private final GaugeMetrics queuePercentageGauge; + + /** + * @since 9.4.0 + */ + private final ServerStatusService serverStatusService; + + // Not going to expose this as a configuration, only for testing purpose + private final boolean isTestingTTL = "true".equalsIgnoreCase(System.getenv("TESTING_TTL")); + private final int queueTotalSize; + + MetricsPersistentMinWorker(ModuleDefineHolder moduleDefineHolder, Model model, IMetricsDAO metricsDAO, + AbstractWorker nextAlarmWorker, AbstractWorker nextExportWorker, + MetricsTransWorker transWorker, boolean supportUpdate, + long storageSessionTimeout, int metricsDataTTL, MetricStreamKind kind) { + super( + moduleDefineHolder, model, metricsDAO, nextAlarmWorker, nextExportWorker, transWorker, supportUpdate, + storageSessionTimeout, metricsDataTTL, kind + ); + + String name = "METRICS_L2_AGGREGATION"; + int size = BulkConsumePool.Creator.recommendMaxSize() / 8; + if (size == 0) { + size = 1; + } + BulkConsumePool.Creator creator = new BulkConsumePool.Creator(name, size, 200); + try { + ConsumerPoolFactory.INSTANCE.createIfAbsent(name, creator); + } catch (Exception e) { + throw new UnexpectedException(e.getMessage(), e); + } + + int bufferSize = 2000; + if (MetricStreamKind.MAL == kind) { + // In MAL meter streaming, the load of data flow is much less as they are statistics already, + // but in OAL sources, they are raw data. + // Set the buffer(size of queue) as 1/2 to reduce unnecessary resource costs. + bufferSize = 1000; + } + this.dataCarrier = new DataCarrier<>("MetricsPersistentWorker." + model.getName(), name, 1, bufferSize); + this.dataCarrier.consume(ConsumerPoolFactory.INSTANCE.get(name), new PersistentConsumer()); + + MetricsCreator metricsCreator = moduleDefineHolder.find(TelemetryModule.NAME) + .provider() + .getService(MetricsCreator.class); + queuePercentageGauge = metricsCreator.createGauge( + "metrics_aggregation_queue_used_percentage", "The percentage of queue used in aggregation.", + new MetricsTag.Keys("metricName", "level", "kind"), + new MetricsTag.Values(model.getName(), "2", kind.name()) + ); + serverStatusService = moduleDefineHolder.find(CoreModule.NAME).provider().getService(ServerStatusService.class); + serverStatusService.registerWatcher(this); + queueTotalSize = Arrays.stream(dataCarrier.getChannels().getBufferChannels()) + .mapToInt(QueueBuffer::getBufferSize) + .sum(); + } + + /** + * Accept all metrics data and push them into the queue for serial processing + */ + @Override + public void in(Metrics metrics) { + final var isExpired = getMetricsDAO().isExpiredCache(getModel(), metrics, System.currentTimeMillis(), getMetricsDataTTL()); + if (isExpired && !isTestingTTL) { + log.debug("Receiving expired metrics: {}, time: {}, ignored", metrics.id(), metrics.getTimeBucket()); + return; + } + getAggregationCounter().inc(); + dataCarrier.produce(metrics); + } + + /** + * Metrics queue processor, merge the received metrics if existing one with same ID(s) and time bucket. + * + * ID is declared through {@link Object#hashCode()} and {@link Object#equals(Object)} as usual. + */ + private class PersistentConsumer implements IConsumer { + @Override + public void consume(List data) { + queuePercentageGauge.setValue(Math.round(100 * (double) data.size() / queueTotalSize)); + MetricsPersistentMinWorker.this.onWork(data); + } + + @Override + public void onError(List data, Throwable t) { + log.error(t.getMessage(), t); + } + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentWorker.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentWorker.java index 47f4a4f30941..07d0a6e40fb7 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentWorker.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentWorker.java @@ -23,10 +23,9 @@ import java.util.List; import java.util.Optional; import java.util.stream.Collectors; +import lombok.AccessLevel; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.apache.skywalking.oap.server.core.CoreModule; -import org.apache.skywalking.oap.server.core.UnexpectedException; -import org.apache.skywalking.oap.server.core.analysis.DownSampling; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.data.MergableBufferedData; import org.apache.skywalking.oap.server.core.analysis.data.ReadWriteSafeCache; @@ -34,20 +33,16 @@ import org.apache.skywalking.oap.server.core.exporter.ExportEvent; import org.apache.skywalking.oap.server.core.status.BootingStatus; import org.apache.skywalking.oap.server.core.status.ClusterStatus; -import org.apache.skywalking.oap.server.core.status.ServerStatusService; import org.apache.skywalking.oap.server.core.status.ServerStatusWatcher; import org.apache.skywalking.oap.server.core.storage.IMetricsDAO; import org.apache.skywalking.oap.server.core.storage.SessionCacheCallback; import org.apache.skywalking.oap.server.core.storage.model.Model; import org.apache.skywalking.oap.server.core.worker.AbstractWorker; import org.apache.skywalking.oap.server.library.client.request.PrepareRequest; -import org.apache.skywalking.oap.server.library.datacarrier.DataCarrier; -import org.apache.skywalking.oap.server.library.datacarrier.consumer.BulkConsumePool; -import org.apache.skywalking.oap.server.library.datacarrier.consumer.ConsumerPoolFactory; -import org.apache.skywalking.oap.server.library.datacarrier.consumer.IConsumer; import org.apache.skywalking.oap.server.library.module.ModuleDefineHolder; import org.apache.skywalking.oap.server.telemetry.TelemetryModule; import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics; +import org.apache.skywalking.oap.server.telemetry.api.GaugeMetrics; import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator; import org.apache.skywalking.oap.server.telemetry.api.MetricsTag; @@ -56,18 +51,20 @@ */ @Slf4j public class MetricsPersistentWorker extends PersistenceWorker implements ServerStatusWatcher { + @Getter(AccessLevel.PROTECTED) private final Model model; private final long storageSessionTimeout; private final MetricsSessionCache sessionCache; + @Getter(AccessLevel.PROTECTED) private final IMetricsDAO metricsDAO; private final Optional> nextAlarmWorker; private final Optional> nextExportWorker; - private final DataCarrier dataCarrier; private final Optional transWorker; private final boolean supportUpdate; /** * The counter of L2 aggregation. */ + @Getter(AccessLevel.PROTECTED) private CounterMetrics aggregationCounter; /** * The counter of metrics reading from Database. @@ -76,7 +73,12 @@ public class MetricsPersistentWorker extends PersistenceWorker implemen /** * The counter of metrics cached in-memory. */ - private CounterMetrics cachedMetricsCounter; + private final CounterMetrics cachedMetricsCounter; + + /** + * The metrics persistent collection cached size + */ + private final GaugeMetrics collectionCachedSizeGauge; /** * The counter for the round of persistent. */ @@ -90,11 +92,9 @@ public class MetricsPersistentWorker extends PersistenceWorker implemen /** * @since 8.7.0 TTL settings from {@link org.apache.skywalking.oap.server.core.CoreModuleConfig#getMetricsDataTTL()} */ + @Getter(AccessLevel.PROTECTED) private int metricsDataTTL; - /** - * @since 9.4.0 - */ - private final ServerStatusService serverStatusService; + /** * The time bucket is 0 or in minute dimensionality of the system in the latest stability status. * @@ -105,10 +105,10 @@ public class MetricsPersistentWorker extends PersistenceWorker implemen // Not going to expose this as a configuration, only for testing purpose private final boolean isTestingTTL = "true".equalsIgnoreCase(System.getenv("TESTING_TTL")); - MetricsPersistentWorker(ModuleDefineHolder moduleDefineHolder, Model model, IMetricsDAO metricsDAO, - AbstractWorker nextAlarmWorker, AbstractWorker nextExportWorker, - MetricsTransWorker transWorker, boolean supportUpdate, - long storageSessionTimeout, int metricsDataTTL, MetricStreamKind kind) { + protected MetricsPersistentWorker(ModuleDefineHolder moduleDefineHolder, Model model, IMetricsDAO metricsDAO, + AbstractWorker nextAlarmWorker, AbstractWorker nextExportWorker, + MetricsTransWorker transWorker, boolean supportUpdate, + long storageSessionTimeout, int metricsDataTTL, MetricStreamKind kind) { super(moduleDefineHolder, new ReadWriteSafeCache<>(new MergableBufferedData(), new MergableBufferedData())); this.model = model; this.storageSessionTimeout = storageSessionTimeout; @@ -122,33 +122,11 @@ public class MetricsPersistentWorker extends PersistenceWorker implemen this.persistentMod = 1; this.metricsDataTTL = metricsDataTTL; - String name = "METRICS_L2_AGGREGATION"; - int size = BulkConsumePool.Creator.recommendMaxSize() / 8; - if (size == 0) { - size = 1; - } - BulkConsumePool.Creator creator = new BulkConsumePool.Creator(name, size, 200); - try { - ConsumerPoolFactory.INSTANCE.createIfAbsent(name, creator); - } catch (Exception e) { - throw new UnexpectedException(e.getMessage(), e); - } - - int bufferSize = 2000; - if (MetricStreamKind.MAL == kind) { - // In MAL meter streaming, the load of data flow is much less as they are statistics already, - // but in OAL sources, they are raw data. - // Set the buffer(size of queue) as 1/2 to reduce unnecessary resource costs. - bufferSize = 1000; - } - this.dataCarrier = new DataCarrier<>("MetricsPersistentWorker." + model.getName(), name, 1, bufferSize); - this.dataCarrier.consume(ConsumerPoolFactory.INSTANCE.get(name), new PersistentConsumer()); - MetricsCreator metricsCreator = moduleDefineHolder.find(TelemetryModule.NAME) .provider() .getService(MetricsCreator.class); aggregationCounter = metricsCreator.createCounter( - "metrics_aggregation", "The number of rows in aggregation", + "metrics_aggregation", "The number of rows in aggregation.", new MetricsTag.Keys("metricName", "level", "dimensionality"), new MetricsTag.Values(model.getName(), "2", model.getDownsampling().getName()) ); @@ -160,10 +138,11 @@ public class MetricsPersistentWorker extends PersistenceWorker implemen "metrics_persistent_cache", "The counter of metrics status, new or cached.", new MetricsTag.Keys("status"), new MetricsTag.Values("cached") ); - serverStatusService = moduleDefineHolder.find(CoreModule.NAME).provider().getService(ServerStatusService.class); - if (model.getDownsampling().equals(DownSampling.Minute)) { - serverStatusService.registerWatcher(this); - } + collectionCachedSizeGauge = metricsCreator.createGauge( + "metrics_persistent_collection_cached_size", "The collection cache size for metrics persistent.", + new MetricsTag.Keys("metricName", "dimensionality", "kind"), + new MetricsTag.Values(model.getName(), model.getDownsampling().getName(), kind.name()) + ); } /** @@ -196,8 +175,13 @@ public void in(Metrics metrics) { log.debug("Receiving expired metrics: {}, time: {}, ignored", metrics.id(), metrics.getTimeBucket()); return; } + aggregationCounter.inc(); - dataCarrier.produce(metrics); + /* + Metrics queue processor, merge the received metrics if existing one with same ID(s) and time bucket. + ID is declared through {@link Object#hashCode()} and {@link Object#equals(Object)} as usual. + */ + super.onWork(List.of(metrics)); } @Override @@ -212,7 +196,8 @@ public List buildBatchRequests() { if (lastCollection.size() == 0) { return Collections.emptyList(); } - + // record the size > 0 cache to avoid too much metrics + collectionCachedSizeGauge.setValue(lastCollection.size()); /* * Hard coded the max size. This only affect the multiIDRead if the data doesn't hit the cache. */ @@ -395,7 +380,7 @@ private Metrics requireInitialization(Metrics metrics) { //The kernel should NOT try to load it from the database. // // Notice, about the condition (2), - // For the specific minutes of metrics before booted, rebalanced(cluster) and expired from cache, + // For the specific minutes of metrics before booted, rebalanced(cluster) and expired from cache, // they are expected to load from the database when don't exist in the cache. if (timeOfLatestStabilitySts > 0 && metrics.getTimeBucket() > timeOfLatestStabilitySts) { @@ -423,21 +408,4 @@ public void onClusterRebalanced(final ClusterStatus clusterStatus) { timeOfLatestStabilitySts = TimeBucket.getMinuteTimeBucket( clusterStatus.getRebalancedTime()); } - - /** - * Metrics queue processor, merge the received metrics if existing one with same ID(s) and time bucket. - * - * ID is declared through {@link Object#hashCode()} and {@link Object#equals(Object)} as usual. - */ - private class PersistentConsumer implements IConsumer { - @Override - public void consume(List data) { - MetricsPersistentWorker.this.onWork(data); - } - - @Override - public void onError(List data, Throwable t) { - log.error(t.getMessage(), t); - } - } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsStreamProcessor.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsStreamProcessor.java index 290b51122488..17e41d72f405 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsStreamProcessor.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsStreamProcessor.java @@ -209,7 +209,7 @@ private MetricsPersistentWorker minutePersistentWorker(ModuleDefineHolder module AlarmNotifyWorker alarmNotifyWorker = new AlarmNotifyWorker(moduleDefineHolder); ExportMetricsWorker exportWorker = new ExportMetricsWorker(moduleDefineHolder); - MetricsPersistentWorker minutePersistentWorker = new MetricsPersistentWorker( + MetricsPersistentWorker minutePersistentWorker = new MetricsPersistentMinWorker( moduleDefineHolder, model, metricsDAO, alarmNotifyWorker, exportWorker, transWorker, supportUpdate, storageSessionTimeout, metricsDataTTL, kind ); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/mqe/Metadata.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/mqe/Metadata.java index c1be33a0efc2..ea1b9f975a95 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/mqe/Metadata.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/mqe/Metadata.java @@ -23,9 +23,11 @@ import java.util.Comparator; import java.util.List; import lombok.Data; +import lombok.EqualsAndHashCode; import org.apache.skywalking.oap.server.core.query.type.KeyValue; @Data +@EqualsAndHashCode public class Metadata { private List labels = new ArrayList<>(); diff --git a/oap-server/server-library/library-datacarrier-queue/src/main/java/org/apache/skywalking/oap/server/library/datacarrier/DataCarrier.java b/oap-server/server-library/library-datacarrier-queue/src/main/java/org/apache/skywalking/oap/server/library/datacarrier/DataCarrier.java index 86dd497f9288..d755111ac088 100644 --- a/oap-server/server-library/library-datacarrier-queue/src/main/java/org/apache/skywalking/oap/server/library/datacarrier/DataCarrier.java +++ b/oap-server/server-library/library-datacarrier-queue/src/main/java/org/apache/skywalking/oap/server/library/datacarrier/DataCarrier.java @@ -19,6 +19,7 @@ package org.apache.skywalking.oap.server.library.datacarrier; import java.util.Properties; +import lombok.Getter; import org.apache.skywalking.oap.server.library.datacarrier.buffer.BufferStrategy; import org.apache.skywalking.oap.server.library.datacarrier.buffer.Channels; import org.apache.skywalking.oap.server.library.datacarrier.consumer.ConsumeDriver; @@ -32,6 +33,7 @@ * DataCarrier main class. use this instance to set Producer/Consumer Model. */ public class DataCarrier { + @Getter private Channels channels; private IDriver driver; private String name; diff --git a/oap-server/server-library/library-datacarrier-queue/src/main/java/org/apache/skywalking/oap/server/library/datacarrier/buffer/Channels.java b/oap-server/server-library/library-datacarrier-queue/src/main/java/org/apache/skywalking/oap/server/library/datacarrier/buffer/Channels.java index d834287cc6ba..2afa7137925f 100644 --- a/oap-server/server-library/library-datacarrier-queue/src/main/java/org/apache/skywalking/oap/server/library/datacarrier/buffer/Channels.java +++ b/oap-server/server-library/library-datacarrier-queue/src/main/java/org/apache/skywalking/oap/server/library/datacarrier/buffer/Channels.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oap.server.library.datacarrier.buffer; +import lombok.Getter; import org.apache.skywalking.oap.server.library.datacarrier.partition.IDataPartitioner; /** @@ -25,6 +26,7 @@ * buffer is full. The Default is BLOCKING

Created by wusheng on 2016/10/25. */ public class Channels { + @Getter private final QueueBuffer[] bufferChannels; private IDataPartitioner dataPartitioner; private final BufferStrategy strategy; diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol index ed68593ec32e..4fc10625ba72 160000 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol @@ -1 +1 @@ -Subproject commit ed68593ec32ecd2cb56326bede0cd7b9f85b7bec +Subproject commit 4fc10625ba72ef4788972b4f7991a535065d609b diff --git a/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml b/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml index 9ee370541ee5..e4fda76c5b8e 100644 --- a/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml +++ b/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml @@ -66,6 +66,8 @@ metricsRules: exp: > metrics_aggregation.tagEqual('dimensionality', 'minute').sum(['service', 'host_name', 'level']).increase('PT1M') .tag({tags -> if (tags['level'] == '1') {tags.level = 'L1 aggregation'} }).tag({tags -> if (tags['level'] == '2') {tags.level = 'L2 aggregation'} }) + - name: instance_metrics_aggregation_queue_used_percentage + exp: metrics_aggregation_queue_used_percentage.sum(['service', 'host_name', 'level', 'kind', 'metricName']) - name: instance_persistence_execute_percentile exp: persistence_timer_bulk_execute_latency.sum(['le', 'service', 'host_name']).increase('PT5M').histogram().histogram_percentile([50,70,90,99]) - name: instance_persistence_prepare_percentile @@ -78,6 +80,8 @@ metricsRules: exp: persistence_timer_bulk_prepare_latency_count.sum(['service', 'host_name']).increase('PT1M') - name: instance_metrics_persistent_cache exp: metrics_persistent_cache.sum(['service', 'host_name', 'status']).increase('PT1M') + - name: instance_metrics_persistent_collection_cached_size + exp: metrics_persistent_collection_cached_size.sum(['service', 'host_name', 'dimensionality', 'kind', 'metricName']) - name: jvm_thread_live_count exp: jvm_threads_current.sum(['service', 'host_name']) - name: jvm_thread_daemon_count diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json index 67f9502b7aa7..277a0b3ca63b 100644 --- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json +++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json @@ -461,6 +461,138 @@ "title": "Watermark Circuit Breaker Status", "tips": "The status of circuit breaker listeners, 0 means all recovered from the breaks" } + }, + { + "x": 12, + "y": 52, + "w": 12, + "h": 13, + "i": "22", + "type": "Widget", + "expressions": [ + "sort_values(meter_oap_instance_metrics_aggregation_queue_used_percentage{level='1',kind='OAL'},10,des,avg)" + ], + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": true, + "showXAxis": true, + "showYAxis": true + }, + "widget": { + "title": "OAL L1 Aggregation Queue Percentage (%)" + } + }, + { + "x": 0, + "y": 65, + "w": 12, + "h": 13, + "i": "23", + "type": "Widget", + "expressions": [ + "sort_values(meter_oap_instance_metrics_aggregation_queue_used_percentage{level='1',kind='MAL'},10,des,avg)" + ], + "widget": { + "title": "MAL L1 Aggregation Queue Percentage (%)" + }, + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": true, + "showXAxis": true, + "showYAxis": true + } + }, + { + "x": 12, + "y": 65, + "w": 12, + "h": 13, + "i": "24", + "type": "Widget", + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": true, + "showXAxis": true, + "showYAxis": true + }, + "expressions": [ + "sort_values(meter_oap_instance_metrics_aggregation_queue_used_percentage{level='2',kind='OAL'},10,des,avg)" + ], + "widget": { + "title": "OAL L2 Aggregation Queue Percentage (%)" + } + }, + { + "x": 0, + "y": 78, + "w": 12, + "h": 13, + "i": "25", + "type": "Widget", + "expressions": [ + "sort_values(meter_oap_instance_metrics_aggregation_queue_used_percentage{level='2',kind='OAL'},10,des,avg)" + ], + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": true, + "showXAxis": true, + "showYAxis": true + }, + "widget": { + "title": "MAL L2 Aggregation Queue Percentage (%)" + } + }, + { + "x": 12, + "y": 78, + "w": 12, + "h": 13, + "i": "26", + "type": "Widget", + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": true, + "showXAxis": true, + "showYAxis": true + }, + "expressions": [ + "sort_values(meter_oap_instance_metrics_persistent_collection_cached_size{dimensionality='minute',kind='OAL'},10,des,avg)" + ], + "widget": { + "title": "OAL Min Metrics Persistent Collection Cached Size" + } + }, + { + "x": 0, + "y": 91, + "w": 12, + "h": 13, + "i": "27", + "type": "Widget", + "expressions": [ + "sort_values(meter_oap_instance_metrics_persistent_collection_cached_size{dimensionality='minute',kind='MAL'},10,des,avg)" + ], + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": true, + "showXAxis": true, + "showYAxis": true + }, + "widget": { + "title": "MAL Min Metrics Persistent Collection Cached Size" + } } ] }, @@ -786,4 +918,4 @@ "isRoot": false } } -] +] \ No newline at end of file diff --git a/test/e2e-v2/cases/mqe/mqe-cases.yaml b/test/e2e-v2/cases/mqe/mqe-cases.yaml index 854b77254afa..6a63cc5de38d 100644 --- a/test/e2e-v2/cases/mqe/mqe-cases.yaml +++ b/test/e2e-v2/cases/mqe/mqe-cases.yaml @@ -100,9 +100,9 @@ cases: # sort-OP e2e used for test MQE expression, more tests can refer to the UT # sort-value-OP - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="sort_values(service_percentile,2,asc)" --service-name=e2e-service-provider + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="sort_values(service_percentile,2,asc,avg)" --service-name=e2e-service-provider expected: expected/sort-value-OP.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="sort_values(service_percentile,des)" --service-name=e2e-service-provider + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="sort_values(service_percentile,1,des,avg)" --service-name=e2e-service-provider expected: expected/sort-value-OP.yml # sort-label-value-OP From b3d09caa9b2e2d0f47392a5788d43158c2f622cf Mon Sep 17 00:00:00 2001 From: JophieQu Date: Wed, 15 Oct 2025 19:36:29 +0800 Subject: [PATCH 29/69] add design doc --- docs/en/concepts-and-designs/profiling.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/en/concepts-and-designs/profiling.md b/docs/en/concepts-and-designs/profiling.md index 5c1ae91e9603..603adb1041ff 100644 --- a/docs/en/concepts-and-designs/profiling.md +++ b/docs/en/concepts-and-designs/profiling.md @@ -47,6 +47,20 @@ Async Profiler can trace the following kinds of events: Only Java agent support this. +### Go App Profiling + +Go App Profiling uses the [Pprof](https://github.com/google/pprof) for sampling. + +pprof is a profiling tool by Google for visualizing and analyzing sampled performance data. +It reads samples in profile.proto format and generates text or graphical reports (via the dot visualization) to highlight performance hotspots. + +pprof supports profiling of: + +- CPU. +- Memory allocs / heap. +- Block / mutex. +- Gouroutine / threadcreate. + ## Out-of-process profiling Out-of-process profiling leverage [eBPF](https://ebpf.io/) technology with origins in the Linux kernel. From c6d7a795b5075cb8295ab7aa0570e52bb2aab2ef Mon Sep 17 00:00:00 2001 From: Wan Kai Date: Mon, 11 Aug 2025 11:54:30 +0800 Subject: [PATCH 30/69] fix so11y UI template MQE (#13411) --- .../ui-initialized-templates/so11y_oap/so11y-instance.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json index 277a0b3ca63b..852ac36f565d 100644 --- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json +++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json @@ -536,7 +536,7 @@ "i": "25", "type": "Widget", "expressions": [ - "sort_values(meter_oap_instance_metrics_aggregation_queue_used_percentage{level='2',kind='OAL'},10,des,avg)" + "sort_values(meter_oap_instance_metrics_aggregation_queue_used_percentage{level='2',kind='MAL'},10,des,avg)" ], "graph": { "type": "Line", From e8253f0f01b588a3bf0bca48c1c15e0c7f0ef11a Mon Sep 17 00:00:00 2001 From: JophieQu Date: Fri, 10 Oct 2025 22:31:12 +0800 Subject: [PATCH 31/69] test ci --- .github/workflows/codeql.yaml | 3 ++- .github/workflows/dead-link-checker.yaml | 3 +++ .github/workflows/publish-docker-e2e-service.yaml | 1 + .github/workflows/publish-docker.yaml | 1 + .github/workflows/skywalking.yaml | 3 +++ 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 59f762f808b6..038e92bcced6 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -18,7 +18,8 @@ name: "CodeQL" on: push: - branches: ["master"] + branches: + - ci pull_request: branches: ["master"] paths: diff --git a/.github/workflows/dead-link-checker.yaml b/.github/workflows/dead-link-checker.yaml index b134daf30021..b05d292ce732 100644 --- a/.github/workflows/dead-link-checker.yaml +++ b/.github/workflows/dead-link-checker.yaml @@ -17,6 +17,9 @@ name: Dead Link Checker on: + push: + branches: + - ci pull_request: paths: - 'docs/**' diff --git a/.github/workflows/publish-docker-e2e-service.yaml b/.github/workflows/publish-docker-e2e-service.yaml index 661053ec40ae..0b8420f67579 100644 --- a/.github/workflows/publish-docker-e2e-service.yaml +++ b/.github/workflows/publish-docker-e2e-service.yaml @@ -20,6 +20,7 @@ on: push: branches: - master + - ci paths: - 'test/e2e-v2/java-test-service/**' - 'test/Makefile' diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml index c5bf734e54ab..a3367b2465f2 100644 --- a/.github/workflows/publish-docker.yaml +++ b/.github/workflows/publish-docker.yaml @@ -20,6 +20,7 @@ on: push: branches: - master + - ci release: types: - released diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index 5b3b3a6c93c0..ebc085074dbb 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -17,6 +17,9 @@ name: CI on: + push: + branches: + - ci pull_request: schedule: - cron: "0 18 * * *" # TimeZone: UTC 0 From a43856970b4f7eebeccddfbf5acf58cbfc6cd291 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Fri, 10 Oct 2025 23:26:23 +0800 Subject: [PATCH 32/69] merge --- .../worker/MetricsAggregateMALWorker.java | 65 +++++++++++++++++++ .../worker/MetricsAggregateOALWorker.java | 48 ++++++++++++++ .../worker/MetricsAggregateWorker.java | 36 ++++------ .../worker/MetricsPersistentMinMALWorker.java | 62 ++++++++++++++++++ .../worker/MetricsPersistentMinOALWorker.java | 52 +++++++++++++++ .../worker/MetricsPersistentMinWorker.java | 28 +++----- .../worker/MetricsStreamProcessor.java | 37 +++++++++-- .../datacarrier/consumer/BulkConsumePool.java | 18 +++-- .../consumer/MultipleChannelsConsumer.java | 39 ++++++++--- .../consumer/ConsumerPoolFactoryTest.java | 2 +- 10 files changed, 324 insertions(+), 63 deletions(-) create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsAggregateMALWorker.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsAggregateOALWorker.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentMinMALWorker.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentMinOALWorker.java diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsAggregateMALWorker.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsAggregateMALWorker.java new file mode 100644 index 000000000000..6ce0bb447d3b --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsAggregateMALWorker.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.analysis.worker; + +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; +import org.apache.skywalking.oap.server.core.worker.AbstractWorker; +import org.apache.skywalking.oap.server.library.datacarrier.consumer.BulkConsumePool; +import org.apache.skywalking.oap.server.library.datacarrier.consumer.ConsumerPoolFactory; +import org.apache.skywalking.oap.server.library.module.ModuleDefineHolder; + +/** + * MetricsAggregateMALWorker provides an in-memory metrics merging capability for MAL + */ +@Slf4j +public class MetricsAggregateMALWorker extends MetricsAggregateWorker { + private final static String POOL_NAME = "METRICS_L1_AGGREGATION_MAL"; + private final BulkConsumePool pool; + + MetricsAggregateMALWorker(ModuleDefineHolder moduleDefineHolder, + AbstractWorker nextWorker, + String modelName, + long l1FlushPeriod, + MetricStreamKind kind) { + super( + moduleDefineHolder, nextWorker, modelName, l1FlushPeriod, kind, + POOL_NAME, + calculatePoolSize(), + true, + 1, + 1_000 + ); + this.pool = (BulkConsumePool) ConsumerPoolFactory.INSTANCE.get(POOL_NAME); + } + + /** + * MetricsAggregateWorker#in operation does include enqueue only + */ + @Override + public final void in(Metrics metrics) { + super.in(metrics); + pool.notifyConsumers(); + } + + private static int calculatePoolSize() { + int size = BulkConsumePool.Creator.recommendMaxSize() / 8; + return size == 0 ? 1 : size; + } +} \ No newline at end of file diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsAggregateOALWorker.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsAggregateOALWorker.java new file mode 100644 index 000000000000..833b24419a6f --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsAggregateOALWorker.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.analysis.worker; + +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; +import org.apache.skywalking.oap.server.core.worker.AbstractWorker; +import org.apache.skywalking.oap.server.library.datacarrier.consumer.BulkConsumePool; +import org.apache.skywalking.oap.server.library.module.ModuleDefineHolder; + +/** + * MetricsAggregateOALWorker provides an in-memory metrics merging capability for OAL + */ +@Slf4j +public class MetricsAggregateOALWorker extends MetricsAggregateWorker { + private final static String POOL_NAME = "METRICS_L1_AGGREGATION_OAL"; + + MetricsAggregateOALWorker(ModuleDefineHolder moduleDefineHolder, + AbstractWorker nextWorker, + String modelName, + long l1FlushPeriod, + MetricStreamKind kind) { + super( + moduleDefineHolder, nextWorker, modelName, l1FlushPeriod, kind, + POOL_NAME, + (int) Math.ceil(BulkConsumePool.Creator.recommendMaxSize() * 1.5), + false, + 2, + 10_000 + ); + } +} \ No newline at end of file diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsAggregateWorker.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsAggregateWorker.java index 46decdf43fc1..b0a8bffa3430 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsAggregateWorker.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsAggregateWorker.java @@ -45,7 +45,7 @@ * payload. */ @Slf4j -public class MetricsAggregateWorker extends AbstractWorker { +public abstract class MetricsAggregateWorker extends AbstractWorker { public final long l1FlushPeriod; private AbstractWorker nextWorker; private final DataCarrier dataCarrier; @@ -54,39 +54,31 @@ public class MetricsAggregateWorker extends AbstractWorker { private CounterMetrics aggregationCounter; private GaugeMetrics queuePercentageGauge; private long lastSendTime = 0; - private final MetricStreamKind kind; private final int queueTotalSize; MetricsAggregateWorker(ModuleDefineHolder moduleDefineHolder, AbstractWorker nextWorker, String modelName, long l1FlushPeriod, - MetricStreamKind kind) { + MetricStreamKind kind, + String poolName, + int poolSize, + boolean isSignalDrivenMode, + int queueChannelSize, + int queueBufferSize + ) { super(moduleDefineHolder); this.nextWorker = nextWorker; this.mergeDataCache = new MergableBufferedData(); - this.kind = kind; - String name = "METRICS_L1_AGGREGATION"; - int queueChannelSize = 2; - int queueBufferSize = 10_000; - if (MetricStreamKind.MAL == kind) { - // In MAL meter streaming, the load of data flow is much less as they are statistics already, - // but in OAL sources, they are raw data. - // Set the buffer(size of queue) as 1/20 to reduce unnecessary resource costs. - queueChannelSize = 1; - queueBufferSize = 1_000; - } + BulkConsumePool.Creator creator = new BulkConsumePool.Creator(poolName, poolSize, 200, isSignalDrivenMode); this.dataCarrier = new DataCarrier<>( - "MetricsAggregateWorker." + modelName, name, queueChannelSize, queueBufferSize, BufferStrategy.IF_POSSIBLE); - - BulkConsumePool.Creator creator = new BulkConsumePool.Creator( - name, BulkConsumePool.Creator.recommendMaxSize() * 2, 200); + "MetricsAggregateWorker." + modelName, poolName, queueChannelSize, queueBufferSize, BufferStrategy.IF_POSSIBLE); try { - ConsumerPoolFactory.INSTANCE.createIfAbsent(name, creator); + ConsumerPoolFactory.INSTANCE.createIfAbsent(poolName, creator); } catch (Exception e) { throw new UnexpectedException(e.getMessage(), e); } - this.dataCarrier.consume(ConsumerPoolFactory.INSTANCE.get(name), new AggregatorConsumer()); + this.dataCarrier.consume(ConsumerPoolFactory.INSTANCE.get(poolName), new AggregatorConsumer()); MetricsCreator metricsCreator = moduleDefineHolder.find(TelemetryModule.NAME) .provider() @@ -116,7 +108,7 @@ public class MetricsAggregateWorker extends AbstractWorker { * MetricsAggregateWorker#in operation does include enqueue only */ @Override - public final void in(Metrics metrics) { + public void in(Metrics metrics) { if (!dataCarrier.produce(metrics)) { abandonCounter.inc(); } @@ -149,7 +141,7 @@ private void flush() { } } - private class AggregatorConsumer implements IConsumer { + protected class AggregatorConsumer implements IConsumer { @Override public void consume(List data) { queuePercentageGauge.setValue(Math.round(100 * (double) data.size() / queueTotalSize)); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentMinMALWorker.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentMinMALWorker.java new file mode 100644 index 000000000000..6ced82869427 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentMinMALWorker.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.analysis.worker; + +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; +import org.apache.skywalking.oap.server.core.exporter.ExportEvent; +import org.apache.skywalking.oap.server.core.storage.IMetricsDAO; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.worker.AbstractWorker; +import org.apache.skywalking.oap.server.library.datacarrier.consumer.BulkConsumePool; +import org.apache.skywalking.oap.server.library.datacarrier.consumer.ConsumerPoolFactory; +import org.apache.skywalking.oap.server.library.module.ModuleDefineHolder; + +@Slf4j +public class MetricsPersistentMinMALWorker extends MetricsPersistentMinWorker { + private final static String POOL_NAME = "METRICS_L2_AGGREGATION_MAL"; + private final BulkConsumePool pool; + + MetricsPersistentMinMALWorker(ModuleDefineHolder moduleDefineHolder, Model model, IMetricsDAO metricsDAO, + AbstractWorker nextAlarmWorker, AbstractWorker nextExportWorker, + MetricsTransWorker transWorker, boolean supportUpdate, + long storageSessionTimeout, int metricsDataTTL, MetricStreamKind kind) { + super( + moduleDefineHolder, model, metricsDAO, nextAlarmWorker, nextExportWorker, transWorker, supportUpdate, + storageSessionTimeout, metricsDataTTL, kind, + POOL_NAME, + calculatePoolSize(), + true, + 1, + 1000 + ); + this.pool = (BulkConsumePool) ConsumerPoolFactory.INSTANCE.get(POOL_NAME); + } + + @Override + public void in(Metrics metrics) { + super.in(metrics); + pool.notifyConsumers(); + } + + private static int calculatePoolSize() { + int size = BulkConsumePool.Creator.recommendMaxSize() / 16; + return size == 0 ? 1 : size; + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentMinOALWorker.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentMinOALWorker.java new file mode 100644 index 000000000000..534b50f9f51e --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentMinOALWorker.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.analysis.worker; + +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; +import org.apache.skywalking.oap.server.core.exporter.ExportEvent; +import org.apache.skywalking.oap.server.core.storage.IMetricsDAO; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.worker.AbstractWorker; +import org.apache.skywalking.oap.server.library.datacarrier.consumer.BulkConsumePool; +import org.apache.skywalking.oap.server.library.module.ModuleDefineHolder; + +@Slf4j +public class MetricsPersistentMinOALWorker extends MetricsPersistentMinWorker { + + MetricsPersistentMinOALWorker(ModuleDefineHolder moduleDefineHolder, Model model, IMetricsDAO metricsDAO, + AbstractWorker nextAlarmWorker, AbstractWorker nextExportWorker, + MetricsTransWorker transWorker, boolean supportUpdate, + long storageSessionTimeout, int metricsDataTTL, MetricStreamKind kind) { + super( + moduleDefineHolder, model, metricsDAO, nextAlarmWorker, nextExportWorker, transWorker, supportUpdate, + storageSessionTimeout, metricsDataTTL, kind, + "METRICS_L2_AGGREGATION_OAL", + calculatePoolSize(), + false, + 1, + 2000 + ); + } + + private static int calculatePoolSize() { + int size = BulkConsumePool.Creator.recommendMaxSize() / 8; + return size == 0 ? 1 : size; + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentMinWorker.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentMinWorker.java index b44988e23fa2..2e2f66704543 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentMinWorker.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentMinWorker.java @@ -26,7 +26,6 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.core.exporter.ExportEvent; import org.apache.skywalking.oap.server.core.status.ServerStatusService; -import org.apache.skywalking.oap.server.core.status.ServerStatusWatcher; import org.apache.skywalking.oap.server.core.storage.IMetricsDAO; import org.apache.skywalking.oap.server.core.storage.model.Model; import org.apache.skywalking.oap.server.core.worker.AbstractWorker; @@ -45,7 +44,7 @@ * MetricsPersistentMinWorker is an extension of {@link MetricsPersistentWorker} and focuses on the Minute Metrics data persistent. */ @Slf4j -public class MetricsPersistentMinWorker extends MetricsPersistentWorker implements ServerStatusWatcher { +public abstract class MetricsPersistentMinWorker extends MetricsPersistentWorker { private final DataCarrier dataCarrier; /** @@ -65,33 +64,22 @@ public class MetricsPersistentMinWorker extends MetricsPersistentWorker implemen MetricsPersistentMinWorker(ModuleDefineHolder moduleDefineHolder, Model model, IMetricsDAO metricsDAO, AbstractWorker nextAlarmWorker, AbstractWorker nextExportWorker, MetricsTransWorker transWorker, boolean supportUpdate, - long storageSessionTimeout, int metricsDataTTL, MetricStreamKind kind) { + long storageSessionTimeout, int metricsDataTTL, MetricStreamKind kind, + String poolName, int poolSize, boolean isSignalDrivenMode, + int queueChannelSize, int queueBufferSize) { super( moduleDefineHolder, model, metricsDAO, nextAlarmWorker, nextExportWorker, transWorker, supportUpdate, storageSessionTimeout, metricsDataTTL, kind ); - String name = "METRICS_L2_AGGREGATION"; - int size = BulkConsumePool.Creator.recommendMaxSize() / 8; - if (size == 0) { - size = 1; - } - BulkConsumePool.Creator creator = new BulkConsumePool.Creator(name, size, 200); + BulkConsumePool.Creator creator = new BulkConsumePool.Creator(poolName, poolSize, 200, isSignalDrivenMode); try { - ConsumerPoolFactory.INSTANCE.createIfAbsent(name, creator); + ConsumerPoolFactory.INSTANCE.createIfAbsent(poolName, creator); } catch (Exception e) { throw new UnexpectedException(e.getMessage(), e); } - - int bufferSize = 2000; - if (MetricStreamKind.MAL == kind) { - // In MAL meter streaming, the load of data flow is much less as they are statistics already, - // but in OAL sources, they are raw data. - // Set the buffer(size of queue) as 1/2 to reduce unnecessary resource costs. - bufferSize = 1000; - } - this.dataCarrier = new DataCarrier<>("MetricsPersistentWorker." + model.getName(), name, 1, bufferSize); - this.dataCarrier.consume(ConsumerPoolFactory.INSTANCE.get(name), new PersistentConsumer()); + this.dataCarrier = new DataCarrier<>("MetricsPersistentWorker." + model.getName(), poolName, queueChannelSize, queueBufferSize); + this.dataCarrier.consume(ConsumerPoolFactory.INSTANCE.get(poolName), new PersistentConsumer()); MetricsCreator metricsCreator = moduleDefineHolder.find(TelemetryModule.NAME) .provider() diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsStreamProcessor.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsStreamProcessor.java index 17e41d72f405..ee1a3bdca6f0 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsStreamProcessor.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsStreamProcessor.java @@ -194,9 +194,19 @@ private void create(ModuleDefineHolder moduleDefineHolder, workerInstanceSetter.put(remoteReceiverWorkerName, minutePersistentWorker, metricsClass); MetricsRemoteWorker remoteWorker = new MetricsRemoteWorker(moduleDefineHolder, remoteReceiverWorkerName); - MetricsAggregateWorker aggregateWorker = new MetricsAggregateWorker( - moduleDefineHolder, remoteWorker, stream.getName(), l1FlushPeriod, kind); - + MetricsAggregateWorker aggregateWorker; + switch (kind) { + case OAL: + aggregateWorker = new MetricsAggregateOALWorker( + moduleDefineHolder, remoteWorker, stream.getName(), l1FlushPeriod, kind); + break; + case MAL: + aggregateWorker = new MetricsAggregateMALWorker( + moduleDefineHolder, remoteWorker, stream.getName(), l1FlushPeriod, kind); + break; + default: + throw new IllegalArgumentException("Unsupported MetricStreamKind: " + kind); + } entryWorkers.put(metricsClass, aggregateWorker); } @@ -209,10 +219,23 @@ private MetricsPersistentWorker minutePersistentWorker(ModuleDefineHolder module AlarmNotifyWorker alarmNotifyWorker = new AlarmNotifyWorker(moduleDefineHolder); ExportMetricsWorker exportWorker = new ExportMetricsWorker(moduleDefineHolder); - MetricsPersistentWorker minutePersistentWorker = new MetricsPersistentMinWorker( - moduleDefineHolder, model, metricsDAO, alarmNotifyWorker, exportWorker, transWorker, - supportUpdate, storageSessionTimeout, metricsDataTTL, kind - ); + MetricsPersistentWorker minutePersistentWorker; + switch (kind) { + case OAL: + minutePersistentWorker = new MetricsPersistentMinOALWorker( + moduleDefineHolder, model, metricsDAO, alarmNotifyWorker, exportWorker, transWorker, + supportUpdate, storageSessionTimeout, metricsDataTTL, kind + ); + break; + case MAL: + minutePersistentWorker = new MetricsPersistentMinMALWorker( + moduleDefineHolder, model, metricsDAO, alarmNotifyWorker, exportWorker, transWorker, + supportUpdate, storageSessionTimeout, metricsDataTTL, kind + ); + break; + default: + throw new IllegalArgumentException("Unsupported MetricStreamKind: " + kind); + } persistentWorkers.add(minutePersistentWorker); return minutePersistentWorker; diff --git a/oap-server/server-library/library-datacarrier-queue/src/main/java/org/apache/skywalking/oap/server/library/datacarrier/consumer/BulkConsumePool.java b/oap-server/server-library/library-datacarrier-queue/src/main/java/org/apache/skywalking/oap/server/library/datacarrier/consumer/BulkConsumePool.java index 9d5bb0f6151e..3ee33f6581f0 100644 --- a/oap-server/server-library/library-datacarrier-queue/src/main/java/org/apache/skywalking/oap/server/library/datacarrier/consumer/BulkConsumePool.java +++ b/oap-server/server-library/library-datacarrier-queue/src/main/java/org/apache/skywalking/oap/server/library/datacarrier/consumer/BulkConsumePool.java @@ -34,11 +34,12 @@ public class BulkConsumePool implements ConsumerPool { private List allConsumers; private volatile boolean isStarted = false; - public BulkConsumePool(String name, int size, long consumeCycle) { + public BulkConsumePool(String name, int size, long consumeCycle, boolean isSignalDrivenMode) { size = EnvUtil.getInt(name + "_THREAD", size); allConsumers = new ArrayList<>(size); for (int i = 0; i < size; i++) { - MultipleChannelsConsumer multipleChannelsConsumer = new MultipleChannelsConsumer("DataCarrier." + name + ".BulkConsumePool." + i + ".Thread", consumeCycle); + MultipleChannelsConsumer multipleChannelsConsumer = new MultipleChannelsConsumer( + "DataCarrier." + name + ".BulkConsumePool." + i + ".Thread", consumeCycle, isSignalDrivenMode); multipleChannelsConsumer.setDaemon(true); allConsumers.add(multipleChannelsConsumer); } @@ -92,6 +93,12 @@ public void begin(Channels channels) { isStarted = true; } + public void notifyConsumers() { + for (MultipleChannelsConsumer consumer : allConsumers) { + consumer.setConsumeFlag(true); + } + } + /** * The creator for {@link BulkConsumePool}. */ @@ -99,16 +106,19 @@ public static class Creator implements Callable { private String name; private int size; private long consumeCycle; + // Consumer has two modes to drive consumption. 1. Polling mode. 2. Signal-Driven mode. + private final boolean isSignalDrivenMode; - public Creator(String name, int poolSize, long consumeCycle) { + public Creator(String name, int poolSize, long consumeCycle, boolean isSignalDrivenMode) { this.name = name; this.size = poolSize; this.consumeCycle = consumeCycle; + this.isSignalDrivenMode = isSignalDrivenMode; } @Override public ConsumerPool call() { - return new BulkConsumePool(name, size, consumeCycle); + return new BulkConsumePool(name, size, consumeCycle, isSignalDrivenMode); } public static int recommendMaxSize() { diff --git a/oap-server/server-library/library-datacarrier-queue/src/main/java/org/apache/skywalking/oap/server/library/datacarrier/consumer/MultipleChannelsConsumer.java b/oap-server/server-library/library-datacarrier-queue/src/main/java/org/apache/skywalking/oap/server/library/datacarrier/consumer/MultipleChannelsConsumer.java index 69e2732f2001..1551caca692d 100644 --- a/oap-server/server-library/library-datacarrier-queue/src/main/java/org/apache/skywalking/oap/server/library/datacarrier/consumer/MultipleChannelsConsumer.java +++ b/oap-server/server-library/library-datacarrier-queue/src/main/java/org/apache/skywalking/oap/server/library/datacarrier/consumer/MultipleChannelsConsumer.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.List; +import lombok.Setter; import org.apache.skywalking.oap.server.library.datacarrier.buffer.Channels; import org.apache.skywalking.oap.server.library.datacarrier.buffer.QueueBuffer; @@ -33,11 +34,17 @@ public class MultipleChannelsConsumer extends Thread { @SuppressWarnings("NonAtomicVolatileUpdate") private volatile long size; private final long consumeCycle; + // The flag to indicate whether the consumer thread should consume data. + @Setter + private volatile boolean consumeFlag = false; + // Consumer has two modes to drive consumption. 1. Polling mode. 2. Signal-Driven mode. + private final boolean isSignalDrivenMode; - public MultipleChannelsConsumer(String threadName, long consumeCycle) { + public MultipleChannelsConsumer(String threadName, long consumeCycle, boolean isSignalDrivenMode) { super(threadName); this.consumeTargets = new ArrayList<>(); this.consumeCycle = consumeCycle; + this.isSignalDrivenMode = isSignalDrivenMode; } @Override @@ -47,15 +54,29 @@ public void run() { final List consumeList = new ArrayList(2000); while (running) { boolean hasData = false; - for (Group target : consumeTargets) { - boolean consumed = consume(target, consumeList); - hasData = hasData || consumed; - } + if (!isSignalDrivenMode) { + for (Group target : consumeTargets) { + boolean consumed = consume(target, consumeList); + hasData = hasData || consumed; + } - if (!hasData) { - try { - Thread.sleep(consumeCycle); - } catch (InterruptedException e) { + if (!hasData) { + try { + Thread.sleep(consumeCycle); + } catch (InterruptedException e) { + } + } + } else { + if (consumeFlag) { + consumeFlag = false; + for (Group target : consumeTargets) { + consume(target, consumeList); + } + } else { + try { + Thread.sleep(consumeCycle); + } catch (InterruptedException e) { + } } } } diff --git a/oap-server/server-library/library-datacarrier-queue/src/test/java/org/apache/skywalking/oap/server/library/datacarrier/consumer/ConsumerPoolFactoryTest.java b/oap-server/server-library/library-datacarrier-queue/src/test/java/org/apache/skywalking/oap/server/library/datacarrier/consumer/ConsumerPoolFactoryTest.java index 4b69309cd584..ef4b2e7e62ce 100644 --- a/oap-server/server-library/library-datacarrier-queue/src/test/java/org/apache/skywalking/oap/server/library/datacarrier/consumer/ConsumerPoolFactoryTest.java +++ b/oap-server/server-library/library-datacarrier-queue/src/test/java/org/apache/skywalking/oap/server/library/datacarrier/consumer/ConsumerPoolFactoryTest.java @@ -29,7 +29,7 @@ public class ConsumerPoolFactoryTest { @BeforeEach public void createIfAbsent() throws Exception { - BulkConsumePool.Creator creator = new BulkConsumePool.Creator("my-test-pool", 10, 20); + BulkConsumePool.Creator creator = new BulkConsumePool.Creator("my-test-pool", 10, 20, false); boolean firstCreated = ConsumerPoolFactory.INSTANCE.createIfAbsent("my-test-pool", creator); assertTrue(firstCreated); From 4df8bd2cfb57472e6b9c0761b2e96a53d5e105a1 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Wed, 15 Oct 2025 19:44:59 +0800 Subject: [PATCH 33/69] merge change --- docs/en/changes/changes.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 8a168f98a19c..687b4e05eafd 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -50,6 +50,15 @@ * GraphQL API: metadata, topology, log and trace support query by name. * [Break Change] MQE function `sort_values` sorts according to the aggregation result and labels rather than the simple time series values. * Self Observability: add `metrics_aggregation_queue_used_percentage` and `metrics_persistent_collection_cached_size` metrics for the OAP server. +* Optimize metrics aggregate/persistent worker: separate `OAL` and `MAL` workers and consume pools. The dataflow signal drives the new MAL consumer, + the following table shows the pool size，driven mode and queue size for each worker. + +| Worker | poolSize | isSignalDrivenMode | queueChannelSize | queueBufferSize | +|-------------------------------|------------------------------------------|--------------------|------------------|-----------------| +| MetricsAggregateOALWorker | Math.ceil(availableProcessors * 2 * 1.5) | false | 2 | 10000 | +| MetricsAggregateMALWorker | availableProcessors * 2 / 8, at least 1 | true | 1 | 1000 | +| MetricsPersistentMinOALWorker | availableProcessors * 2 / 8, at least 1 | false | 1 | 2000 | +| MetricsPersistentMinMALWorker | availableProcessors * 2 / 16, at least 1 | true | 1 | 1000 | * Support pprof profiling feature #### UI From b5f3e8964695823c381a35228185a78d348bd8b4 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Wed, 15 Oct 2025 20:00:53 +0800 Subject: [PATCH 34/69] rollback ci --- .github/workflows/codeql.yaml | 3 +- .github/workflows/dead-link-checker.yaml | 3 - .../workflows/publish-docker-e2e-service.yaml | 1 - .github/workflows/publish-docker.yaml | 1 - .github/workflows/skywalking.yaml | 375 ++++++++++++++++++ 5 files changed, 376 insertions(+), 7 deletions(-) diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 038e92bcced6..59f762f808b6 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -18,8 +18,7 @@ name: "CodeQL" on: push: - branches: - - ci + branches: ["master"] pull_request: branches: ["master"] paths: diff --git a/.github/workflows/dead-link-checker.yaml b/.github/workflows/dead-link-checker.yaml index b05d292ce732..b134daf30021 100644 --- a/.github/workflows/dead-link-checker.yaml +++ b/.github/workflows/dead-link-checker.yaml @@ -17,9 +17,6 @@ name: Dead Link Checker on: - push: - branches: - - ci pull_request: paths: - 'docs/**' diff --git a/.github/workflows/publish-docker-e2e-service.yaml b/.github/workflows/publish-docker-e2e-service.yaml index 0b8420f67579..661053ec40ae 100644 --- a/.github/workflows/publish-docker-e2e-service.yaml +++ b/.github/workflows/publish-docker-e2e-service.yaml @@ -20,7 +20,6 @@ on: push: branches: - master - - ci paths: - 'test/e2e-v2/java-test-service/**' - 'test/Makefile' diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml index a3367b2465f2..c5bf734e54ab 100644 --- a/.github/workflows/publish-docker.yaml +++ b/.github/workflows/publish-docker.yaml @@ -20,7 +20,6 @@ on: push: branches: - master - - ci release: types: - released diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index 53f9a87d4e28..c2136b1feb53 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -345,6 +345,381 @@ jobs: fail-fast: false matrix: test: + - name: Cluster ZK/ES + config: test/e2e-v2/cases/cluster/zk/es/e2e.yaml + + - name: Agent NodeJS Backend + config: test/e2e-v2/cases/nodejs/e2e.yaml + - name: Agent Golang + config: test/e2e-v2/cases/go/e2e.yaml + - name: Agent NodeJS Frontend + config: test/e2e-v2/cases/browser/e2e.yaml + - name: Agent NodeJS Frontend ES + config: test/e2e-v2/cases/browser/es/e2e.yaml + - name: Agent NodeJS Frontend ES Sharding + config: test/e2e-v2/cases/browser/es/es-sharding/e2e.yaml + - name: Agent PHP + config: test/e2e-v2/cases/php/e2e.yaml + - name: Agent Python + config: test/e2e-v2/cases/python/e2e.yaml + - name: Agent Lua + config: test/e2e-v2/cases/lua/e2e.yaml + + - name: BanyanDB + config: test/e2e-v2/cases/storage/banyandb/e2e.yaml + - name: BanyanDB TLS + config: test/e2e-v2/cases/storage/banyandb/tls/e2e.yaml + - name: Storage MySQL + config: test/e2e-v2/cases/storage/mysql/e2e.yaml + - name: Storage PostgreSQL + config: test/e2e-v2/cases/storage/postgres/e2e.yaml + - name: Storage ES 7.16.3 + config: test/e2e-v2/cases/storage/es/e2e.yaml + env: ES_VERSION=7.16.3 + - name: Storage ES 7.17.10 + config: test/e2e-v2/cases/storage/es/e2e.yaml + env: ES_VERSION=7.17.10 + - name: Storage ES 8.1.0 + config: test/e2e-v2/cases/storage/es/e2e.yaml + env: ES_VERSION=8.1.0 + - name: Storage ES 8.9.0 + config: test/e2e-v2/cases/storage/es/e2e.yaml + env: ES_VERSION=8.9.0 + - name: Storage ES 8.9.0 + config: test/e2e-v2/cases/storage/es/e2e.yaml + env: ES_VERSION=8.18.1 + - name: Storage OpenSearch 1.1.0 + config: test/e2e-v2/cases/storage/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=1.1.0 + - name: Storage OpenSearch 1.3.10 + config: test/e2e-v2/cases/storage/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=1.3.10 + - name: Storage OpenSearch 2.4.0 + config: test/e2e-v2/cases/storage/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=2.4.0 + - name: Storage OpenSearch 2.8.0 + config: test/e2e-v2/cases/storage/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=2.8.0 + - name: Storage OpenSearch 3.0.0 + config: test/e2e-v2/cases/storage/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=3.0.0 + - name: Storage ES Sharding + config: test/e2e-v2/cases/storage/es/es-sharding/e2e.yaml + + - name: Alarm ES + config: test/e2e-v2/cases/alarm/es/e2e.yaml + - name: Alarm ES Sharding + config: test/e2e-v2/cases/alarm/es/es-sharding/e2e.yaml + - name: Alarm MySQL + config: test/e2e-v2/cases/alarm/mysql/e2e.yaml + - name: Alarm PostgreSQL + config: test/e2e-v2/cases/alarm/postgres/e2e.yaml + - name: Alarm BanyanDB + config: test/e2e-v2/cases/alarm/banyandb/e2e.yaml + + - name: Baseline-driven Alarm ES + config: test/e2e-v2/cases/baseline/es/e2e.yaml + - name: Baseline-driven Alarm ES Sharding + config: test/e2e-v2/cases/baseline/es/es-sharding/e2e.yaml + - name: Baseline-driven Alarm BanyanDB + config: test/e2e-v2/cases/baseline/banyandb/e2e.yaml + + - name: TTL ES 7.16.3 + config: test/e2e-v2/cases/ttl/es/e2e.yaml + env: ES_VERSION=7.16.3 + - name: TTL ES 8.8.1 + config: test/e2e-v2/cases/ttl/es/e2e.yaml + env: ES_VERSION=8.8.1 + - name: TTL ES 8.18.1 + config: test/e2e-v2/cases/ttl/es/e2e.yaml + env: ES_VERSION=8.18.1 + + - name: Event BanyanDB + config: test/e2e-v2/cases/event/banyandb/e2e.yaml + - name: Event ES + config: test/e2e-v2/cases/event/es/e2e.yaml + - name: Event MySQL + config: test/e2e-v2/cases/event/mysql/e2e.yaml + + - name: Log MySQL + config: test/e2e-v2/cases/log/mysql/e2e.yaml + - name: Log PostgreSQL + config: test/e2e-v2/cases/log/postgres/e2e.yaml + - name: Log ES 7.16.3 + config: test/e2e-v2/cases/log/es/e2e.yaml + env: ES_VERSION=7.16.3 + - name: Log ES 7.17.10 + config: test/e2e-v2/cases/log/es/e2e.yaml + env: ES_VERSION=7.17.10 + - name: Log ES 8.8.1 Sharding + config: test/e2e-v2/cases/log/es/es-sharding/e2e.yaml + env: ES_VERSION=8.8.1 + - name: Log ES 8.18.1 Sharding + config: test/e2e-v2/cases/log/es/es-sharding/e2e.yaml + env: ES_VERSION=8.18.1 + - name: Log BanyanDB + config: test/e2e-v2/cases/log/banyandb/e2e.yaml + + - name: Log FluentBit ES 7.16.3 + config: test/e2e-v2/cases/log/fluent-bit/e2e.yaml + env: ES_VERSION=7.16.3 + - name: Log FluentBit ES 7.17.10 + config: test/e2e-v2/cases/log/fluent-bit/e2e.yaml + env: ES_VERSION=7.17.10 + - name: Log FluentBit ES 8.8.1 + config: test/e2e-v2/cases/log/fluent-bit/e2e.yaml + env: ES_VERSION=8.8.1 + - name: Log FluentBit ES 8.18.1 + config: test/e2e-v2/cases/log/fluent-bit/e2e.yaml + env: ES_VERSION=8.18.1 + + - name: Trace Profiling BanyanDB + config: test/e2e-v2/cases/profiling/trace/banyandb/e2e.yaml + - name: Trace Profiling ES + config: test/e2e-v2/cases/profiling/trace/es/e2e.yaml + - name: Trace Profiling ES Sharding + config: test/e2e-v2/cases/profiling/trace/es/es-sharding/e2e.yaml + - name: Trace Profiling MySQL + config: test/e2e-v2/cases/profiling/trace/mysql/e2e.yaml + - name: Trace Profiling Postgres + config: test/e2e-v2/cases/profiling/trace/postgres/e2e.yaml + - name: Trace Profiling OpenSearch 1.1.0 + config: test/e2e-v2/cases/profiling/trace/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=1.1.0 + - name: Trace Profiling OpenSearch 1.3.6 + config: test/e2e-v2/cases/profiling/trace/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=1.3.6 + - name: Trace Profiling OpenSearch 2.4.0 + config: test/e2e-v2/cases/profiling/trace/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=2.4.0 + + - name: eBPF Profiling On CPU BanyanDB + config: test/e2e-v2/cases/profiling/ebpf/oncpu/banyandb/e2e.yaml + docker: + base: test/e2e-v2/cases/profiling/ebpf/oncpu/ + file: Dockerfile.sqrt + name: test/oncpu:test + - name: eBPF Profiling On CPU ES + config: test/e2e-v2/cases/profiling/ebpf/oncpu/es/e2e.yaml + docker: + base: test/e2e-v2/cases/profiling/ebpf/oncpu/ + file: Dockerfile.sqrt + name: test/oncpu:test + - name: eBPF Profiling On CPU ES Sharding + config: test/e2e-v2/cases/profiling/ebpf/oncpu/es/es-sharding/e2e.yaml + docker: + base: test/e2e-v2/cases/profiling/ebpf/oncpu/ + file: Dockerfile.sqrt + name: test/oncpu:test + - name: eBPF Profiling Off CPU + config: test/e2e-v2/cases/profiling/ebpf/offcpu/e2e.yaml + docker: + base: test/e2e-v2/cases/profiling/ebpf/offcpu/ + file: Dockerfile.file + name: test/offcpu:test + runs-on: ubuntu-24.04 + + - name: eBPF Profiling Network BanyanDB + config: test/e2e-v2/cases/profiling/ebpf/network/banyandb/e2e.yaml + docker: + base: test/e2e-v2/cases/profiling/ebpf/network/ + file: Dockerfile.service + name: test/network:test + - name: eBPF Profiling Network ES + config: test/e2e-v2/cases/profiling/ebpf/network/es/e2e.yaml + docker: + base: test/e2e-v2/cases/profiling/ebpf/network/ + file: Dockerfile.service + name: test/network:test + - name: eBPF Profiling Network ES Sharding + config: test/e2e-v2/cases/profiling/ebpf/network/es/es-sharding/e2e.yaml + docker: + base: test/e2e-v2/cases/profiling/ebpf/network/ + file: Dockerfile.service + name: test/network:test + + - name: Continuous Profiling BanyanDB + config: test/e2e-v2/cases/profiling/ebpf/continuous/banyandb/e2e.yaml + docker: + base: test/e2e-v2/cases/profiling/ebpf/continuous/ + file: Dockerfile.sqrt + name: test/continuous:test + - name: Continuous Profiling ES + config: test/e2e-v2/cases/profiling/ebpf/continuous/es/e2e.yaml + docker: + base: test/e2e-v2/cases/profiling/ebpf/continuous/ + file: Dockerfile.sqrt + name: test/continuous:test + - name: Continuous Profiling Sharding ES + config: test/e2e-v2/cases/profiling/ebpf/continuous/es/es-sharding/e2e.yaml + docker: + base: test/e2e-v2/cases/profiling/ebpf/continuous/ + file: Dockerfile.sqrt + name: test/continuous:test + + # eBPF Access Log + - name: eBPF Access Log BanyanDB + config: test/e2e-v2/cases/profiling/ebpf/access_log/banyandb/e2e.yaml + - name: eBPF Access Log ES + config: test/e2e-v2/cases/profiling/ebpf/access_log/es/e2e.yaml + - name: eBPF Access Log ES Sharding + config: test/e2e-v2/cases/profiling/ebpf/access_log/es/es-sharding/e2e.yaml + + - name: Kafka Basic + config: test/e2e-v2/cases/kafka/simple-so11y/e2e.yaml + - name: Kafka Profiling + config: test/e2e-v2/cases/kafka/profile/e2e.yaml + - name: Kafka Meter + config: test/e2e-v2/cases/kafka/meter/e2e.yaml + - name: Kafka Log + config: test/e2e-v2/cases/kafka/log/e2e.yaml + + - name: Istio Metrics Service 1.20.0 + config: test/e2e-v2/cases/istio/metrics/e2e.yaml + env: | + ISTIO_VERSION=1.20.0 + KUBERNETES_VERSION=28 + - name: Istio Metrics Service 1.21.0 + config: test/e2e-v2/cases/istio/metrics/e2e.yaml + env: | + ISTIO_VERSION=1.21.0 + KUBERNETES_VERSION=28 + - name: Istio Metrics Service 1.22.0 + config: test/e2e-v2/cases/istio/metrics/e2e.yaml + env: | + ISTIO_VERSION=1.22.0 + KUBERNETES_VERSION=28 + - name: Istio Metrics Service 1.23.0 + config: test/e2e-v2/cases/istio/metrics/e2e.yaml + env: | + ISTIO_VERSION=1.23.0 + KUBERNETES_VERSION=28 + - name: Istio Metrics Service 1.24.0 + config: test/e2e-v2/cases/istio/metrics/e2e.yaml + env: | + ISTIO_VERSION=1.24.0 + KUBERNETES_VERSION=28 + + - name: Rover with Istio Process 1.15.0 + config: test/e2e-v2/cases/rover/process/istio/e2e.yaml + env: ISTIO_VERSION=1.15.0 + runs-on: ubuntu-24.04 + + - name: Satellite + config: test/e2e-v2/cases/satellite/native-protocols/e2e.yaml + - name: Auth + config: test/e2e-v2/cases/simple/auth/e2e.yaml + - name: SSL + config: test/e2e-v2/cases/simple/ssl/e2e.yaml + - name: mTLS + config: test/e2e-v2/cases/simple/mtls/e2e.yaml + - name: Virtual Gateway + config: test/e2e-v2/cases/gateway/e2e.yaml + - name: Meter + config: test/e2e-v2/cases/meter/e2e.yaml + - name: VM Zabbix + config: test/e2e-v2/cases/vm/zabbix/e2e.yaml + - name: VM Prometheus + config: test/e2e-v2/cases/vm/prometheus-node-exporter/e2e.yaml + - name: VM Telegraf + config: test/e2e-v2/cases/vm/telegraf/e2e.yaml + - name: So11y + config: test/e2e-v2/cases/so11y/e2e.yaml + - name: MySQL Prometheus and slowsql + config: test/e2e-v2/cases/mysql/mysql-slowsql/e2e.yaml + - name: PostgreSQL Prometheus + config: test/e2e-v2/cases/postgresql/postgres-exporter/e2e.yaml + - name: MariaDB Prometheus and slowsql + config: test/e2e-v2/cases/mariadb/mariadb-slowsql/e2e.yaml + + - name: Zipkin ES + config: test/e2e-v2/cases/zipkin/es/e2e.yaml + - name: Zipkin ES Sharding + config: test/e2e-v2/cases/zipkin/es/es-sharding/e2e.yaml + - name: Zipkin MySQL + config: test/e2e-v2/cases/zipkin/mysql/e2e.yaml + - name: Zipkin Opensearch + config: test/e2e-v2/cases/zipkin/opensearch/e2e.yaml + - name: Zipkin Postgres + config: test/e2e-v2/cases/zipkin/postgres/e2e.yaml + - name: Zipkin Kafka + config: test/e2e-v2/cases/zipkin/kafka/e2e.yaml + - name: Zipkin BanyanDB + config: test/e2e-v2/cases/zipkin/banyandb/e2e.yaml + + - name: Nginx + config: test/e2e-v2/cases/nginx/e2e.yaml + - name: APISIX metrics + config: test/e2e-v2/cases/apisix/otel-collector/e2e.yaml + - name: Exporter Kafka + config: test/e2e-v2/cases/exporter/kafka/e2e.yaml + - name: Virtual MQ + config: test/e2e-v2/cases/virtual-mq/e2e.yaml + - name: AWS Cloud EKS + config: test/e2e-v2/cases/aws/eks/e2e.yaml + - name: Windows + config: test/e2e-v2/cases/win/e2e.yaml + - name: AWS Cloud S3 + config: test/e2e-v2/cases/aws/s3/e2e.yaml + - name: AWS Cloud DynamoDB + config: test/e2e-v2/cases/aws/dynamodb/e2e.yaml + - name: PromQL Service + config: test/e2e-v2/cases/promql/e2e.yaml + - name: LogQL Service + config: test/e2e-v2/cases/logql/e2e.yaml + - name: AWS API Gateway + config: test/e2e-v2/cases/aws/api-gateway/e2e.yaml + - name: Redis Prometheus and Log Collecting + config: test/e2e-v2/cases/redis/redis-exporter/e2e.yaml + - name: Elasticsearch + config: test/e2e-v2/cases/elasticsearch/e2e.yaml + - name: MongoDB + config: test/e2e-v2/cases/mongodb/e2e.yaml + - name: RabbitMQ + config: test/e2e-v2/cases/rabbitmq/e2e.yaml + - name: Kafka + config: test/e2e-v2/cases/kafka/kafka-monitoring/e2e.yaml + - name: MQE Service + config: test/e2e-v2/cases/mqe/e2e.yaml + - name: Pulsar and BookKeeper + config: test/e2e-v2/cases/pulsar/e2e.yaml + - name: RocketMQ + config: test/e2e-v2/cases/rocketmq/e2e.yaml + - name: ClickHouse + config: test/e2e-v2/cases/clickhouse/clickhouse-prometheus-endpoint/e2e.yaml + - name: ActiveMQ + config: test/e2e-v2/cases/activemq/e2e.yaml + - name: Kong + config: test/e2e-v2/cases/kong/e2e.yaml + - name: Flink + config: test/e2e-v2/cases/flink/e2e.yaml + + - name: UI Menu BanyanDB + config: test/e2e-v2/cases/menu/banyandb/e2e.yaml + - name: UI Menu ES + config: test/e2e-v2/cases/menu/es/e2e.yaml + - name: UI Menu Sharding ES + config: test/e2e-v2/cases/menu/es/es-sharding/e2e.yaml + - name: UI Menu MySQL + config: test/e2e-v2/cases/menu/mysql/e2e.yaml + - name: UI Menu Postgres + config: test/e2e-v2/cases/menu/postgres/e2e.yaml + - name: UI Menu OpenSearch 1.1.0 + config: test/e2e-v2/cases/menu/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=1.1.0 + - name: UI Menu OpenSearch 1.3.6 + config: test/e2e-v2/cases/menu/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=1.3.6 + - name: UI Menu OpenSearch 2.4.0 + config: test/e2e-v2/cases/menu/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=2.4.0 + + - name: OTLP Trace + config: test/e2e-v2/cases/otlp-traces/e2e.yaml + + - name: Cilium Service + config: test/e2e-v2/cases/cilium/e2e.yaml + - name: Async Profiler ES config: test/e2e-v2/cases/profiling/async-profiler/es/e2e.yaml - name: Async Profiler BanyanDB From 57fc59fbf030038e6eb70d035fe30f1638545bb5 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Wed, 15 Oct 2025 20:02:43 +0800 Subject: [PATCH 35/69] rollback ci --- .github/workflows/skywalking.yaml | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index c2136b1feb53..0bd11fca045c 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -17,9 +17,6 @@ name: CI on: - push: - branches: - - ci pull_request: schedule: - cron: "0 18 * * *" # TimeZone: UTC 0 @@ -35,7 +32,7 @@ env: jobs: license-header: - if: (github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || (github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') name: License header runs-on: ubuntu-latest timeout-minutes: 10 @@ -48,7 +45,7 @@ jobs: uses: apache/skywalking-eyes@5b7ee1731d036b5aac68f8bd3fc9e6f98ada082e code-style: - if: (github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || (github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') name: Code style runs-on: ubuntu-latest timeout-minutes: 10 @@ -63,7 +60,7 @@ jobs: dependency-license: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.pom == 'true' || needs.changes.outputs.ui == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.pom == 'true' || needs.changes.outputs.ui == 'true') name: Dependency licenses needs: [changes] runs-on: ubuntu-latest @@ -93,7 +90,7 @@ jobs: fi sanity-check: - if: ( always() && ! cancelled() ) && (github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || (github.event_name != 'schedule') + if: ( always() && ! cancelled() ) && (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') name: Sanity check results needs: [license-header, code-style, dependency-license] runs-on: ubuntu-latest @@ -162,7 +159,7 @@ jobs: dist-tar: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: Build dist tar needs: [changes] runs-on: ubuntu-latest @@ -194,7 +191,7 @@ jobs: docker: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: Docker images needs: [sanity-check, dist-tar, changes] runs-on: ubuntu-latest @@ -233,7 +230,7 @@ jobs: unit-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: Unit test needs: [sanity-check, changes] runs-on: ${{ matrix.os }} @@ -268,7 +265,7 @@ jobs: integration-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: Integration test needs: [sanity-check, changes] runs-on: ubuntu-latest @@ -301,7 +298,7 @@ jobs: slow-integration-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: Slow Integration Tests needs: [sanity-check, changes] runs-on: ubuntu-latest @@ -334,7 +331,7 @@ jobs: e2e-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker, dist-tar] runs-on: ${{ matrix.test.runs-on || 'ubuntu-latest' }} @@ -796,7 +793,7 @@ jobs: e2e-test-istio: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-24.04 @@ -864,7 +861,7 @@ jobs: e2e-test-istio-ambient: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-24.04 @@ -925,7 +922,7 @@ jobs: e2e-test-java-versions: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-latest From 39911d487f37407dcffcd54305bdc3319da5e2e7 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Thu, 16 Oct 2025 16:23:06 +0800 Subject: [PATCH 36/69] fix ci --- test/e2e-v2/cases/go/docker-compose.yml | 5 +++++ test/e2e-v2/cases/go/service/Dockerfile | 4 +--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/test/e2e-v2/cases/go/docker-compose.yml b/test/e2e-v2/cases/go/docker-compose.yml index cfae30d2df11..8253b7a1c1d6 100644 --- a/test/e2e-v2/cases/go/docker-compose.yml +++ b/test/e2e-v2/cases/go/docker-compose.yml @@ -20,8 +20,13 @@ services: extends: file: ../../script/docker-compose/base-compose.yml service: oap + environment: + SW_STORAGE: banyandb ports: - 12800 + depends_on: + banyandb: + condition: service_healthy banyandb: extends: diff --git a/test/e2e-v2/cases/go/service/Dockerfile b/test/e2e-v2/cases/go/service/Dockerfile index 2f16f90a1cda..73ff4131d4fe 100644 --- a/test/e2e-v2/cases/go/service/Dockerfile +++ b/test/e2e-v2/cases/go/service/Dockerfile @@ -13,9 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG SW_AGENT_GO_COMMIT=aa948377ecdb4724fad1cc365c13a1188021316f -ARG GO_VERSION=go1.19 -FROM ghcr.io/apache/skywalking-go/skywalking-go:${SW_AGENT_GO_COMMIT}-${GO_VERSION} AS base +FROM ghcr.io/apache/skywalking-go/skywalking-go:${SW_AGENT_GO_COMMIT}-go1.19 AS base ENV CGO_ENABLED=0 ENV GO111MODULE=on From 78a2c9cb6a3e5e7ed5dc6780159e1c06c248a9a1 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Thu, 16 Oct 2025 16:24:16 +0800 Subject: [PATCH 37/69] test ci(to roll back) --- .github/workflows/skywalking.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index 0bd11fca045c..a15881275b21 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -17,6 +17,9 @@ name: CI on: + push: + branches: + - citest pull_request: schedule: - cron: "0 18 * * *" # TimeZone: UTC 0 From a4882ff67ea226c14a4662185c89d96f4abd0f77 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Thu, 16 Oct 2025 16:26:03 +0800 Subject: [PATCH 38/69] test ci(to roll back) --- .github/workflows/skywalking.yaml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index a15881275b21..a953ec90f9af 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -35,7 +35,7 @@ env: jobs: license-header: - if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || (github.event_name != 'schedule') name: License header runs-on: ubuntu-latest timeout-minutes: 10 @@ -48,7 +48,7 @@ jobs: uses: apache/skywalking-eyes@5b7ee1731d036b5aac68f8bd3fc9e6f98ada082e code-style: - if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || (github.event_name != 'schedule') name: Code style runs-on: ubuntu-latest timeout-minutes: 10 @@ -63,7 +63,7 @@ jobs: dependency-license: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.pom == 'true' || needs.changes.outputs.ui == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.pom == 'true' || needs.changes.outputs.ui == 'true') name: Dependency licenses needs: [changes] runs-on: ubuntu-latest @@ -93,7 +93,7 @@ jobs: fi sanity-check: - if: ( always() && ! cancelled() ) && (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') + if: ( always() && ! cancelled() ) && (github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || (github.event_name != 'schedule') name: Sanity check results needs: [license-header, code-style, dependency-license] runs-on: ubuntu-latest @@ -162,7 +162,7 @@ jobs: dist-tar: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: Build dist tar needs: [changes] runs-on: ubuntu-latest @@ -194,7 +194,7 @@ jobs: docker: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: Docker images needs: [sanity-check, dist-tar, changes] runs-on: ubuntu-latest @@ -233,7 +233,7 @@ jobs: unit-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: Unit test needs: [sanity-check, changes] runs-on: ${{ matrix.os }} @@ -268,7 +268,7 @@ jobs: integration-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: Integration test needs: [sanity-check, changes] runs-on: ubuntu-latest @@ -301,7 +301,7 @@ jobs: slow-integration-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: Slow Integration Tests needs: [sanity-check, changes] runs-on: ubuntu-latest @@ -334,7 +334,7 @@ jobs: e2e-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker, dist-tar] runs-on: ${{ matrix.test.runs-on || 'ubuntu-latest' }} @@ -796,7 +796,7 @@ jobs: e2e-test-istio: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-24.04 @@ -864,7 +864,7 @@ jobs: e2e-test-istio-ambient: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-24.04 @@ -925,7 +925,7 @@ jobs: e2e-test-java-versions: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-latest From ba4f02f6cb49ce3559135e3c69698c25d35dc643 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Thu, 16 Oct 2025 20:51:19 +0800 Subject: [PATCH 39/69] fix ci --- .github/workflows/skywalking.yaml | 375 +----------------------- test/e2e-v2/cases/go/docker-compose.yml | 2 - test/e2e-v2/cases/go/service/Dockerfile | 3 + 3 files changed, 4 insertions(+), 376 deletions(-) diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index a953ec90f9af..7d1f10c1db72 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -345,381 +345,8 @@ jobs: fail-fast: false matrix: test: - - name: Cluster ZK/ES - config: test/e2e-v2/cases/cluster/zk/es/e2e.yaml - - - name: Agent NodeJS Backend - config: test/e2e-v2/cases/nodejs/e2e.yaml - name: Agent Golang - config: test/e2e-v2/cases/go/e2e.yaml - - name: Agent NodeJS Frontend - config: test/e2e-v2/cases/browser/e2e.yaml - - name: Agent NodeJS Frontend ES - config: test/e2e-v2/cases/browser/es/e2e.yaml - - name: Agent NodeJS Frontend ES Sharding - config: test/e2e-v2/cases/browser/es/es-sharding/e2e.yaml - - name: Agent PHP - config: test/e2e-v2/cases/php/e2e.yaml - - name: Agent Python - config: test/e2e-v2/cases/python/e2e.yaml - - name: Agent Lua - config: test/e2e-v2/cases/lua/e2e.yaml - - - name: BanyanDB - config: test/e2e-v2/cases/storage/banyandb/e2e.yaml - - name: BanyanDB TLS - config: test/e2e-v2/cases/storage/banyandb/tls/e2e.yaml - - name: Storage MySQL - config: test/e2e-v2/cases/storage/mysql/e2e.yaml - - name: Storage PostgreSQL - config: test/e2e-v2/cases/storage/postgres/e2e.yaml - - name: Storage ES 7.16.3 - config: test/e2e-v2/cases/storage/es/e2e.yaml - env: ES_VERSION=7.16.3 - - name: Storage ES 7.17.10 - config: test/e2e-v2/cases/storage/es/e2e.yaml - env: ES_VERSION=7.17.10 - - name: Storage ES 8.1.0 - config: test/e2e-v2/cases/storage/es/e2e.yaml - env: ES_VERSION=8.1.0 - - name: Storage ES 8.9.0 - config: test/e2e-v2/cases/storage/es/e2e.yaml - env: ES_VERSION=8.9.0 - - name: Storage ES 8.9.0 - config: test/e2e-v2/cases/storage/es/e2e.yaml - env: ES_VERSION=8.18.1 - - name: Storage OpenSearch 1.1.0 - config: test/e2e-v2/cases/storage/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=1.1.0 - - name: Storage OpenSearch 1.3.10 - config: test/e2e-v2/cases/storage/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=1.3.10 - - name: Storage OpenSearch 2.4.0 - config: test/e2e-v2/cases/storage/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=2.4.0 - - name: Storage OpenSearch 2.8.0 - config: test/e2e-v2/cases/storage/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=2.8.0 - - name: Storage OpenSearch 3.0.0 - config: test/e2e-v2/cases/storage/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=3.0.0 - - name: Storage ES Sharding - config: test/e2e-v2/cases/storage/es/es-sharding/e2e.yaml - - - name: Alarm ES - config: test/e2e-v2/cases/alarm/es/e2e.yaml - - name: Alarm ES Sharding - config: test/e2e-v2/cases/alarm/es/es-sharding/e2e.yaml - - name: Alarm MySQL - config: test/e2e-v2/cases/alarm/mysql/e2e.yaml - - name: Alarm PostgreSQL - config: test/e2e-v2/cases/alarm/postgres/e2e.yaml - - name: Alarm BanyanDB - config: test/e2e-v2/cases/alarm/banyandb/e2e.yaml - - - name: Baseline-driven Alarm ES - config: test/e2e-v2/cases/baseline/es/e2e.yaml - - name: Baseline-driven Alarm ES Sharding - config: test/e2e-v2/cases/baseline/es/es-sharding/e2e.yaml - - name: Baseline-driven Alarm BanyanDB - config: test/e2e-v2/cases/baseline/banyandb/e2e.yaml - - - name: TTL ES 7.16.3 - config: test/e2e-v2/cases/ttl/es/e2e.yaml - env: ES_VERSION=7.16.3 - - name: TTL ES 8.8.1 - config: test/e2e-v2/cases/ttl/es/e2e.yaml - env: ES_VERSION=8.8.1 - - name: TTL ES 8.18.1 - config: test/e2e-v2/cases/ttl/es/e2e.yaml - env: ES_VERSION=8.18.1 - - - name: Event BanyanDB - config: test/e2e-v2/cases/event/banyandb/e2e.yaml - - name: Event ES - config: test/e2e-v2/cases/event/es/e2e.yaml - - name: Event MySQL - config: test/e2e-v2/cases/event/mysql/e2e.yaml - - - name: Log MySQL - config: test/e2e-v2/cases/log/mysql/e2e.yaml - - name: Log PostgreSQL - config: test/e2e-v2/cases/log/postgres/e2e.yaml - - name: Log ES 7.16.3 - config: test/e2e-v2/cases/log/es/e2e.yaml - env: ES_VERSION=7.16.3 - - name: Log ES 7.17.10 - config: test/e2e-v2/cases/log/es/e2e.yaml - env: ES_VERSION=7.17.10 - - name: Log ES 8.8.1 Sharding - config: test/e2e-v2/cases/log/es/es-sharding/e2e.yaml - env: ES_VERSION=8.8.1 - - name: Log ES 8.18.1 Sharding - config: test/e2e-v2/cases/log/es/es-sharding/e2e.yaml - env: ES_VERSION=8.18.1 - - name: Log BanyanDB - config: test/e2e-v2/cases/log/banyandb/e2e.yaml - - - name: Log FluentBit ES 7.16.3 - config: test/e2e-v2/cases/log/fluent-bit/e2e.yaml - env: ES_VERSION=7.16.3 - - name: Log FluentBit ES 7.17.10 - config: test/e2e-v2/cases/log/fluent-bit/e2e.yaml - env: ES_VERSION=7.17.10 - - name: Log FluentBit ES 8.8.1 - config: test/e2e-v2/cases/log/fluent-bit/e2e.yaml - env: ES_VERSION=8.8.1 - - name: Log FluentBit ES 8.18.1 - config: test/e2e-v2/cases/log/fluent-bit/e2e.yaml - env: ES_VERSION=8.18.1 - - - name: Trace Profiling BanyanDB - config: test/e2e-v2/cases/profiling/trace/banyandb/e2e.yaml - - name: Trace Profiling ES - config: test/e2e-v2/cases/profiling/trace/es/e2e.yaml - - name: Trace Profiling ES Sharding - config: test/e2e-v2/cases/profiling/trace/es/es-sharding/e2e.yaml - - name: Trace Profiling MySQL - config: test/e2e-v2/cases/profiling/trace/mysql/e2e.yaml - - name: Trace Profiling Postgres - config: test/e2e-v2/cases/profiling/trace/postgres/e2e.yaml - - name: Trace Profiling OpenSearch 1.1.0 - config: test/e2e-v2/cases/profiling/trace/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=1.1.0 - - name: Trace Profiling OpenSearch 1.3.6 - config: test/e2e-v2/cases/profiling/trace/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=1.3.6 - - name: Trace Profiling OpenSearch 2.4.0 - config: test/e2e-v2/cases/profiling/trace/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=2.4.0 - - - name: eBPF Profiling On CPU BanyanDB - config: test/e2e-v2/cases/profiling/ebpf/oncpu/banyandb/e2e.yaml - docker: - base: test/e2e-v2/cases/profiling/ebpf/oncpu/ - file: Dockerfile.sqrt - name: test/oncpu:test - - name: eBPF Profiling On CPU ES - config: test/e2e-v2/cases/profiling/ebpf/oncpu/es/e2e.yaml - docker: - base: test/e2e-v2/cases/profiling/ebpf/oncpu/ - file: Dockerfile.sqrt - name: test/oncpu:test - - name: eBPF Profiling On CPU ES Sharding - config: test/e2e-v2/cases/profiling/ebpf/oncpu/es/es-sharding/e2e.yaml - docker: - base: test/e2e-v2/cases/profiling/ebpf/oncpu/ - file: Dockerfile.sqrt - name: test/oncpu:test - - name: eBPF Profiling Off CPU - config: test/e2e-v2/cases/profiling/ebpf/offcpu/e2e.yaml - docker: - base: test/e2e-v2/cases/profiling/ebpf/offcpu/ - file: Dockerfile.file - name: test/offcpu:test - runs-on: ubuntu-24.04 - - - name: eBPF Profiling Network BanyanDB - config: test/e2e-v2/cases/profiling/ebpf/network/banyandb/e2e.yaml - docker: - base: test/e2e-v2/cases/profiling/ebpf/network/ - file: Dockerfile.service - name: test/network:test - - name: eBPF Profiling Network ES - config: test/e2e-v2/cases/profiling/ebpf/network/es/e2e.yaml - docker: - base: test/e2e-v2/cases/profiling/ebpf/network/ - file: Dockerfile.service - name: test/network:test - - name: eBPF Profiling Network ES Sharding - config: test/e2e-v2/cases/profiling/ebpf/network/es/es-sharding/e2e.yaml - docker: - base: test/e2e-v2/cases/profiling/ebpf/network/ - file: Dockerfile.service - name: test/network:test - - - name: Continuous Profiling BanyanDB - config: test/e2e-v2/cases/profiling/ebpf/continuous/banyandb/e2e.yaml - docker: - base: test/e2e-v2/cases/profiling/ebpf/continuous/ - file: Dockerfile.sqrt - name: test/continuous:test - - name: Continuous Profiling ES - config: test/e2e-v2/cases/profiling/ebpf/continuous/es/e2e.yaml - docker: - base: test/e2e-v2/cases/profiling/ebpf/continuous/ - file: Dockerfile.sqrt - name: test/continuous:test - - name: Continuous Profiling Sharding ES - config: test/e2e-v2/cases/profiling/ebpf/continuous/es/es-sharding/e2e.yaml - docker: - base: test/e2e-v2/cases/profiling/ebpf/continuous/ - file: Dockerfile.sqrt - name: test/continuous:test - - # eBPF Access Log - - name: eBPF Access Log BanyanDB - config: test/e2e-v2/cases/profiling/ebpf/access_log/banyandb/e2e.yaml - - name: eBPF Access Log ES - config: test/e2e-v2/cases/profiling/ebpf/access_log/es/e2e.yaml - - name: eBPF Access Log ES Sharding - config: test/e2e-v2/cases/profiling/ebpf/access_log/es/es-sharding/e2e.yaml - - - name: Kafka Basic - config: test/e2e-v2/cases/kafka/simple-so11y/e2e.yaml - - name: Kafka Profiling - config: test/e2e-v2/cases/kafka/profile/e2e.yaml - - name: Kafka Meter - config: test/e2e-v2/cases/kafka/meter/e2e.yaml - - name: Kafka Log - config: test/e2e-v2/cases/kafka/log/e2e.yaml - - - name: Istio Metrics Service 1.20.0 - config: test/e2e-v2/cases/istio/metrics/e2e.yaml - env: | - ISTIO_VERSION=1.20.0 - KUBERNETES_VERSION=28 - - name: Istio Metrics Service 1.21.0 - config: test/e2e-v2/cases/istio/metrics/e2e.yaml - env: | - ISTIO_VERSION=1.21.0 - KUBERNETES_VERSION=28 - - name: Istio Metrics Service 1.22.0 - config: test/e2e-v2/cases/istio/metrics/e2e.yaml - env: | - ISTIO_VERSION=1.22.0 - KUBERNETES_VERSION=28 - - name: Istio Metrics Service 1.23.0 - config: test/e2e-v2/cases/istio/metrics/e2e.yaml - env: | - ISTIO_VERSION=1.23.0 - KUBERNETES_VERSION=28 - - name: Istio Metrics Service 1.24.0 - config: test/e2e-v2/cases/istio/metrics/e2e.yaml - env: | - ISTIO_VERSION=1.24.0 - KUBERNETES_VERSION=28 - - - name: Rover with Istio Process 1.15.0 - config: test/e2e-v2/cases/rover/process/istio/e2e.yaml - env: ISTIO_VERSION=1.15.0 - runs-on: ubuntu-24.04 - - - name: Satellite - config: test/e2e-v2/cases/satellite/native-protocols/e2e.yaml - - name: Auth - config: test/e2e-v2/cases/simple/auth/e2e.yaml - - name: SSL - config: test/e2e-v2/cases/simple/ssl/e2e.yaml - - name: mTLS - config: test/e2e-v2/cases/simple/mtls/e2e.yaml - - name: Virtual Gateway - config: test/e2e-v2/cases/gateway/e2e.yaml - - name: Meter - config: test/e2e-v2/cases/meter/e2e.yaml - - name: VM Zabbix - config: test/e2e-v2/cases/vm/zabbix/e2e.yaml - - name: VM Prometheus - config: test/e2e-v2/cases/vm/prometheus-node-exporter/e2e.yaml - - name: VM Telegraf - config: test/e2e-v2/cases/vm/telegraf/e2e.yaml - - name: So11y - config: test/e2e-v2/cases/so11y/e2e.yaml - - name: MySQL Prometheus and slowsql - config: test/e2e-v2/cases/mysql/mysql-slowsql/e2e.yaml - - name: PostgreSQL Prometheus - config: test/e2e-v2/cases/postgresql/postgres-exporter/e2e.yaml - - name: MariaDB Prometheus and slowsql - config: test/e2e-v2/cases/mariadb/mariadb-slowsql/e2e.yaml - - - name: Zipkin ES - config: test/e2e-v2/cases/zipkin/es/e2e.yaml - - name: Zipkin ES Sharding - config: test/e2e-v2/cases/zipkin/es/es-sharding/e2e.yaml - - name: Zipkin MySQL - config: test/e2e-v2/cases/zipkin/mysql/e2e.yaml - - name: Zipkin Opensearch - config: test/e2e-v2/cases/zipkin/opensearch/e2e.yaml - - name: Zipkin Postgres - config: test/e2e-v2/cases/zipkin/postgres/e2e.yaml - - name: Zipkin Kafka - config: test/e2e-v2/cases/zipkin/kafka/e2e.yaml - - name: Zipkin BanyanDB - config: test/e2e-v2/cases/zipkin/banyandb/e2e.yaml - - - name: Nginx - config: test/e2e-v2/cases/nginx/e2e.yaml - - name: APISIX metrics - config: test/e2e-v2/cases/apisix/otel-collector/e2e.yaml - - name: Exporter Kafka - config: test/e2e-v2/cases/exporter/kafka/e2e.yaml - - name: Virtual MQ - config: test/e2e-v2/cases/virtual-mq/e2e.yaml - - name: AWS Cloud EKS - config: test/e2e-v2/cases/aws/eks/e2e.yaml - - name: Windows - config: test/e2e-v2/cases/win/e2e.yaml - - name: AWS Cloud S3 - config: test/e2e-v2/cases/aws/s3/e2e.yaml - - name: AWS Cloud DynamoDB - config: test/e2e-v2/cases/aws/dynamodb/e2e.yaml - - name: PromQL Service - config: test/e2e-v2/cases/promql/e2e.yaml - - name: LogQL Service - config: test/e2e-v2/cases/logql/e2e.yaml - - name: AWS API Gateway - config: test/e2e-v2/cases/aws/api-gateway/e2e.yaml - - name: Redis Prometheus and Log Collecting - config: test/e2e-v2/cases/redis/redis-exporter/e2e.yaml - - name: Elasticsearch - config: test/e2e-v2/cases/elasticsearch/e2e.yaml - - name: MongoDB - config: test/e2e-v2/cases/mongodb/e2e.yaml - - name: RabbitMQ - config: test/e2e-v2/cases/rabbitmq/e2e.yaml - - name: Kafka - config: test/e2e-v2/cases/kafka/kafka-monitoring/e2e.yaml - - name: MQE Service - config: test/e2e-v2/cases/mqe/e2e.yaml - - name: Pulsar and BookKeeper - config: test/e2e-v2/cases/pulsar/e2e.yaml - - name: RocketMQ - config: test/e2e-v2/cases/rocketmq/e2e.yaml - - name: ClickHouse - config: test/e2e-v2/cases/clickhouse/clickhouse-prometheus-endpoint/e2e.yaml - - name: ActiveMQ - config: test/e2e-v2/cases/activemq/e2e.yaml - - name: Kong - config: test/e2e-v2/cases/kong/e2e.yaml - - name: Flink - config: test/e2e-v2/cases/flink/e2e.yaml - - - name: UI Menu BanyanDB - config: test/e2e-v2/cases/menu/banyandb/e2e.yaml - - name: UI Menu ES - config: test/e2e-v2/cases/menu/es/e2e.yaml - - name: UI Menu Sharding ES - config: test/e2e-v2/cases/menu/es/es-sharding/e2e.yaml - - name: UI Menu MySQL - config: test/e2e-v2/cases/menu/mysql/e2e.yaml - - name: UI Menu Postgres - config: test/e2e-v2/cases/menu/postgres/e2e.yaml - - name: UI Menu OpenSearch 1.1.0 - config: test/e2e-v2/cases/menu/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=1.1.0 - - name: UI Menu OpenSearch 1.3.6 - config: test/e2e-v2/cases/menu/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=1.3.6 - - name: UI Menu OpenSearch 2.4.0 - config: test/e2e-v2/cases/menu/opensearch/e2e.yaml - env: OPENSEARCH_VERSION=2.4.0 - - - name: OTLP Trace - config: test/e2e-v2/cases/otlp-traces/e2e.yaml - - - name: Cilium Service - config: test/e2e-v2/cases/cilium/e2e.yaml - + config: test/e2e-v2/cases/go/docker-compose.yml - name: Async Profiler ES config: test/e2e-v2/cases/profiling/async-profiler/es/e2e.yaml - name: Async Profiler BanyanDB diff --git a/test/e2e-v2/cases/go/docker-compose.yml b/test/e2e-v2/cases/go/docker-compose.yml index 8253b7a1c1d6..83ee6e3ff37c 100644 --- a/test/e2e-v2/cases/go/docker-compose.yml +++ b/test/e2e-v2/cases/go/docker-compose.yml @@ -61,8 +61,6 @@ services: depends_on: oap: condition: service_healthy - provider: - condition: service_healthy healthcheck: test: ["CMD", "sh", "-c", "nc -z 127.0.0.1 8080"] interval: 5s diff --git a/test/e2e-v2/cases/go/service/Dockerfile b/test/e2e-v2/cases/go/service/Dockerfile index 73ff4131d4fe..e76b93ffb26a 100644 --- a/test/e2e-v2/cases/go/service/Dockerfile +++ b/test/e2e-v2/cases/go/service/Dockerfile @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +ARG SW_AGENT_GO_COMMIT FROM ghcr.io/apache/skywalking-go/skywalking-go:${SW_AGENT_GO_COMMIT}-go1.19 AS base ENV CGO_ENABLED=0 @@ -25,6 +26,8 @@ RUN go mod tidy && go build -toolexec="skywalking-go-agent" -a -o service FROM alpine:3.10 +RUN apk add --no-cache busybox-extras + COPY --from=base /go-service/service /service ENTRYPOINT ["/service"] \ No newline at end of file From 9c1379a3c52ae2a0c9e12afc8c845c8c511b734f Mon Sep 17 00:00:00 2001 From: JophieQu Date: Thu, 16 Oct 2025 22:13:33 +0800 Subject: [PATCH 40/69] fix --- .github/workflows/skywalking.yaml | 13 ++++++++++--- .../profiling/pprof/banyandb/docker-compose.yml | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index 7d1f10c1db72..5d23f7662a73 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -410,9 +410,16 @@ jobs: e2e-file: $GITHUB_WORKSPACE/${{ matrix.test.config }} - if: ${{ failure() }} run: | - df -h - du -sh . - docker images + echo "=== Container Status ===" + docker ps -a + + echo "=== Container Logs ===" + docker-compose -f ${{ matrix.test.config }} logs --tail=100 + + echo "=== Health Check Details ===" + for container in $(docker ps -a -q); do + docker inspect $container | jq '.[0].State.Health' + done - uses: actions/upload-artifact@v4 if: ${{ failure() }} name: Upload Logs diff --git a/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml index f8964fe7a8d3..eba20b929672 100644 --- a/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml @@ -44,7 +44,7 @@ services: oap: condition: service_healthy healthcheck: - test: ["CMD", "sh", "-c", "nc -z 127.0.0.1 8080"] + test: ["CMD", "sh", "-c", "nc -z 127.0.0.1 8080 || (echo '[HEALTHCHECK] nc command failed' >&2; which nc >&2 || echo '[HEALTHCHECK] nc not found' >&2; netstat -tuln 2>&1 | grep 8080 || echo '[HEALTHCHECK] Port 8080 not listening' >&2; exit 1)"] interval: 5s timeout: 60s retries: 120 From 6a2dd3a1575640c6c562248dbd79ebdb04a75d7d Mon Sep 17 00:00:00 2001 From: JophieQu Date: Thu, 16 Oct 2025 23:12:32 +0800 Subject: [PATCH 41/69] test --- .github/workflows/skywalking.yaml | 7 +++++-- .../core/profiling/pprof/storage/PprofTaskRecord.java | 1 + test/e2e-v2/cases/go/docker-compose.yml | 6 ------ test/e2e-v2/cases/go/service/Dockerfile | 2 -- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index 5d23f7662a73..10dbc1e1e0d1 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -414,11 +414,14 @@ jobs: docker ps -a echo "=== Container Logs ===" - docker-compose -f ${{ matrix.test.config }} logs --tail=100 + for container in $(docker ps -a --format '{{.Names}}'); do + echo "--- Logs for $container ---" + docker logs --tail=100 $container 2>&1 || true + done echo "=== Health Check Details ===" for container in $(docker ps -a -q); do - docker inspect $container | jq '.[0].State.Health' + docker inspect $container | jq '.[0].State.Health' 2>&1 || true done - uses: actions/upload-artifact@v4 if: ${{ failure() }} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java index e00090d7fa05..cf97a5c971c4 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java @@ -45,6 +45,7 @@ @ScopeDeclaration(id = PPROF_TASK, name = "PprofTask") @Stream(name = PprofTaskRecord.INDEX_NAME, scopeId = PPROF_TASK, builder = PprofTaskRecord.Builder.class, processor = NoneStreamProcessor.class) @BanyanDB.TimestampColumn(PprofTaskRecord.CREATE_TIME) +@BanyanDB.Group(streamGroup = BanyanDB.StreamGroup.RECORDS) public class PprofTaskRecord extends NoneStream { private static final Gson GSON = new Gson(); diff --git a/test/e2e-v2/cases/go/docker-compose.yml b/test/e2e-v2/cases/go/docker-compose.yml index 83ee6e3ff37c..b4b9c894e47c 100644 --- a/test/e2e-v2/cases/go/docker-compose.yml +++ b/test/e2e-v2/cases/go/docker-compose.yml @@ -20,13 +20,8 @@ services: extends: file: ../../script/docker-compose/base-compose.yml service: oap - environment: - SW_STORAGE: banyandb ports: - 12800 - depends_on: - banyandb: - condition: service_healthy banyandb: extends: @@ -55,7 +50,6 @@ services: - 8080 environment: SW_AGENT_NAME: go-service - SW_AGENT_INSTANCE_NAME: provider1 SW_AGENT_REPORTER_GRPC_BACKEND_SERVICE: oap:11800 UPSTREAM_URL: http://provider:9090/correlation depends_on: diff --git a/test/e2e-v2/cases/go/service/Dockerfile b/test/e2e-v2/cases/go/service/Dockerfile index e76b93ffb26a..63c49d17cde6 100644 --- a/test/e2e-v2/cases/go/service/Dockerfile +++ b/test/e2e-v2/cases/go/service/Dockerfile @@ -26,8 +26,6 @@ RUN go mod tidy && go build -toolexec="skywalking-go-agent" -a -o service FROM alpine:3.10 -RUN apk add --no-cache busybox-extras - COPY --from=base /go-service/service /service ENTRYPOINT ["/service"] \ No newline at end of file From 050a33901357dcf5f4bbfc3fcd9ddebede684d07 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Thu, 16 Oct 2025 23:27:55 +0800 Subject: [PATCH 42/69] Fix --- .../profiling/pprof/storage/PprofProfilingDataDispatcher.java | 2 +- .../core/profiling/pprof/storage/PprofProfilingDataRecord.java | 1 + .../server/core/profiling/pprof/storage/PprofTaskLogRecord.java | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java index 21df992f1d54..c35e197e9f18 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java @@ -16,7 +16,7 @@ * */ - package org.apache.skywalking.oap.server.core.profiling.pprof.storage; +package org.apache.skywalking.oap.server.core.profiling.pprof.storage; import com.google.gson.Gson; import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java index 71492a1bf5ee..fd091c51b5c0 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java @@ -36,6 +36,7 @@ @Stream(name = PprofProfilingDataRecord.INDEX_NAME, scopeId = PPROF_PROFILING_DATA, builder = PprofProfilingDataRecord.Builder.class, processor = RecordStreamProcessor.class) @BanyanDB.TimestampColumn(PprofProfilingDataRecord.UPLOAD_TIME) +@BanyanDB.Group(streamGroup = BanyanDB.StreamGroup.RECORDS) public class PprofProfilingDataRecord extends Record { public static final String INDEX_NAME = "pprof_profiling_data"; public static final String TASK_ID = "task_id"; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java index f3ee5ec7338e..f26496c84f2b 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java @@ -40,6 +40,7 @@ @ScopeDeclaration(id = PPROF_TASK_LOG, name = "PprofTaskLog") @Stream(name = PprofTaskLogRecord.INDEX_NAME, scopeId = PPROF_TASK_LOG, builder = PprofTaskLogRecord.Builder.class, processor = RecordStreamProcessor.class) @BanyanDB.TimestampColumn(PprofTaskLogRecord.TIMESTAMP) +@BanyanDB.Group(streamGroup = BanyanDB.StreamGroup.RECORDS) public class PprofTaskLogRecord extends Record { public static final String INDEX_NAME = "pprof_task_log"; public static final String TASK_ID = "task_id"; From 602917d4b3a0b3eeb7b8e2ba6af13ecf4fee4b90 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Fri, 17 Oct 2025 00:14:45 +0800 Subject: [PATCH 43/69] fix go agent e2e --- .github/workflows/skywalking.yaml | 391 +++++++++++++++++++++++- test/e2e-v2/cases/go/docker-compose.yml | 2 + test/e2e-v2/cases/go/service/Dockerfile | 4 +- test/e2e-v2/cases/go/service/go.mod | 1 + 4 files changed, 382 insertions(+), 16 deletions(-) diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index 10dbc1e1e0d1..a953ec90f9af 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -345,8 +345,381 @@ jobs: fail-fast: false matrix: test: + - name: Cluster ZK/ES + config: test/e2e-v2/cases/cluster/zk/es/e2e.yaml + + - name: Agent NodeJS Backend + config: test/e2e-v2/cases/nodejs/e2e.yaml - name: Agent Golang - config: test/e2e-v2/cases/go/docker-compose.yml + config: test/e2e-v2/cases/go/e2e.yaml + - name: Agent NodeJS Frontend + config: test/e2e-v2/cases/browser/e2e.yaml + - name: Agent NodeJS Frontend ES + config: test/e2e-v2/cases/browser/es/e2e.yaml + - name: Agent NodeJS Frontend ES Sharding + config: test/e2e-v2/cases/browser/es/es-sharding/e2e.yaml + - name: Agent PHP + config: test/e2e-v2/cases/php/e2e.yaml + - name: Agent Python + config: test/e2e-v2/cases/python/e2e.yaml + - name: Agent Lua + config: test/e2e-v2/cases/lua/e2e.yaml + + - name: BanyanDB + config: test/e2e-v2/cases/storage/banyandb/e2e.yaml + - name: BanyanDB TLS + config: test/e2e-v2/cases/storage/banyandb/tls/e2e.yaml + - name: Storage MySQL + config: test/e2e-v2/cases/storage/mysql/e2e.yaml + - name: Storage PostgreSQL + config: test/e2e-v2/cases/storage/postgres/e2e.yaml + - name: Storage ES 7.16.3 + config: test/e2e-v2/cases/storage/es/e2e.yaml + env: ES_VERSION=7.16.3 + - name: Storage ES 7.17.10 + config: test/e2e-v2/cases/storage/es/e2e.yaml + env: ES_VERSION=7.17.10 + - name: Storage ES 8.1.0 + config: test/e2e-v2/cases/storage/es/e2e.yaml + env: ES_VERSION=8.1.0 + - name: Storage ES 8.9.0 + config: test/e2e-v2/cases/storage/es/e2e.yaml + env: ES_VERSION=8.9.0 + - name: Storage ES 8.9.0 + config: test/e2e-v2/cases/storage/es/e2e.yaml + env: ES_VERSION=8.18.1 + - name: Storage OpenSearch 1.1.0 + config: test/e2e-v2/cases/storage/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=1.1.0 + - name: Storage OpenSearch 1.3.10 + config: test/e2e-v2/cases/storage/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=1.3.10 + - name: Storage OpenSearch 2.4.0 + config: test/e2e-v2/cases/storage/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=2.4.0 + - name: Storage OpenSearch 2.8.0 + config: test/e2e-v2/cases/storage/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=2.8.0 + - name: Storage OpenSearch 3.0.0 + config: test/e2e-v2/cases/storage/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=3.0.0 + - name: Storage ES Sharding + config: test/e2e-v2/cases/storage/es/es-sharding/e2e.yaml + + - name: Alarm ES + config: test/e2e-v2/cases/alarm/es/e2e.yaml + - name: Alarm ES Sharding + config: test/e2e-v2/cases/alarm/es/es-sharding/e2e.yaml + - name: Alarm MySQL + config: test/e2e-v2/cases/alarm/mysql/e2e.yaml + - name: Alarm PostgreSQL + config: test/e2e-v2/cases/alarm/postgres/e2e.yaml + - name: Alarm BanyanDB + config: test/e2e-v2/cases/alarm/banyandb/e2e.yaml + + - name: Baseline-driven Alarm ES + config: test/e2e-v2/cases/baseline/es/e2e.yaml + - name: Baseline-driven Alarm ES Sharding + config: test/e2e-v2/cases/baseline/es/es-sharding/e2e.yaml + - name: Baseline-driven Alarm BanyanDB + config: test/e2e-v2/cases/baseline/banyandb/e2e.yaml + + - name: TTL ES 7.16.3 + config: test/e2e-v2/cases/ttl/es/e2e.yaml + env: ES_VERSION=7.16.3 + - name: TTL ES 8.8.1 + config: test/e2e-v2/cases/ttl/es/e2e.yaml + env: ES_VERSION=8.8.1 + - name: TTL ES 8.18.1 + config: test/e2e-v2/cases/ttl/es/e2e.yaml + env: ES_VERSION=8.18.1 + + - name: Event BanyanDB + config: test/e2e-v2/cases/event/banyandb/e2e.yaml + - name: Event ES + config: test/e2e-v2/cases/event/es/e2e.yaml + - name: Event MySQL + config: test/e2e-v2/cases/event/mysql/e2e.yaml + + - name: Log MySQL + config: test/e2e-v2/cases/log/mysql/e2e.yaml + - name: Log PostgreSQL + config: test/e2e-v2/cases/log/postgres/e2e.yaml + - name: Log ES 7.16.3 + config: test/e2e-v2/cases/log/es/e2e.yaml + env: ES_VERSION=7.16.3 + - name: Log ES 7.17.10 + config: test/e2e-v2/cases/log/es/e2e.yaml + env: ES_VERSION=7.17.10 + - name: Log ES 8.8.1 Sharding + config: test/e2e-v2/cases/log/es/es-sharding/e2e.yaml + env: ES_VERSION=8.8.1 + - name: Log ES 8.18.1 Sharding + config: test/e2e-v2/cases/log/es/es-sharding/e2e.yaml + env: ES_VERSION=8.18.1 + - name: Log BanyanDB + config: test/e2e-v2/cases/log/banyandb/e2e.yaml + + - name: Log FluentBit ES 7.16.3 + config: test/e2e-v2/cases/log/fluent-bit/e2e.yaml + env: ES_VERSION=7.16.3 + - name: Log FluentBit ES 7.17.10 + config: test/e2e-v2/cases/log/fluent-bit/e2e.yaml + env: ES_VERSION=7.17.10 + - name: Log FluentBit ES 8.8.1 + config: test/e2e-v2/cases/log/fluent-bit/e2e.yaml + env: ES_VERSION=8.8.1 + - name: Log FluentBit ES 8.18.1 + config: test/e2e-v2/cases/log/fluent-bit/e2e.yaml + env: ES_VERSION=8.18.1 + + - name: Trace Profiling BanyanDB + config: test/e2e-v2/cases/profiling/trace/banyandb/e2e.yaml + - name: Trace Profiling ES + config: test/e2e-v2/cases/profiling/trace/es/e2e.yaml + - name: Trace Profiling ES Sharding + config: test/e2e-v2/cases/profiling/trace/es/es-sharding/e2e.yaml + - name: Trace Profiling MySQL + config: test/e2e-v2/cases/profiling/trace/mysql/e2e.yaml + - name: Trace Profiling Postgres + config: test/e2e-v2/cases/profiling/trace/postgres/e2e.yaml + - name: Trace Profiling OpenSearch 1.1.0 + config: test/e2e-v2/cases/profiling/trace/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=1.1.0 + - name: Trace Profiling OpenSearch 1.3.6 + config: test/e2e-v2/cases/profiling/trace/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=1.3.6 + - name: Trace Profiling OpenSearch 2.4.0 + config: test/e2e-v2/cases/profiling/trace/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=2.4.0 + + - name: eBPF Profiling On CPU BanyanDB + config: test/e2e-v2/cases/profiling/ebpf/oncpu/banyandb/e2e.yaml + docker: + base: test/e2e-v2/cases/profiling/ebpf/oncpu/ + file: Dockerfile.sqrt + name: test/oncpu:test + - name: eBPF Profiling On CPU ES + config: test/e2e-v2/cases/profiling/ebpf/oncpu/es/e2e.yaml + docker: + base: test/e2e-v2/cases/profiling/ebpf/oncpu/ + file: Dockerfile.sqrt + name: test/oncpu:test + - name: eBPF Profiling On CPU ES Sharding + config: test/e2e-v2/cases/profiling/ebpf/oncpu/es/es-sharding/e2e.yaml + docker: + base: test/e2e-v2/cases/profiling/ebpf/oncpu/ + file: Dockerfile.sqrt + name: test/oncpu:test + - name: eBPF Profiling Off CPU + config: test/e2e-v2/cases/profiling/ebpf/offcpu/e2e.yaml + docker: + base: test/e2e-v2/cases/profiling/ebpf/offcpu/ + file: Dockerfile.file + name: test/offcpu:test + runs-on: ubuntu-24.04 + + - name: eBPF Profiling Network BanyanDB + config: test/e2e-v2/cases/profiling/ebpf/network/banyandb/e2e.yaml + docker: + base: test/e2e-v2/cases/profiling/ebpf/network/ + file: Dockerfile.service + name: test/network:test + - name: eBPF Profiling Network ES + config: test/e2e-v2/cases/profiling/ebpf/network/es/e2e.yaml + docker: + base: test/e2e-v2/cases/profiling/ebpf/network/ + file: Dockerfile.service + name: test/network:test + - name: eBPF Profiling Network ES Sharding + config: test/e2e-v2/cases/profiling/ebpf/network/es/es-sharding/e2e.yaml + docker: + base: test/e2e-v2/cases/profiling/ebpf/network/ + file: Dockerfile.service + name: test/network:test + + - name: Continuous Profiling BanyanDB + config: test/e2e-v2/cases/profiling/ebpf/continuous/banyandb/e2e.yaml + docker: + base: test/e2e-v2/cases/profiling/ebpf/continuous/ + file: Dockerfile.sqrt + name: test/continuous:test + - name: Continuous Profiling ES + config: test/e2e-v2/cases/profiling/ebpf/continuous/es/e2e.yaml + docker: + base: test/e2e-v2/cases/profiling/ebpf/continuous/ + file: Dockerfile.sqrt + name: test/continuous:test + - name: Continuous Profiling Sharding ES + config: test/e2e-v2/cases/profiling/ebpf/continuous/es/es-sharding/e2e.yaml + docker: + base: test/e2e-v2/cases/profiling/ebpf/continuous/ + file: Dockerfile.sqrt + name: test/continuous:test + + # eBPF Access Log + - name: eBPF Access Log BanyanDB + config: test/e2e-v2/cases/profiling/ebpf/access_log/banyandb/e2e.yaml + - name: eBPF Access Log ES + config: test/e2e-v2/cases/profiling/ebpf/access_log/es/e2e.yaml + - name: eBPF Access Log ES Sharding + config: test/e2e-v2/cases/profiling/ebpf/access_log/es/es-sharding/e2e.yaml + + - name: Kafka Basic + config: test/e2e-v2/cases/kafka/simple-so11y/e2e.yaml + - name: Kafka Profiling + config: test/e2e-v2/cases/kafka/profile/e2e.yaml + - name: Kafka Meter + config: test/e2e-v2/cases/kafka/meter/e2e.yaml + - name: Kafka Log + config: test/e2e-v2/cases/kafka/log/e2e.yaml + + - name: Istio Metrics Service 1.20.0 + config: test/e2e-v2/cases/istio/metrics/e2e.yaml + env: | + ISTIO_VERSION=1.20.0 + KUBERNETES_VERSION=28 + - name: Istio Metrics Service 1.21.0 + config: test/e2e-v2/cases/istio/metrics/e2e.yaml + env: | + ISTIO_VERSION=1.21.0 + KUBERNETES_VERSION=28 + - name: Istio Metrics Service 1.22.0 + config: test/e2e-v2/cases/istio/metrics/e2e.yaml + env: | + ISTIO_VERSION=1.22.0 + KUBERNETES_VERSION=28 + - name: Istio Metrics Service 1.23.0 + config: test/e2e-v2/cases/istio/metrics/e2e.yaml + env: | + ISTIO_VERSION=1.23.0 + KUBERNETES_VERSION=28 + - name: Istio Metrics Service 1.24.0 + config: test/e2e-v2/cases/istio/metrics/e2e.yaml + env: | + ISTIO_VERSION=1.24.0 + KUBERNETES_VERSION=28 + + - name: Rover with Istio Process 1.15.0 + config: test/e2e-v2/cases/rover/process/istio/e2e.yaml + env: ISTIO_VERSION=1.15.0 + runs-on: ubuntu-24.04 + + - name: Satellite + config: test/e2e-v2/cases/satellite/native-protocols/e2e.yaml + - name: Auth + config: test/e2e-v2/cases/simple/auth/e2e.yaml + - name: SSL + config: test/e2e-v2/cases/simple/ssl/e2e.yaml + - name: mTLS + config: test/e2e-v2/cases/simple/mtls/e2e.yaml + - name: Virtual Gateway + config: test/e2e-v2/cases/gateway/e2e.yaml + - name: Meter + config: test/e2e-v2/cases/meter/e2e.yaml + - name: VM Zabbix + config: test/e2e-v2/cases/vm/zabbix/e2e.yaml + - name: VM Prometheus + config: test/e2e-v2/cases/vm/prometheus-node-exporter/e2e.yaml + - name: VM Telegraf + config: test/e2e-v2/cases/vm/telegraf/e2e.yaml + - name: So11y + config: test/e2e-v2/cases/so11y/e2e.yaml + - name: MySQL Prometheus and slowsql + config: test/e2e-v2/cases/mysql/mysql-slowsql/e2e.yaml + - name: PostgreSQL Prometheus + config: test/e2e-v2/cases/postgresql/postgres-exporter/e2e.yaml + - name: MariaDB Prometheus and slowsql + config: test/e2e-v2/cases/mariadb/mariadb-slowsql/e2e.yaml + + - name: Zipkin ES + config: test/e2e-v2/cases/zipkin/es/e2e.yaml + - name: Zipkin ES Sharding + config: test/e2e-v2/cases/zipkin/es/es-sharding/e2e.yaml + - name: Zipkin MySQL + config: test/e2e-v2/cases/zipkin/mysql/e2e.yaml + - name: Zipkin Opensearch + config: test/e2e-v2/cases/zipkin/opensearch/e2e.yaml + - name: Zipkin Postgres + config: test/e2e-v2/cases/zipkin/postgres/e2e.yaml + - name: Zipkin Kafka + config: test/e2e-v2/cases/zipkin/kafka/e2e.yaml + - name: Zipkin BanyanDB + config: test/e2e-v2/cases/zipkin/banyandb/e2e.yaml + + - name: Nginx + config: test/e2e-v2/cases/nginx/e2e.yaml + - name: APISIX metrics + config: test/e2e-v2/cases/apisix/otel-collector/e2e.yaml + - name: Exporter Kafka + config: test/e2e-v2/cases/exporter/kafka/e2e.yaml + - name: Virtual MQ + config: test/e2e-v2/cases/virtual-mq/e2e.yaml + - name: AWS Cloud EKS + config: test/e2e-v2/cases/aws/eks/e2e.yaml + - name: Windows + config: test/e2e-v2/cases/win/e2e.yaml + - name: AWS Cloud S3 + config: test/e2e-v2/cases/aws/s3/e2e.yaml + - name: AWS Cloud DynamoDB + config: test/e2e-v2/cases/aws/dynamodb/e2e.yaml + - name: PromQL Service + config: test/e2e-v2/cases/promql/e2e.yaml + - name: LogQL Service + config: test/e2e-v2/cases/logql/e2e.yaml + - name: AWS API Gateway + config: test/e2e-v2/cases/aws/api-gateway/e2e.yaml + - name: Redis Prometheus and Log Collecting + config: test/e2e-v2/cases/redis/redis-exporter/e2e.yaml + - name: Elasticsearch + config: test/e2e-v2/cases/elasticsearch/e2e.yaml + - name: MongoDB + config: test/e2e-v2/cases/mongodb/e2e.yaml + - name: RabbitMQ + config: test/e2e-v2/cases/rabbitmq/e2e.yaml + - name: Kafka + config: test/e2e-v2/cases/kafka/kafka-monitoring/e2e.yaml + - name: MQE Service + config: test/e2e-v2/cases/mqe/e2e.yaml + - name: Pulsar and BookKeeper + config: test/e2e-v2/cases/pulsar/e2e.yaml + - name: RocketMQ + config: test/e2e-v2/cases/rocketmq/e2e.yaml + - name: ClickHouse + config: test/e2e-v2/cases/clickhouse/clickhouse-prometheus-endpoint/e2e.yaml + - name: ActiveMQ + config: test/e2e-v2/cases/activemq/e2e.yaml + - name: Kong + config: test/e2e-v2/cases/kong/e2e.yaml + - name: Flink + config: test/e2e-v2/cases/flink/e2e.yaml + + - name: UI Menu BanyanDB + config: test/e2e-v2/cases/menu/banyandb/e2e.yaml + - name: UI Menu ES + config: test/e2e-v2/cases/menu/es/e2e.yaml + - name: UI Menu Sharding ES + config: test/e2e-v2/cases/menu/es/es-sharding/e2e.yaml + - name: UI Menu MySQL + config: test/e2e-v2/cases/menu/mysql/e2e.yaml + - name: UI Menu Postgres + config: test/e2e-v2/cases/menu/postgres/e2e.yaml + - name: UI Menu OpenSearch 1.1.0 + config: test/e2e-v2/cases/menu/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=1.1.0 + - name: UI Menu OpenSearch 1.3.6 + config: test/e2e-v2/cases/menu/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=1.3.6 + - name: UI Menu OpenSearch 2.4.0 + config: test/e2e-v2/cases/menu/opensearch/e2e.yaml + env: OPENSEARCH_VERSION=2.4.0 + + - name: OTLP Trace + config: test/e2e-v2/cases/otlp-traces/e2e.yaml + + - name: Cilium Service + config: test/e2e-v2/cases/cilium/e2e.yaml + - name: Async Profiler ES config: test/e2e-v2/cases/profiling/async-profiler/es/e2e.yaml - name: Async Profiler BanyanDB @@ -410,19 +783,9 @@ jobs: e2e-file: $GITHUB_WORKSPACE/${{ matrix.test.config }} - if: ${{ failure() }} run: | - echo "=== Container Status ===" - docker ps -a - - echo "=== Container Logs ===" - for container in $(docker ps -a --format '{{.Names}}'); do - echo "--- Logs for $container ---" - docker logs --tail=100 $container 2>&1 || true - done - - echo "=== Health Check Details ===" - for container in $(docker ps -a -q); do - docker inspect $container | jq '.[0].State.Health' 2>&1 || true - done + df -h + du -sh . + docker images - uses: actions/upload-artifact@v4 if: ${{ failure() }} name: Upload Logs diff --git a/test/e2e-v2/cases/go/docker-compose.yml b/test/e2e-v2/cases/go/docker-compose.yml index b4b9c894e47c..8dc12b67604d 100644 --- a/test/e2e-v2/cases/go/docker-compose.yml +++ b/test/e2e-v2/cases/go/docker-compose.yml @@ -55,6 +55,8 @@ services: depends_on: oap: condition: service_healthy + provider: + condition: service_healthy healthcheck: test: ["CMD", "sh", "-c", "nc -z 127.0.0.1 8080"] interval: 5s diff --git a/test/e2e-v2/cases/go/service/Dockerfile b/test/e2e-v2/cases/go/service/Dockerfile index 63c49d17cde6..79dc8b1a8366 100644 --- a/test/e2e-v2/cases/go/service/Dockerfile +++ b/test/e2e-v2/cases/go/service/Dockerfile @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG SW_AGENT_GO_COMMIT -FROM ghcr.io/apache/skywalking-go/skywalking-go:${SW_AGENT_GO_COMMIT}-go1.19 AS base +ARG SW_AGENT_GO_COMMIT= +FROM ghcr.io/apache/skywalking-go/skywalking-go:${SW_AGENT_GO_COMMIT}-go1.19 as base ENV CGO_ENABLED=0 ENV GO111MODULE=on diff --git a/test/e2e-v2/cases/go/service/go.mod b/test/e2e-v2/cases/go/service/go.mod index bc338fccb1f4..6b5f043ed2a8 100644 --- a/test/e2e-v2/cases/go/service/go.mod +++ b/test/e2e-v2/cases/go/service/go.mod @@ -61,4 +61,5 @@ require ( google.golang.org/grpc v1.55.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + skywalking.apache.org/repo/goapi v0.0.0-20230314034821-0c5a44bb767a // indirect ) From 32f384d93a597b713f9f4db36237e0e9d6301736 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Fri, 17 Oct 2025 10:17:17 +0800 Subject: [PATCH 44/69] fix --- .github/workflows/skywalking.yaml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index a953ec90f9af..39d98b0a199c 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -35,7 +35,7 @@ env: jobs: license-header: - if: (github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || (github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || (github.event_name != 'schedule') name: License header runs-on: ubuntu-latest timeout-minutes: 10 @@ -48,7 +48,7 @@ jobs: uses: apache/skywalking-eyes@5b7ee1731d036b5aac68f8bd3fc9e6f98ada082e code-style: - if: (github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || (github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || (github.event_name != 'schedule') name: Code style runs-on: ubuntu-latest timeout-minutes: 10 @@ -63,7 +63,7 @@ jobs: dependency-license: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.pom == 'true' || needs.changes.outputs.ui == 'true') + ((github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || needs.changes.outputs.pom == 'true' || needs.changes.outputs.ui == 'true') name: Dependency licenses needs: [changes] runs-on: ubuntu-latest @@ -93,7 +93,7 @@ jobs: fi sanity-check: - if: ( always() && ! cancelled() ) && (github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || (github.event_name != 'schedule') + if: ( always() && ! cancelled() ) && (github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || (github.event_name != 'schedule') name: Sanity check results needs: [license-header, code-style, dependency-license] runs-on: ubuntu-latest @@ -162,7 +162,7 @@ jobs: dist-tar: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || needs.changes.outputs.oap == 'true') name: Build dist tar needs: [changes] runs-on: ubuntu-latest @@ -194,7 +194,7 @@ jobs: docker: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || needs.changes.outputs.oap == 'true') name: Docker images needs: [sanity-check, dist-tar, changes] runs-on: ubuntu-latest @@ -233,7 +233,7 @@ jobs: unit-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || needs.changes.outputs.oap == 'true') name: Unit test needs: [sanity-check, changes] runs-on: ${{ matrix.os }} @@ -268,7 +268,7 @@ jobs: integration-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || needs.changes.outputs.oap == 'true') name: Integration test needs: [sanity-check, changes] runs-on: ubuntu-latest @@ -301,7 +301,7 @@ jobs: slow-integration-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || needs.changes.outputs.oap == 'true') name: Slow Integration Tests needs: [sanity-check, changes] runs-on: ubuntu-latest @@ -334,7 +334,7 @@ jobs: e2e-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker, dist-tar] runs-on: ${{ matrix.test.runs-on || 'ubuntu-latest' }} @@ -796,7 +796,7 @@ jobs: e2e-test-istio: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-24.04 @@ -864,7 +864,7 @@ jobs: e2e-test-istio-ambient: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-24.04 @@ -925,7 +925,7 @@ jobs: e2e-test-java-versions: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-latest From 296ed0fb45ef73d180d63647c4e98f2a9321cafd Mon Sep 17 00:00:00 2001 From: JophieQu Date: Fri, 17 Oct 2025 14:19:00 +0800 Subject: [PATCH 45/69] fix e2e, roll back LICENSE --- .github/workflows/skywalking.yaml | 20 +++---- dist-material/release-docs/LICENSE | 58 +++++++++---------- .../pprof/banyandb/docker-compose.yml | 2 +- 3 files changed, 38 insertions(+), 42 deletions(-) diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index 39d98b0a199c..7290e589b5c8 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -35,7 +35,7 @@ env: jobs: license-header: - if: (github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || (github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') name: License header runs-on: ubuntu-latest timeout-minutes: 10 @@ -48,7 +48,7 @@ jobs: uses: apache/skywalking-eyes@5b7ee1731d036b5aac68f8bd3fc9e6f98ada082e code-style: - if: (github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || (github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') name: Code style runs-on: ubuntu-latest timeout-minutes: 10 @@ -63,7 +63,7 @@ jobs: dependency-license: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || needs.changes.outputs.pom == 'true' || needs.changes.outputs.ui == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.pom == 'true' || needs.changes.outputs.ui == 'true') name: Dependency licenses needs: [changes] runs-on: ubuntu-latest @@ -93,7 +93,7 @@ jobs: fi sanity-check: - if: ( always() && ! cancelled() ) && (github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || (github.event_name != 'schedule') + if: ( always() && ! cancelled() ) && (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') name: Sanity check results needs: [license-header, code-style, dependency-license] runs-on: ubuntu-latest @@ -162,7 +162,7 @@ jobs: dist-tar: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: Build dist tar needs: [changes] runs-on: ubuntu-latest @@ -194,7 +194,7 @@ jobs: docker: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: Docker images needs: [sanity-check, dist-tar, changes] runs-on: ubuntu-latest @@ -233,7 +233,7 @@ jobs: unit-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: Unit test needs: [sanity-check, changes] runs-on: ${{ matrix.os }} @@ -268,7 +268,7 @@ jobs: integration-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: Integration test needs: [sanity-check, changes] runs-on: ubuntu-latest @@ -301,7 +301,7 @@ jobs: slow-integration-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: Slow Integration Tests needs: [sanity-check, changes] runs-on: ubuntu-latest @@ -334,7 +334,7 @@ jobs: e2e-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker, dist-tar] runs-on: ${{ matrix.test.runs-on || 'ubuntu-latest' }} diff --git a/dist-material/release-docs/LICENSE b/dist-material/release-docs/LICENSE index df015b932865..c8c8b008d32f 100644 --- a/dist-material/release-docs/LICENSE +++ b/dist-material/release-docs/LICENSE @@ -208,8 +208,8 @@ Apache-2.0 licenses ======================================================================== The following components are provided under the Apache-2.0 License. See project link for details. The text of each license is the standard Apache 2.0 license. - https://mvnrepository.com/artifact/build.buf.protoc-gen-validate/pgv-java-stub/1.2.1 Apache-2.0 - https://mvnrepository.com/artifact/build.buf.protoc-gen-validate/protoc-gen-validate/1.2.1 Apache-2.0 + https://mvnrepository.com/artifact/build.buf.protoc-gen-validate/pgv-java-stub/0.6.13 Apache-2.0 + https://mvnrepository.com/artifact/build.buf.protoc-gen-validate/protoc-gen-validate/0.6.13 Apache-2.0 https://mvnrepository.com/artifact/com.aayushatharva.brotli4j/brotli4j/1.18.0 Apache-2.0 https://mvnrepository.com/artifact/com.aayushatharva.brotli4j/service/1.18.0 Apache-2.0 https://mvnrepository.com/artifact/com.alibaba.nacos/nacos-auth-plugin/2.3.2 Apache-2.0 @@ -254,7 +254,7 @@ The text of each license is the standard Apache 2.0 license. https://mvnrepository.com/artifact/commons-codec/commons-codec/1.11 Apache-2.0 https://mvnrepository.com/artifact/commons-io/commons-io/2.17.0 Apache-2.0 https://mvnrepository.com/artifact/commons-net/commons-net/3.9.0 Apache-2.0 - https://mvnrepository.com/artifact/commons-validator/commons-validator/1.9.0 Apache-2.0 + https://mvnrepository.com/artifact/commons-validator/commons-validator/1.7 Apache-2.0 https://npmjs.com/package/d3-flame-graph/v/4.1.3 4.1.3 Apache-2.0 https://npmjs.com/package/echarts/v/5.4.1 5.4.1 Apache-2.0 https://mvnrepository.com/artifact/io.etcd/jetcd-api/0.6.1 Apache-2.0 @@ -303,33 +303,29 @@ The text of each license is the standard Apache 2.0 license. https://mvnrepository.com/artifact/io.micrometer/micrometer-commons/1.14.4 Apache-2.0 https://mvnrepository.com/artifact/io.micrometer/micrometer-core/1.14.4 Apache-2.0 https://mvnrepository.com/artifact/io.micrometer/micrometer-observation/1.14.4 Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-buffer/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-codec/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-codec-base/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-codec-compression/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-codec-dns/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-codec-haproxy/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-codec-http/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-codec-http2/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-codec-marshalling/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-codec-protobuf/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-codec-socks/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-common/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-handler/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-handler-proxy/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-resolver/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-resolver-dns/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-resolver-dns-classes-macos/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-resolver-dns-native-macos/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-buffer/4.1.118.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-codec/4.1.118.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-codec-dns/4.1.118.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-codec-haproxy/4.1.118.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-codec-http/4.1.118.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-codec-http2/4.1.118.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-codec-socks/4.1.118.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-common/4.1.118.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-handler/4.1.118.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-handler-proxy/4.1.118.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-resolver/4.1.118.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-resolver-dns/4.1.118.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-resolver-dns-classes-macos/4.1.118.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-resolver-dns-native-macos/4.1.118.Final Apache-2.0 https://mvnrepository.com/artifact/io.netty/netty-tcnative-boringssl-static/2.0.69.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-tcnative-boringssl-static/2.0.73.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-tcnative-classes/2.0.73.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-transport/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-transport-classes-epoll/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-transport-classes-kqueue/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-transport-native-epoll/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-transport-native-kqueue/4.2.5.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-transport-native-unix-common/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-tcnative-boringssl-static/2.0.70.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-tcnative-classes/2.0.70.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-transport/4.1.118.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-transport-classes-epoll/4.1.118.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-transport-classes-kqueue/4.1.118.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-transport-native-epoll/4.1.118.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-transport-native-kqueue/4.1.118.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-transport-native-unix-common/4.1.118.Final Apache-2.0 https://mvnrepository.com/artifact/io.perfmark/perfmark-api/0.27.0 Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient/0.6.0 Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient_common/0.6.0 Apache-2.0 @@ -341,7 +337,7 @@ The text of each license is the standard Apache 2.0 license. https://mvnrepository.com/artifact/javax.inject/javax.inject/1 Apache-2.0 https://mvnrepository.com/artifact/joda-time/joda-time/2.10.5 Apache-2.0 https://mvnrepository.com/artifact/net.jodah/failsafe/2.4.4 Apache-2.0 - https://mvnrepository.com/artifact/org.apache.commons/commons-lang3/3.18.0 Apache-2.0 + https://mvnrepository.com/artifact/org.apache.commons/commons-lang3/3.12.0 Apache-2.0 https://mvnrepository.com/artifact/org.apache.commons/commons-text/1.4 Apache-2.0 https://mvnrepository.com/artifact/org.apache.curator/curator-client/4.3.0 Apache-2.0 https://mvnrepository.com/artifact/org.apache.curator/curator-framework/4.3.0 Apache-2.0 @@ -592,7 +588,7 @@ https://golang.org/LICENSE licenses The following components are provided under the https://golang.org/LICENSE License. See project link for details. The text of each license is also included in licenses/LICENSE-[project].txt. - https://mvnrepository.com/artifact/com.google.re2j/re2j/1.7 https://golang.org/LICENSE + https://mvnrepository.com/artifact/com.google.re2j/re2j/1.5 https://golang.org/LICENSE ======================================================================== https://opensource.org/licenses/BSD-2-Clause;description=BSD 2-Clause License licenses diff --git a/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml index eba20b929672..f8964fe7a8d3 100644 --- a/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml @@ -44,7 +44,7 @@ services: oap: condition: service_healthy healthcheck: - test: ["CMD", "sh", "-c", "nc -z 127.0.0.1 8080 || (echo '[HEALTHCHECK] nc command failed' >&2; which nc >&2 || echo '[HEALTHCHECK] nc not found' >&2; netstat -tuln 2>&1 | grep 8080 || echo '[HEALTHCHECK] Port 8080 not listening' >&2; exit 1)"] + test: ["CMD", "sh", "-c", "nc -z 127.0.0.1 8080"] interval: 5s timeout: 60s retries: 120 From 19061828ef9fa3355d6d5d2d60e1832bdcce9ce8 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Fri, 17 Oct 2025 14:19:16 +0800 Subject: [PATCH 46/69] fix change.md --- docs/en/changes/changes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 212a059742a6..1c54269bdef5 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -49,11 +49,11 @@ * Add UI dashboard for Ruby runtime metrics. * Tracing Query Execution HTTP APIs: make the argument `service layer` optional. * GraphQL API: metadata, topology, log and trace support query by name. -* Support pprof profiling feature * [Break Change] MQE function `sort_values` sorts according to the aggregation result and labels rather than the simple time series values. * Self Observability: add `metrics_aggregation_queue_used_percentage` and `metrics_persistent_collection_cached_size` metrics for the OAP server. * Optimize metrics aggregate/persistent worker: separate `OAL` and `MAL` workers and consume pools. The dataflow signal drives the new MAL consumer, the following table shows the pool size，driven mode and queue size for each worker. +* Support pprof profiling feature. | Worker | poolSize | isSignalDrivenMode | queueChannelSize | queueBufferSize | |-------------------------------|------------------------------------------|--------------------|------------------|-----------------| From 7bc7d60d0788e1c645e9c48e9e37345e83c95195 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Fri, 17 Oct 2025 14:22:03 +0800 Subject: [PATCH 47/69] roll back ci --- .github/workflows/skywalking.yaml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index 7290e589b5c8..0bd11fca045c 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -17,9 +17,6 @@ name: CI on: - push: - branches: - - citest pull_request: schedule: - cron: "0 18 * * *" # TimeZone: UTC 0 @@ -796,7 +793,7 @@ jobs: e2e-test-istio: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-24.04 @@ -864,7 +861,7 @@ jobs: e2e-test-istio-ambient: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-24.04 @@ -925,7 +922,7 @@ jobs: e2e-test-java-versions: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'Apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-latest From 9e9eb9d281b102782e84b9cda02eaa8adaf504f1 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Fri, 17 Oct 2025 14:25:17 +0800 Subject: [PATCH 48/69] roll back LICENSE --- dist-material/release-docs/LICENSE | 58 ++++++++++++++++-------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/dist-material/release-docs/LICENSE b/dist-material/release-docs/LICENSE index c8c8b008d32f..df015b932865 100644 --- a/dist-material/release-docs/LICENSE +++ b/dist-material/release-docs/LICENSE @@ -208,8 +208,8 @@ Apache-2.0 licenses ======================================================================== The following components are provided under the Apache-2.0 License. See project link for details. The text of each license is the standard Apache 2.0 license. - https://mvnrepository.com/artifact/build.buf.protoc-gen-validate/pgv-java-stub/0.6.13 Apache-2.0 - https://mvnrepository.com/artifact/build.buf.protoc-gen-validate/protoc-gen-validate/0.6.13 Apache-2.0 + https://mvnrepository.com/artifact/build.buf.protoc-gen-validate/pgv-java-stub/1.2.1 Apache-2.0 + https://mvnrepository.com/artifact/build.buf.protoc-gen-validate/protoc-gen-validate/1.2.1 Apache-2.0 https://mvnrepository.com/artifact/com.aayushatharva.brotli4j/brotli4j/1.18.0 Apache-2.0 https://mvnrepository.com/artifact/com.aayushatharva.brotli4j/service/1.18.0 Apache-2.0 https://mvnrepository.com/artifact/com.alibaba.nacos/nacos-auth-plugin/2.3.2 Apache-2.0 @@ -254,7 +254,7 @@ The text of each license is the standard Apache 2.0 license. https://mvnrepository.com/artifact/commons-codec/commons-codec/1.11 Apache-2.0 https://mvnrepository.com/artifact/commons-io/commons-io/2.17.0 Apache-2.0 https://mvnrepository.com/artifact/commons-net/commons-net/3.9.0 Apache-2.0 - https://mvnrepository.com/artifact/commons-validator/commons-validator/1.7 Apache-2.0 + https://mvnrepository.com/artifact/commons-validator/commons-validator/1.9.0 Apache-2.0 https://npmjs.com/package/d3-flame-graph/v/4.1.3 4.1.3 Apache-2.0 https://npmjs.com/package/echarts/v/5.4.1 5.4.1 Apache-2.0 https://mvnrepository.com/artifact/io.etcd/jetcd-api/0.6.1 Apache-2.0 @@ -303,29 +303,33 @@ The text of each license is the standard Apache 2.0 license. https://mvnrepository.com/artifact/io.micrometer/micrometer-commons/1.14.4 Apache-2.0 https://mvnrepository.com/artifact/io.micrometer/micrometer-core/1.14.4 Apache-2.0 https://mvnrepository.com/artifact/io.micrometer/micrometer-observation/1.14.4 Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-buffer/4.1.118.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-codec/4.1.118.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-codec-dns/4.1.118.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-codec-haproxy/4.1.118.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-codec-http/4.1.118.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-codec-http2/4.1.118.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-codec-socks/4.1.118.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-common/4.1.118.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-handler/4.1.118.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-handler-proxy/4.1.118.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-resolver/4.1.118.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-resolver-dns/4.1.118.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-resolver-dns-classes-macos/4.1.118.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-resolver-dns-native-macos/4.1.118.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-buffer/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-codec/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-codec-base/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-codec-compression/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-codec-dns/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-codec-haproxy/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-codec-http/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-codec-http2/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-codec-marshalling/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-codec-protobuf/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-codec-socks/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-common/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-handler/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-handler-proxy/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-resolver/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-resolver-dns/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-resolver-dns-classes-macos/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-resolver-dns-native-macos/4.2.5.Final Apache-2.0 https://mvnrepository.com/artifact/io.netty/netty-tcnative-boringssl-static/2.0.69.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-tcnative-boringssl-static/2.0.70.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-tcnative-classes/2.0.70.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-transport/4.1.118.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-transport-classes-epoll/4.1.118.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-transport-classes-kqueue/4.1.118.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-transport-native-epoll/4.1.118.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-transport-native-kqueue/4.1.118.Final Apache-2.0 - https://mvnrepository.com/artifact/io.netty/netty-transport-native-unix-common/4.1.118.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-tcnative-boringssl-static/2.0.73.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-tcnative-classes/2.0.73.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-transport/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-transport-classes-epoll/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-transport-classes-kqueue/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-transport-native-epoll/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-transport-native-kqueue/4.2.5.Final Apache-2.0 + https://mvnrepository.com/artifact/io.netty/netty-transport-native-unix-common/4.2.5.Final Apache-2.0 https://mvnrepository.com/artifact/io.perfmark/perfmark-api/0.27.0 Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient/0.6.0 Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient_common/0.6.0 Apache-2.0 @@ -337,7 +341,7 @@ The text of each license is the standard Apache 2.0 license. https://mvnrepository.com/artifact/javax.inject/javax.inject/1 Apache-2.0 https://mvnrepository.com/artifact/joda-time/joda-time/2.10.5 Apache-2.0 https://mvnrepository.com/artifact/net.jodah/failsafe/2.4.4 Apache-2.0 - https://mvnrepository.com/artifact/org.apache.commons/commons-lang3/3.12.0 Apache-2.0 + https://mvnrepository.com/artifact/org.apache.commons/commons-lang3/3.18.0 Apache-2.0 https://mvnrepository.com/artifact/org.apache.commons/commons-text/1.4 Apache-2.0 https://mvnrepository.com/artifact/org.apache.curator/curator-client/4.3.0 Apache-2.0 https://mvnrepository.com/artifact/org.apache.curator/curator-framework/4.3.0 Apache-2.0 @@ -588,7 +592,7 @@ https://golang.org/LICENSE licenses The following components are provided under the https://golang.org/LICENSE License. See project link for details. The text of each license is also included in licenses/LICENSE-[project].txt. - https://mvnrepository.com/artifact/com.google.re2j/re2j/1.5 https://golang.org/LICENSE + https://mvnrepository.com/artifact/com.google.re2j/re2j/1.7 https://golang.org/LICENSE ======================================================================== https://opensource.org/licenses/BSD-2-Clause;description=BSD 2-Clause License licenses From 6a4ed7f4c6955ebe1f6558b06baa499d5b3fba72 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Fri, 17 Oct 2025 14:27:11 +0800 Subject: [PATCH 49/69] roll back --- dist-material/release-docs/LICENSE | 33 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/dist-material/release-docs/LICENSE b/dist-material/release-docs/LICENSE index df015b932865..8d264d2c8a98 100644 --- a/dist-material/release-docs/LICENSE +++ b/dist-material/release-docs/LICENSE @@ -477,21 +477,20 @@ The text of each license is also included in licenses/LICENSE-[project].txt. https://npmjs.com/package/@floating-ui/core/v/1.6.9 1.6.9 MIT https://npmjs.com/package/@floating-ui/dom/v/1.6.13 1.6.13 MIT https://npmjs.com/package/@floating-ui/utils/v/0.2.9 0.2.9 MIT - https://npmjs.com/package/@interactjs/actions/v/1.10.17 1.10.17 MIT - https://npmjs.com/package/@interactjs/auto-scroll/v/1.10.17 1.10.17 MIT - https://npmjs.com/package/@interactjs/auto-start/v/1.10.17 1.10.17 MIT - https://npmjs.com/package/@interactjs/core/v/1.10.17 1.10.17 MIT - https://npmjs.com/package/@interactjs/dev-tools/v/1.10.17 1.10.17 MIT - https://npmjs.com/package/@interactjs/inertia/v/1.10.17 1.10.17 MIT - https://npmjs.com/package/@interactjs/interact/v/1.10.17 1.10.17 MIT - https://npmjs.com/package/@interactjs/interactjs/v/1.10.17 1.10.17 MIT - https://npmjs.com/package/@interactjs/modifiers/v/1.10.17 1.10.17 MIT - https://npmjs.com/package/@interactjs/offset/v/1.10.17 1.10.17 MIT - https://npmjs.com/package/@interactjs/pointer-events/v/1.10.17 1.10.17 MIT - https://npmjs.com/package/@interactjs/reflow/v/1.10.17 1.10.17 MIT - https://npmjs.com/package/@interactjs/snappers/v/1.10.17 1.10.17 MIT - https://npmjs.com/package/@interactjs/types/v/1.10.17 1.10.17 MIT - https://npmjs.com/package/@interactjs/utils/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/actions/v/1.10.27 1.10.27 MIT + https://npmjs.com/package/@interactjs/auto-scroll/v/1.10.27 1.10.27 MIT + https://npmjs.com/package/@interactjs/auto-start/v/1.10.27 1.10.27 MIT + https://npmjs.com/package/@interactjs/core/v/1.10.27 1.10.27 MIT + https://npmjs.com/package/@interactjs/dev-tools/v/1.10.27 1.10.27 MIT + https://npmjs.com/package/@interactjs/inertia/v/1.10.27 1.10.27 MIT + https://npmjs.com/package/@interactjs/interact/v/1.10.27 1.10.27 MIT + https://npmjs.com/package/@interactjs/interactjs/v/1.10.27 1.10.27 MIT + https://npmjs.com/package/@interactjs/modifiers/v/1.10.27 1.10.27 MIT + https://npmjs.com/package/@interactjs/offset/v/1.10.27 1.10.27 MIT + https://npmjs.com/package/@interactjs/pointer-events/v/1.10.27 1.10.27 MIT + https://npmjs.com/package/@interactjs/reflow/v/1.10.27 1.10.27 MIT + https://npmjs.com/package/@interactjs/snappers/v/1.10.27 1.10.27 MIT + https://npmjs.com/package/@interactjs/utils/v/1.10.27 1.10.27 MIT https://npmjs.com/package/@intlify/core-base/v/9.14.5 9.14.5 MIT https://npmjs.com/package/@intlify/message-compiler/v/9.14.5 9.14.5 MIT https://npmjs.com/package/@intlify/shared/v/9.14.5 9.14.5 MIT @@ -529,7 +528,7 @@ The text of each license is also included in licenses/LICENSE-[project].txt. https://npmjs.com/package/d3-dsv/node_modules/commander/v/7.2.0 7.2.0 MIT https://npmjs.com/package/d3-tip/v/0.9.1 0.9.1 MIT https://npmjs.com/package/dayjs/v/1.11.13 1.11.13 MIT - https://npmjs.com/package/element-plus/v/2.9.4 2.9.4 MIT + https://npmjs.com/package/element-plus/v/2.11.0 2.11.0 MIT https://npmjs.com/package/element-resize-detector/v/1.2.4 1.2.4 MIT https://npmjs.com/package/escape-html/v/1.0.3 1.0.3 MIT https://npmjs.com/package/estree-walker/v/2.0.2 2.0.2 MIT @@ -612,4 +611,4 @@ The text of each license is also included in licenses/LICENSE-[project].txt. ======================================================================= The zipkin-lens.jar dependency has more front-end dependencies in it and the front-end dependencies' licenses -are listed in zipkin-LICENSE. +are listed in zipkin-LICENSE. \ No newline at end of file From 93b407b12c56a85d9cb36a3c8b475cb0adc35d28 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Fri, 17 Oct 2025 14:36:06 +0800 Subject: [PATCH 50/69] fix doc --- dist-material/release-docs/LICENSE | 2 +- docs/en/api/query-protocol.md | 21 ++++++++++++++++++- .../src/main/resources/log4j2.xml | 12 +++++------ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/dist-material/release-docs/LICENSE b/dist-material/release-docs/LICENSE index 8d264d2c8a98..2556de81b338 100644 --- a/dist-material/release-docs/LICENSE +++ b/dist-material/release-docs/LICENSE @@ -611,4 +611,4 @@ The text of each license is also included in licenses/LICENSE-[project].txt. ======================================================================= The zipkin-lens.jar dependency has more front-end dependencies in it and the front-end dependencies' licenses -are listed in zipkin-LICENSE. \ No newline at end of file +are listed in zipkin-LICENSE. \ No newline at end of file diff --git a/docs/en/api/query-protocol.md b/docs/en/api/query-protocol.md index 1f97e908e376..90c9fb818844 100644 --- a/docs/en/api/query-protocol.md +++ b/docs/en/api/query-protocol.md @@ -215,7 +215,7 @@ extend type Query { Event query fetches the event list based on given sources and time range conditions. ### Profiling -SkyWalking offers two types of [profiling](../concepts-and-designs/profiling.md), in-process(tracing profiling and async-profiler) and out-process(ebpf profiling), allowing users to create tasks and check their execution status. +SkyWalking offers two types of [profiling](../concepts-and-designs/profiling.md), in-process(tracing profiling, async-profiler and pprof) and out-process(ebpf profiling), allowing users to create tasks and check their execution status. #### In-process profiling @@ -256,6 +256,25 @@ extend type Query { } ``` +##### pprof + +```graphql +extend type Mutation { + # Create a new pprof task + createPprofTask(pprofTaskCreationRequest: PprofTaskCreationRequest!): PprofTaskCreationResult! +} + +extend type Query { + # Query all task lists and sort them in descending order by create time + queryPprofTaskList(request: PprofTaskListRequest!): PprofTaskListResult! + # Query task progress, including task logs + queryPprofTaskProgress(taskId: String!): PprofTaskProgress! + # Query the flame graph produced by pprof + queryPprofAnalyze(request: PprofAnalyzationRequest!): PprofAnalyzation! +} +``` + + #### Out-process profiling ```graphql diff --git a/oap-server/server-starter/src/main/resources/log4j2.xml b/oap-server/server-starter/src/main/resources/log4j2.xml index 6cbaa5c9dd0e..0702a7e77dcc 100644 --- a/oap-server/server-starter/src/main/resources/log4j2.xml +++ b/oap-server/server-starter/src/main/resources/log4j2.xml @@ -17,7 +17,7 @@ ~ --> - + @@ -32,15 +32,15 @@ - + - - + + - - + + From 55d67eb29efe683383201cb88cac9325e721d193 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Fri, 17 Oct 2025 16:16:49 +0800 Subject: [PATCH 51/69] add doc --- dist-material/release-docs/LICENSE | 2 +- .../setup/backend/backend-go-app-profiling.md | 104 ++++++++++++++++++ 2 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 docs/en/setup/backend/backend-go-app-profiling.md diff --git a/dist-material/release-docs/LICENSE b/dist-material/release-docs/LICENSE index 2556de81b338..8d264d2c8a98 100644 --- a/dist-material/release-docs/LICENSE +++ b/dist-material/release-docs/LICENSE @@ -611,4 +611,4 @@ The text of each license is also included in licenses/LICENSE-[project].txt. ======================================================================= The zipkin-lens.jar dependency has more front-end dependencies in it and the front-end dependencies' licenses -are listed in zipkin-LICENSE. \ No newline at end of file +are listed in zipkin-LICENSE. \ No newline at end of file diff --git a/docs/en/setup/backend/backend-go-app-profiling.md b/docs/en/setup/backend/backend-go-app-profiling.md new file mode 100644 index 000000000000..1a00587d3041 --- /dev/null +++ b/docs/en/setup/backend/backend-go-app-profiling.md @@ -0,0 +1,104 @@ +# Go App Profiling + +Go App Profiling uses the Pprof for sampling + +Pprof is bound within the auto-instrument agent and corresponds to [In-Process Profiling](../../concepts-and-designs/profiling.md#in-process-profiling). + +It is delivered to the agent in the form of a task, allowing it to be enabled or disabled dynamically. +When service encounters performance issues (cpu usage, memory allocation, etc.), Pprof task can be created. +When the agent receives a task, it enables Pprof for sampling. +After sampling is completed, the sampling results are analyzed by requesting the server to render a flame graph for performance +analysis to determine the specific business code lines that cause performance problems. + +## Activate Pprof in the OAP +OAP and the agent use a brand-new protocol to exchange Pprof data, so it is necessary to start OAP with the following configuration: + +```yaml +receiver-pprof: + selector: ${SW_RECEIVER_PPROF:default} + default: + # Used to manage the maximum size of the pprof file that can be received, the unit is Byte, default is 30M + pprofMaxSize: ${SW_RECEIVER_PPROF_MAX_SIZE:31457280} + # Used to determine whether to receive pprof in memory file or physical file mode + # + # The memory file mode have fewer local file system limitations, so they are by default. But it costs more memory. + # + # The physical file mode will use less memory when parsing and is more friendly to parsing large files. + # However, if the storage of the tmp directory in the container is insufficient, the oap server instance may crash. + # It is recommended to use physical file mode when volume mounting is used or the tmp directory has sufficient storage. + memoryParserEnabled: ${SW_RECEIVER_PPROF_MEMORY_PARSER_ENABLED:true} +``` + +## Pprof Task with Analysis + +To use the Pprof feature, please follow these steps: + +1. **Create Pprof task**: Use the UI or CLI tool to create a task. +2. **Wait agent collect data and upload**: Wait for Pprof to collect pprof data and report. +3. **Query task progress**: Query the progress of tasks, including analyzing successful and failed instances and task logs. +4. **Analyze the data**: Analyze the pprof data to determine where performance bottlenecks exist in the service. + +### Create an Pprof task + +Create an Pprof task to notify some go-agent instances in the execution service to start Pprof for data collection. + +When creating a task, the following configuration fields are required: + +1. **serviceId**: Define the service to execute the task. +2. **serviceInstanceIds**: Define which instances need to execute tasks. +3. **duration**: Define the duration of this task in minutes, required for CPU, BLOCK, MUTEX events. +4. **events**: Define which event types this task needs to collect. +5. **dumpPeriod**: Define the period of the pprof dump, required for BLOCK, MUTEX events. + +When the Agent receives a Pprof task from OAP, it automatically generates a log to notify that the task has been acknowledged. The log contains the following field information: + +1. **Instance**: The name of the instance where the Agent is located. +2. **Type**: Supports "NOTIFIED" and "EXECUTION_FINISHED" and "PPROF_UPLOAD_FILE_TOO_LARGE_ERROR", "EXECUTION_TASK_ERROR", with the current log displaying "NOTIFIED". +3. **Time**: The time when the Agent received the task. + +### Wait the agent to collect data and upload + +At this point, Pprof will trace the events you selected when you created the task: + +1. CPU: samples CPU usage over time to show which functions consume the most processing time. +2. ALLOC, HEAP: + - HEAP: a sampling of memory allocations of live objects. + - ALLOC: a sampling of all past memory allocations. +3. BLOCK, MUTEX: + - BLOCK: stack traces that led to blocking on synchronization primitives. + - MUTEX: stack traces of holders of contended mutexes. +4. GOROUTINE, THREADCREAT: + - GOROUTINE: stack traces of all current goroutines. + - THREADCREATE: stack traces that led to the creation of new OS threads. + +Finally, the agent will upload the pprof file produced by Pprof to the oap server for online performance analysis. + +### Query the profiling task progresses + +Wait for Pprof to complete data collection and upload successfully. +We can query the execution logs of the Pprof task and the task status, which includes the following information: + +1. **successInstanceIds**: SuccessInstanceIds gives instances that have executed the task successfully. +2. **errorInstanceIds**: ErrorInstanceIds gives instances that failed to execute the task. +3. **logs**: All task execution logs of the current task. + 1. **id**: The task id. + 2. **instanceId**: InstanceId is the id of the instance which reported this task log. + 3. **instanceName**: InstanceName is the name of the instance which reported this task log. + 4. **operationType**: Contains "NOTIFIED" and "EXECUTION_FINISHED" and "PPROF_UPLOAD_FILE_TOO_LARGE_ERROR", "EXECUTION_TASK_ERROR". + 5. **operationTime**: operationTime is the time when the operation occurs. + +### Analyze the profiling data + +Once some agents completed the task, we can analyze the data through the following query: + +1. **taskId**: The task id. +2. **instanceIds**: InstanceIds defines the instances to be included for analysis + +After the query, the following data would be returned to render a flame graph: +1. **taskId**: The task id. +2. **elements**: Combined with "id" to determine the hierarchical relationship. + 1. **Id**: Id is the identity of the stack element. + 2. **parentId**: Parent element ID. The dependency relationship between elements can be determined using the element ID and parent element ID. + 3. **codeSignature**: Method signatures in tree nodes. + 4. **total**:The total number of samples of the current tree node, including child nodes. + 5. **self**: The sampling number of the current tree node, excluding samples of the children. \ No newline at end of file From 8790b794aec040551e4e494d10c614bb98d59a51 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Sat, 18 Oct 2025 13:11:17 +0800 Subject: [PATCH 52/69] roll back --- dist-material/release-docs/LICENSE | 2 +- .../server-starter/src/main/resources/log4j2.xml | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dist-material/release-docs/LICENSE b/dist-material/release-docs/LICENSE index 8d264d2c8a98..35777513907e 100644 --- a/dist-material/release-docs/LICENSE +++ b/dist-material/release-docs/LICENSE @@ -611,4 +611,4 @@ The text of each license is also included in licenses/LICENSE-[project].txt. ======================================================================= The zipkin-lens.jar dependency has more front-end dependencies in it and the front-end dependencies' licenses -are listed in zipkin-LICENSE. \ No newline at end of file +are listed in zipkin-LICENSE. diff --git a/oap-server/server-starter/src/main/resources/log4j2.xml b/oap-server/server-starter/src/main/resources/log4j2.xml index 0702a7e77dcc..6cbaa5c9dd0e 100644 --- a/oap-server/server-starter/src/main/resources/log4j2.xml +++ b/oap-server/server-starter/src/main/resources/log4j2.xml @@ -17,7 +17,7 @@ ~ --> - + @@ -32,15 +32,15 @@ - + - - + + - - + + From 2431af5aef506ea8ba911858e0fbc2d926f12af3 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Mon, 20 Oct 2025 15:08:52 +0800 Subject: [PATCH 53/69] roll back ui submodule --- skywalking-ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skywalking-ui b/skywalking-ui index ad4b0639cd9b..3cefbf1bd5d5 160000 --- a/skywalking-ui +++ b/skywalking-ui @@ -1 +1 @@ -Subproject commit ad4b0639cd9bef691a12f894d95d385fb44e889c +Subproject commit 3cefbf1bd5d588355829cbe9dd23cf2253af547e From 0e397afa5b2ecb807803b3d98879423e53ab2f91 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Mon, 20 Oct 2025 16:32:15 +0800 Subject: [PATCH 54/69] roll back & test ci --- .github/workflows/skywalking.yaml | 31 +++++++------ apm-protocol/apm-network/src/main/proto | 2 +- .../server/core/query/type/AlarmMessage.java | 1 - .../src/main/resources/query-protocol | 2 +- .../PprofByteBufCollectionObserver.java | 7 --- .../stream/PprofFileCollectionObserver.java | 7 --- .../storage-jdbc-hikaricp-plugin/pom.xml | 46 ------------------- .../profiling/pprof/expected/progress.yml | 4 +- 8 files changed, 21 insertions(+), 79 deletions(-) diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index 0bd11fca045c..39ca2610ed5c 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -17,6 +17,9 @@ name: CI on: + push + branches: + - ci1 pull_request: schedule: - cron: "0 18 * * *" # TimeZone: UTC 0 @@ -32,7 +35,7 @@ env: jobs: license-header: - if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || (github.event_name != 'schedule') name: License header runs-on: ubuntu-latest timeout-minutes: 10 @@ -45,7 +48,7 @@ jobs: uses: apache/skywalking-eyes@5b7ee1731d036b5aac68f8bd3fc9e6f98ada082e code-style: - if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || (github.event_name != 'schedule') name: Code style runs-on: ubuntu-latest timeout-minutes: 10 @@ -60,7 +63,7 @@ jobs: dependency-license: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.pom == 'true' || needs.changes.outputs.ui == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.pom == 'true' || needs.changes.outputs.ui == 'true') name: Dependency licenses needs: [changes] runs-on: ubuntu-latest @@ -90,7 +93,7 @@ jobs: fi sanity-check: - if: ( always() && ! cancelled() ) && (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') + if: ( always() && ! cancelled() ) && (github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || (github.event_name != 'schedule') name: Sanity check results needs: [license-header, code-style, dependency-license] runs-on: ubuntu-latest @@ -159,7 +162,7 @@ jobs: dist-tar: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: Build dist tar needs: [changes] runs-on: ubuntu-latest @@ -191,7 +194,7 @@ jobs: docker: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: Docker images needs: [sanity-check, dist-tar, changes] runs-on: ubuntu-latest @@ -230,7 +233,7 @@ jobs: unit-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: Unit test needs: [sanity-check, changes] runs-on: ${{ matrix.os }} @@ -265,7 +268,7 @@ jobs: integration-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: Integration test needs: [sanity-check, changes] runs-on: ubuntu-latest @@ -298,7 +301,7 @@ jobs: slow-integration-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: Slow Integration Tests needs: [sanity-check, changes] runs-on: ubuntu-latest @@ -331,7 +334,7 @@ jobs: e2e-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker, dist-tar] runs-on: ${{ matrix.test.runs-on || 'ubuntu-latest' }} @@ -793,7 +796,7 @@ jobs: e2e-test-istio: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-24.04 @@ -861,7 +864,7 @@ jobs: e2e-test-istio-ambient: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-24.04 @@ -922,7 +925,7 @@ jobs: e2e-test-java-versions: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-latest @@ -972,7 +975,7 @@ jobs: # e2e-test-banyandb-stages: # if: | # ( always() && ! cancelled() ) && -# ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') +# ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') # name: E2E test # needs: [docker, dist-tar] # runs-on: ${{ matrix.test.runs-on || 'ubuntu-latest' }} diff --git a/apm-protocol/apm-network/src/main/proto b/apm-protocol/apm-network/src/main/proto index fb3fb005650e..6ec6d71168c1 160000 --- a/apm-protocol/apm-network/src/main/proto +++ b/apm-protocol/apm-network/src/main/proto @@ -1 +1 @@ -Subproject commit fb3fb005650e2489164978b7804117c7ade1529a +Subproject commit 6ec6d71168c1068d1f09f19d57120dd344a1d585 diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/AlarmMessage.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/AlarmMessage.java index 67dd14f234a8..7ff8e5306ccd 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/AlarmMessage.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/AlarmMessage.java @@ -35,7 +35,6 @@ public class AlarmMessage { private String name; private String message; private Long startTime; - private Long recoveryTime; private transient String id1; private final List tags; private List events = new ArrayList<>(2); diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol index 4fc10625ba72..003664676984 160000 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol @@ -1 +1 @@ -Subproject commit 4fc10625ba72ef4788972b4f7991a535065d609b +Subproject commit 0036646769842e915e2828fde0b6c1da0179a1e5 diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java index 833305e8f0b9..3990996d140a 100644 --- a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java @@ -130,8 +130,6 @@ private void parseAndStorageData(PprofCollectionMetaData taskMetaData, ByteBuffe } public void parsePprofAndStorage(PprofCollectionMetaData taskMetaData, ByteBuffer buf) throws IOException { - log.info("Parsing pprof file for service: {}, instance: {}", - taskMetaData.getServiceId(), taskMetaData.getInstanceId()); PprofTask task = taskMetaData.getTask(); FrameTree tree = PprofParser.dumpTree(buf); PprofProfilingData data = new PprofProfilingData(); @@ -140,11 +138,6 @@ public void parsePprofAndStorage(PprofCollectionMetaData taskMetaData, ByteBuffe data.setTaskId(task.getId()); data.setInstanceId(taskMetaData.getInstanceId()); data.setUploadTime(taskMetaData.getUploadTime()); - log.info("data eventType: {}", data.getEventType()); - log.info("data frameTree: {}", tree); - log.info("data taskId: {}", task.getId()); - log.info("data instanceId: {}", taskMetaData.getInstanceId()); - log.info("data uploadTime: {}", taskMetaData.getUploadTime()); sourceReceiver.receive(data); } } diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java index a5cd831e9685..9ae09a6d97b3 100644 --- a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java @@ -150,8 +150,6 @@ private void parseAndStorageData(PprofCollectionMetaData taskMetaData, String fi public void parsePprofAndStorage(PprofCollectionMetaData taskMetaData, String fileName) throws IOException { - log.info("Parsing pprof file for service: {}, instance: {}", - taskMetaData.getServiceId(), taskMetaData.getInstanceId()); PprofTask task = taskMetaData.getTask(); FrameTree tree = PprofParser.dumpTree(fileName); PprofProfilingData data = new PprofProfilingData(); @@ -160,11 +158,6 @@ public void parsePprofAndStorage(PprofCollectionMetaData taskMetaData, data.setTaskId(task.getId()); data.setInstanceId(taskMetaData.getInstanceId()); data.setUploadTime(taskMetaData.getUploadTime()); - log.info("data eventType: {}", data.getEventType()); - log.info("data frameTree: {}", tree); - log.info("data taskId: {}", task.getId()); - log.info("data instanceId: {}", taskMetaData.getInstanceId()); - log.info("data uploadTime: {}", taskMetaData.getUploadTime()); sourceReceiver.receive(data); } } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml index 025c702d12a8..3285d110a4e1 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml @@ -48,12 +48,6 @@ org.postgresql postgresql - - mysql - mysql-connector-java - 8.0.13 - provided - org.testcontainers @@ -61,44 +55,4 @@ test - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - default-test - test - - test - - - - **/PreventRedistributionMySQLDriverTest.java - - - - - - test-without-mysql - test - - test - - - - **/PreventRedistributionMySQLDriverTest.java - - - mysql:mysql-connector-java - - - - - - - diff --git a/test/e2e-v2/cases/profiling/pprof/expected/progress.yml b/test/e2e-v2/cases/profiling/pprof/expected/progress.yml index 9c285f08c530..56de9d629489 100644 --- a/test/e2e-v2/cases/profiling/pprof/expected/progress.yml +++ b/test/e2e-v2/cases/profiling/pprof/expected/progress.yml @@ -17,8 +17,8 @@ logs: {{- contains .logs }} - id: {{ notEmpty .id}} instanceid: {{ b64enc "go-service" }}.1_{{ b64enc "provider1" }} - instancename: {{ notEmpty .instancename}} - operationtype: {{ notEmpty .operationtype}} + instancename: provider1 + operationtype: EXECUTION_FINISHED operationtime: {{ ge .operationtime 0 }} {{- end }} errorinstanceids: [] From 901eeaa16c8728d89ec7e76582a8d0d3af4baeac Mon Sep 17 00:00:00 2001 From: JophieQu Date: Mon, 20 Oct 2025 16:34:55 +0800 Subject: [PATCH 55/69] test ci --- .github/workflows/skywalking.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index 39ca2610ed5c..e50112abaf53 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -17,7 +17,7 @@ name: CI on: - push + push: branches: - ci1 pull_request: From ad6e7ae24cc3df4ccb8a4546b29d0d58d4c43c22 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Tue, 21 Oct 2025 15:35:20 +0800 Subject: [PATCH 56/69] fix: format the code --- .../oap/server/core/cache/PprofTaskCache.java | 22 +-- .../profiling/pprof/PprofMutationService.java | 61 ++++---- .../profiling/pprof/PprofQueryService.java | 70 +++++---- .../storage/PprofProfilingDataDispatcher.java | 24 +-- .../storage/PprofProfilingDataRecord.java | 25 +-- .../pprof/storage/PprofTaskLogRecord.java | 13 +- .../pprof/storage/PprofTaskRecord.java | 5 +- .../query/input/PprofAnalyzationRequest.java | 3 +- .../query/input/PprofTaskCreationRequest.java | 3 +- .../core/query/type/PprofEventType.java | 1 - .../core/query/type/PprofStackElement.java | 2 +- .../core/query/type/PprofStackTree.java | 4 +- .../oap/server/core/query/type/PprofTask.java | 3 +- .../core/query/type/PprofTaskListResult.java | 3 +- .../query/type/PprofTaskLogOperationType.java | 3 +- .../core/query/type/PprofTaskProgress.java | 3 +- .../profiling/pprof/IPprofDataQueryDAO.java | 9 +- .../pprof/IPprofTaskLogQueryDAO.java | 4 +- .../profiling/pprof/IPprofTaskQueryDAO.java | 6 +- .../pprof/parser/PprofMergeBuilder.java | 4 +- .../library/pprof/parser/PprofParser.java | 4 +- .../oap/server/library/pprof/type/Frame.java | 3 +- .../server/library/pprof/type/FrameTree.java | 5 +- .../library/pprof/type/FrameTreeBuilder.java | 7 +- .../pprof/provider/PprofModuleProvider.java | 16 +- .../provider/handler/PprofServiceHandler.java | 62 ++++---- .../PprofByteBufCollectionObserver.java | 88 ++++++----- .../stream/PprofCollectionMetaData.java | 5 +- .../stream/PprofFileCollectionObserver.java | 96 +++++++----- .../stream/BanyanDBPprofDataQueryDAO.java | 48 +++--- .../stream/BanyanDBPprofTaskLogQueryDAO.java | 43 +++--- .../stream/BanyanDBPprofTaskQueryDAO.java | 14 +- .../query/PprofDataQueryEsDAO.java | 37 +++-- .../query/PprofTaskLogQueryEsDAO.java | 1 - .../query/PprofTaskQueryEsDAO.java | 32 ++-- .../common/dao/JDBCPprofDataQueryDAO.java | 143 +++++++++--------- .../common/dao/JDBCPprofTaskLogQueryDAO.java | 43 +++--- .../common/dao/JDBCPprofTaskQueryDAO.java | 100 ++++++------ 38 files changed, 528 insertions(+), 487 deletions(-) diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java index 6a06227a5001..00aa57357068 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java @@ -18,28 +18,28 @@ package org.apache.skywalking.oap.server.core.cache; -import org.apache.skywalking.oap.server.library.module.Service; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import java.time.Duration; +import java.util.concurrent.TimeUnit; import org.apache.skywalking.oap.server.core.CoreModuleConfig; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.query.type.PprofTask; -import java.time.Duration; -import java.util.concurrent.TimeUnit; +import org.apache.skywalking.oap.server.library.module.Service; public class PprofTaskCache implements Service { private final Cache serviceId2taskCache; - + public PprofTaskCache(CoreModuleConfig moduleConfig) { long initialSize = moduleConfig.getMaxSizeOfProfileTask() / 10L; int initialCapacitySize = (int) (initialSize > Integer.MAX_VALUE ? Integer.MAX_VALUE : initialSize); - + serviceId2taskCache = CacheBuilder.newBuilder() - .initialCapacity(initialCapacitySize) - .maximumSize(moduleConfig.getMaxSizeOfProfileTask()) - // remove old pprof task data - .expireAfterWrite(Duration.ofMinutes(1)) - .build(); + .initialCapacity(initialCapacitySize) + .maximumSize(moduleConfig.getMaxSizeOfProfileTask()) + // remove old pprof task data + .expireAfterWrite(Duration.ofMinutes(1)) + .build(); } public PprofTask getPprofTask(String serviceId) { @@ -49,7 +49,7 @@ public PprofTask getPprofTask(String serviceId) { public void saveTask(String serviceId, PprofTask task) { if (task == null) { - return ; + return; } serviceId2taskCache.put(serviceId, task); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java index 3d8264e90644..527a24051eb1 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java @@ -18,25 +18,24 @@ package org.apache.skywalking.oap.server.core.profiling.pprof; -import lombok.RequiredArgsConstructor; -import org.apache.skywalking.oap.server.library.module.Service; -import org.apache.skywalking.oap.server.library.module.ModuleManager; -import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; -import org.apache.skywalking.oap.server.core.storage.StorageModule; -import org.apache.skywalking.oap.server.core.query.type.PprofEventType; import java.io.IOException; import java.util.List; -import org.apache.skywalking.oap.server.core.analysis.worker.NoneStreamProcessor; import java.util.concurrent.TimeUnit; -import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.worker.NoneStreamProcessor; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; import org.apache.skywalking.oap.server.core.query.type.PprofTaskCreationResult; import org.apache.skywalking.oap.server.core.query.type.PprofTaskCreationType; -import org.apache.skywalking.oap.server.core.query.type.PprofTask; -import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; - +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.Service; import org.apache.skywalking.oap.server.library.util.CollectionUtils; -import lombok.extern.slf4j.Slf4j; @Slf4j @RequiredArgsConstructor @@ -48,12 +47,12 @@ public class PprofMutationService implements Service { private IPprofTaskQueryDAO getPprofTaskDAO() { if (taskQueryDAO == null) { this.taskQueryDAO = moduleManager.find(StorageModule.NAME) - .provider() - .getService(IPprofTaskQueryDAO.class); + .provider() + .getService(IPprofTaskQueryDAO.class); } return taskQueryDAO; } - + public PprofTaskCreationResult createTask(String serviceId, List serviceInstanceIds, int duration, @@ -62,7 +61,7 @@ public PprofTaskCreationResult createTask(String serviceId, long createTime = System.currentTimeMillis(); // check data PprofTaskCreationResult checkResult = checkDataSuccess( - serviceId, serviceInstanceIds, duration, createTime, events, dumpPeriod + serviceId, serviceInstanceIds, duration, createTime, events, dumpPeriod ); if (checkResult != null) { return checkResult; @@ -80,30 +79,30 @@ public PprofTaskCreationResult createTask(String serviceId, task.setTimeBucket(TimeBucket.getRecordTimeBucket(createTime)); NoneStreamProcessor.getInstance().in(task); return PprofTaskCreationResult.builder() - .id(task.id().build()) - .code(PprofTaskCreationType.SUCCESS) - .build(); + .id(task.id().build()) + .code(PprofTaskCreationType.SUCCESS) + .build(); } private PprofTaskCreationResult checkDataSuccess(String serviceId, - List serviceInstanceIds, - int duration, - long createTime, - PprofEventType events, - int dumpPeriod) throws IOException { + List serviceInstanceIds, + int duration, + long createTime, + PprofEventType events, + int dumpPeriod) throws IOException { String checkArgumentMessage = checkArgumentError(serviceId, serviceInstanceIds, duration, events, dumpPeriod); if (checkArgumentMessage != null) { return PprofTaskCreationResult.builder() - .code(PprofTaskCreationType.ARGUMENT_ERROR) - .errorReason(checkArgumentMessage) - .build(); + .code(PprofTaskCreationType.ARGUMENT_ERROR) + .errorReason(checkArgumentMessage) + .build(); } String checkTaskProfilingMessage = checkTaskProfiling(serviceId, createTime); if (checkTaskProfilingMessage != null) { return PprofTaskCreationResult.builder() - .code(PprofTaskCreationType.ALREADY_PROFILING_ERROR) - .errorReason(checkTaskProfilingMessage) - .build(); + .code(PprofTaskCreationType.ALREADY_PROFILING_ERROR) + .errorReason(checkTaskProfilingMessage) + .build(); } return null; } @@ -140,7 +139,7 @@ private String checkTaskProfiling(String serviceId, // Each service can only enable one task at a time long endTimeBucket = TimeBucket.getMinuteTimeBucket(createTime); final List alreadyHaveTaskList = getPprofTaskDAO().getTaskList( - serviceId, null, endTimeBucket, 1 + serviceId, null, endTimeBucket, 1 ); if (CollectionUtils.isNotEmpty(alreadyHaveTaskList)) { for (PprofTask task : alreadyHaveTaskList) { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofQueryService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofQueryService.java index 2d456996ac02..101b5a685fbd 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofQueryService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofQueryService.java @@ -18,27 +18,27 @@ package org.apache.skywalking.oap.server.core.profiling.pprof; -import org.apache.skywalking.oap.server.core.analysis.IDManager; -import org.apache.skywalking.oap.server.library.module.Service; -import org.apache.skywalking.oap.server.library.module.ModuleManager; -import org.apache.skywalking.oap.server.core.storage.StorageModule; -import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; -import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; -import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; -import org.apache.skywalking.oap.server.core.query.PprofTaskLog; -import org.apache.skywalking.oap.server.core.query.input.Duration; -import org.apache.skywalking.oap.server.core.query.type.PprofTask; -import java.util.Objects; -import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import com.google.gson.Gson; import java.io.IOException; -import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; -import org.apache.skywalking.oap.server.core.query.type.PprofStackTree; -import org.apache.skywalking.oap.server.library.pprof.parser.PprofMergeBuilder; import java.util.List; -import com.google.gson.Gson; +import java.util.Objects; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.type.PprofStackTree; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.Service; +import org.apache.skywalking.oap.server.library.pprof.parser.PprofMergeBuilder; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; @Slf4j @RequiredArgsConstructor @@ -54,8 +54,8 @@ public class PprofQueryService implements Service { private IPprofTaskQueryDAO getTaskQueryDAO() { if (taskQueryDAO == null) { this.taskQueryDAO = moduleManager.find(StorageModule.NAME) - .provider() - .getService(IPprofTaskQueryDAO.class); + .provider() + .getService(IPprofTaskQueryDAO.class); } return taskQueryDAO; } @@ -63,8 +63,8 @@ private IPprofTaskQueryDAO getTaskQueryDAO() { private IPprofDataQueryDAO getPprofDataQueryDAO() { if (dataQueryDAO == null) { this.dataQueryDAO = moduleManager.find(StorageModule.NAME) - .provider() - .getService(IPprofDataQueryDAO.class); + .provider() + .getService(IPprofDataQueryDAO.class); } return dataQueryDAO; } @@ -72,8 +72,8 @@ private IPprofDataQueryDAO getPprofDataQueryDAO() { private IPprofTaskLogQueryDAO getTaskLogQueryDAO() { if (logQueryDAO == null) { this.logQueryDAO = moduleManager.find(StorageModule.NAME) - .provider() - .getService(IPprofTaskLogQueryDAO.class); + .provider() + .getService(IPprofTaskLogQueryDAO.class); } return logQueryDAO; } @@ -90,13 +90,17 @@ public List queryTask(String serviceId, Duration duration, Integer li } public PprofStackTree queryPprofData(String taskId, List instanceIds) throws IOException { - List pprofDataList = getPprofDataQueryDAO().getByTaskIdAndInstances(taskId, instanceIds); + List pprofDataList = getPprofDataQueryDAO().getByTaskIdAndInstances( + taskId, instanceIds); List trees = pprofDataList.stream() - .map(data -> GSON.fromJson(new String(data.getDataBinary()), FrameTree.class)) - .collect(Collectors.toList()); + .map(data -> GSON.fromJson( + new String(data.getDataBinary()), + FrameTree.class + )) + .collect(Collectors.toList()); FrameTree resultTree = new PprofMergeBuilder() - .merge(trees) - .build(); + .merge(trees) + .build(); return new PprofStackTree(resultTree); } @@ -104,19 +108,19 @@ public List queryPprofTaskLogs(String taskId) throws IOException { List taskLogList = getTaskLogQueryDAO().getTaskLogList(); return findMatchedLogs(taskId, taskLogList); } - + private List findMatchedLogs(final String taskID, final List allLogs) { return allLogs.stream() - .filter(l -> Objects.equals(l.getId(), taskID)) - .map(this::extendTaskLog) - .collect(Collectors.toList()); + .filter(l -> Objects.equals(l.getId(), taskID)) + .map(this::extendTaskLog) + .collect(Collectors.toList()); } private PprofTaskLog extendTaskLog(PprofTaskLog log) { final IDManager.ServiceInstanceID.InstanceIDDefinition instanceIDDefinition = IDManager.ServiceInstanceID - .analysisId(log.getInstanceId()); + .analysisId(log.getInstanceId()); log.setInstanceName(instanceIDDefinition.getName()); return log; } - + } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java index c35e197e9f18..631d11bf6f07 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java @@ -20,21 +20,21 @@ import com.google.gson.Gson; import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher; -import org.apache.skywalking.oap.server.core.source.PprofProfilingData; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.source.PprofProfilingData; public class PprofProfilingDataDispatcher implements SourceDispatcher { - private static final Gson GSON = new Gson(); + private static final Gson GSON = new Gson(); - @Override - public void dispatch(PprofProfilingData source) { - PprofProfilingDataRecord record = new PprofProfilingDataRecord(); - record.setTaskId(source.getTaskId()); - record.setInstanceId(source.getInstanceId()); - record.setDataBinary(GSON.toJson(source.getFrameTree()).getBytes()); - record.setUploadTime(source.getUploadTime()); - record.setTimeBucket(TimeBucket.getRecordTimeBucket(source.getUploadTime())); - RecordStreamProcessor.getInstance().in(record); - } + @Override + public void dispatch(PprofProfilingData source) { + PprofProfilingDataRecord record = new PprofProfilingDataRecord(); + record.setTaskId(source.getTaskId()); + record.setInstanceId(source.getInstanceId()); + record.setDataBinary(GSON.toJson(source.getFrameTree()).getBytes()); + record.setUploadTime(source.getUploadTime()); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(source.getUploadTime())); + RecordStreamProcessor.getInstance().in(record); + } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java index fd091c51b5c0..10d724934e1b 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java @@ -18,23 +18,24 @@ package org.apache.skywalking.oap.server.core.profiling.pprof.storage; +import com.google.common.hash.Hashing; +import java.nio.charset.StandardCharsets; import lombok.Data; +import org.apache.skywalking.oap.server.core.analysis.Stream; import org.apache.skywalking.oap.server.core.analysis.record.Record; import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.storage.StorageID; import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB; import org.apache.skywalking.oap.server.core.storage.annotation.Column; -import org.apache.skywalking.oap.server.core.analysis.Stream; -import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.PPROF_PROFILING_DATA; -import org.apache.skywalking.oap.server.core.storage.StorageID; import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; -import com.google.common.hash.Hashing; -import java.nio.charset.StandardCharsets; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.PPROF_PROFILING_DATA; @Data @Stream(name = PprofProfilingDataRecord.INDEX_NAME, scopeId = PPROF_PROFILING_DATA, - builder = PprofProfilingDataRecord.Builder.class, processor = RecordStreamProcessor.class) + builder = PprofProfilingDataRecord.Builder.class, processor = RecordStreamProcessor.class) @BanyanDB.TimestampColumn(PprofProfilingDataRecord.UPLOAD_TIME) @BanyanDB.Group(streamGroup = BanyanDB.StreamGroup.RECORDS) public class PprofProfilingDataRecord extends Record { @@ -57,14 +58,14 @@ public class PprofProfilingDataRecord extends Record { @Column(name = DATA_BINARY, storageOnly = true) private byte[] dataBinary; - @Override + @Override public StorageID id() { return new StorageID().append( - Hashing.sha256().newHasher() - .putString(taskId, StandardCharsets.UTF_8) - .putString(instanceId, StandardCharsets.UTF_8) - .putLong(uploadTime) - .hash().toString() + Hashing.sha256().newHasher() + .putString(taskId, StandardCharsets.UTF_8) + .putString(instanceId, StandardCharsets.UTF_8) + .putLong(uploadTime) + .hash().toString() ); } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java index f26496c84f2b..c1cf1b09e843 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java @@ -18,6 +18,8 @@ package org.apache.skywalking.oap.server.core.profiling.pprof.storage; +import lombok.Getter; +import lombok.Setter; import org.apache.skywalking.oap.server.core.analysis.Stream; import org.apache.skywalking.oap.server.core.analysis.record.Record; import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; @@ -32,9 +34,6 @@ import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.PPROF_TASK_LOG; -import lombok.Getter; -import lombok.Setter; - @Getter @Setter @ScopeDeclaration(id = PPROF_TASK_LOG, name = "PprofTaskLog") @@ -68,10 +67,10 @@ public class PprofTaskLogRecord extends Record { @Override public StorageID id() { return new StorageID() - .append(TASK_ID, getTaskId()) - .append(INSTANCE_ID, getInstanceId()) - .append(OPERATION_TYPE, getOperationType()) - .append(OPERATION_TIME, getOperationTime()); + .append(TASK_ID, getTaskId()) + .append(INSTANCE_ID, getInstanceId()) + .append(OPERATION_TYPE, getOperationType()) + .append(OPERATION_TIME, getOperationTime()); } public static class Builder implements StorageBuilder { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java index cf97a5c971c4..ab207b5a6a2c 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java @@ -18,6 +18,8 @@ package org.apache.skywalking.oap.server.core.profiling.pprof.storage; +import com.google.gson.Gson; +import java.util.List; import lombok.Getter; import lombok.Setter; import org.apache.skywalking.oap.server.core.analysis.Stream; @@ -31,9 +33,6 @@ import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; -import java.util.List; - -import com.google.gson.Gson; import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.PPROF_TASK; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofAnalyzationRequest.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofAnalyzationRequest.java index cb62845d2bee..b25a9eea93fd 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofAnalyzationRequest.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofAnalyzationRequest.java @@ -18,11 +18,10 @@ package org.apache.skywalking.oap.server.core.query.input; +import java.util.List; import lombok.Getter; import lombok.Setter; -import java.util.List; - @Getter @Setter public class PprofAnalyzationRequest { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskCreationRequest.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskCreationRequest.java index af19a9466d1e..a487bdf0a42f 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskCreationRequest.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskCreationRequest.java @@ -18,12 +18,11 @@ package org.apache.skywalking.oap.server.core.query.input; +import java.util.List; import lombok.Getter; import lombok.Setter; import org.apache.skywalking.oap.server.core.query.type.PprofEventType; -import java.util.List; - @Getter @Setter public class PprofTaskCreationRequest { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofEventType.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofEventType.java index 333b0b061dd9..2429a102380f 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofEventType.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofEventType.java @@ -32,7 +32,6 @@ public enum PprofEventType { THREADCREATE(5, "threadcreate"), ALLOCS(6, "allocs"); - private final int code; private final String name; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackElement.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackElement.java index b82f3e1ec670..404b18cc4197 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackElement.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackElement.java @@ -34,5 +34,5 @@ public class PprofStackElement { private long total; private long self; - + } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackTree.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackTree.java index 2600133dd98b..53e8f88ba7a2 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackTree.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackTree.java @@ -19,12 +19,12 @@ package org.apache.skywalking.oap.server.core.query.type; import com.google.common.collect.Lists; +import java.util.List; +import java.util.Objects; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; -import java.util.List; -import java.util.Objects; @Setter @Getter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTask.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTask.java index 3d8e4e3ba5bc..e54b3fab55e9 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTask.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTask.java @@ -19,7 +19,6 @@ package org.apache.skywalking.oap.server.core.query.type; import java.util.List; - import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; @@ -32,7 +31,7 @@ @AllArgsConstructor @Builder public class PprofTask { - + private String id; private String serviceId; private List serviceInstanceIds; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskListResult.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskListResult.java index e3ee7a75a99d..f136b35f504a 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskListResult.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskListResult.java @@ -19,11 +19,10 @@ package org.apache.skywalking.oap.server.core.query.type; +import java.util.List; import lombok.AllArgsConstructor; import lombok.Data; -import java.util.List; - @Data @AllArgsConstructor public class PprofTaskListResult { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskLogOperationType.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskLogOperationType.java index e220ceb23d37..3c1e544759f1 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskLogOperationType.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskLogOperationType.java @@ -24,7 +24,8 @@ public enum PprofTaskLogOperationType { NOTIFIED(1), // when sniffer has execution finished to report EXECUTION_FINISHED(2), // when sniffer has execution finished to report - PPROF_UPLOAD_FILE_TOO_LARGE_ERROR(3), // when sniffer finished task but jfr file is to large that oap server can not receive + PPROF_UPLOAD_FILE_TOO_LARGE_ERROR( + 3), // when sniffer finished task but jfr file is to large that oap server can not receive EXECUTION_TASK_ERROR(4) // when sniffer fails to execute its task ; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskProgress.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskProgress.java index 991e6e4f6d95..95de35c035de 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskProgress.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskProgress.java @@ -18,11 +18,10 @@ package org.apache.skywalking.oap.server.core.query.type; +import java.util.List; import lombok.Data; import org.apache.skywalking.oap.server.core.query.PprofTaskLog; -import java.util.List; - @Data public class PprofTaskProgress { private List logs; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofDataQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofDataQueryDAO.java index 523fa73d419b..f593d1f07e89 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofDataQueryDAO.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofDataQueryDAO.java @@ -19,18 +19,19 @@ package org.apache.skywalking.oap.server.core.storage.profiling.pprof; -import org.apache.skywalking.oap.server.library.module.Service; -import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; import java.io.IOException; import java.util.List; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import org.apache.skywalking.oap.server.library.module.Service; public interface IPprofDataQueryDAO extends Service { /** * get pprof data record * - * @param taskId taskId + * @param taskId taskId * @param instanceIds instances of successfully uploaded file and parsed * @return record list */ - List getByTaskIdAndInstances(final String taskId, List instanceIds) throws IOException; + List getByTaskIdAndInstances(final String taskId, + List instanceIds) throws IOException; } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java index ba6b652dc396..37b0205d8784 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java @@ -18,10 +18,10 @@ package org.apache.skywalking.oap.server.core.storage.profiling.pprof; -import org.apache.skywalking.oap.server.core.query.PprofTaskLog; -import org.apache.skywalking.oap.server.core.storage.DAO; import java.io.IOException; import java.util.List; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.storage.DAO; public interface IPprofTaskLogQueryDAO extends DAO { /** diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskQueryDAO.java index 4c4945654512..f97a3249f3d6 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskQueryDAO.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskQueryDAO.java @@ -18,10 +18,10 @@ package org.apache.skywalking.oap.server.core.storage.profiling.pprof; -import org.apache.skywalking.oap.server.core.storage.DAO; -import org.apache.skywalking.oap.server.core.query.type.PprofTask; import java.io.IOException; import java.util.List; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.storage.DAO; public interface IPprofTaskQueryDAO extends DAO { @@ -34,7 +34,7 @@ public interface IPprofTaskQueryDAO extends DAO { * @param limit limit count, if null means query all */ List getTaskList(final String serviceId, final Long startTimeBucket, - final Long endTimeBucket, final Integer limit) throws IOException; + final Long endTimeBucket, final Integer limit) throws IOException; /** * query profile task by id diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java index 3e4e0f9f0082..dd9f3130d47f 100644 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java @@ -18,9 +18,9 @@ package org.apache.skywalking.oap.server.library.pprof.parser; -import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; -import org.apache.skywalking.oap.server.library.pprof.type.Frame; import java.util.List; +import org.apache.skywalking.oap.server.library.pprof.type.Frame; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; public class PprofMergeBuilder { private final Frame root = new Frame("root"); diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java index 9fe8ab776b3f..2475a1969477 100644 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java @@ -19,7 +19,6 @@ package org.apache.skywalking.oap.server.library.pprof.parser; import com.google.perftools.profiles.ProfileProto; -import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -33,7 +32,7 @@ * Parses pprof protobuf format files and converts them to frame trees. */ public class PprofParser { - + public static FrameTree dumpTree(ByteBuffer buf) throws IOException { byte[] bytes = new byte[buf.remaining()]; buf.get(bytes); @@ -42,7 +41,6 @@ public static FrameTree dumpTree(ByteBuffer buf) throws IOException { ProfileProto.Profile profile = ProfileProto.Profile.parseFrom(inputStream); FrameTree tree = new FrameTreeBuilder(profile).build(); return tree; - } public static FrameTree dumpTree(String filePath) throws IOException { diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java index 194829d9262b..34a050cee0b2 100644 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java @@ -28,8 +28,7 @@ public class Frame extends HashMap { final String signature; long total; long self; - - + public Frame(String signature) { this.signature = signature; } diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java index 9a2f240ccdd3..3b038a7e7559 100755 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.List; import lombok.Getter; + @Getter public class FrameTree { private String signature; @@ -34,6 +35,7 @@ public FrameTree(Frame frame) { this.self = frame.getSelf(); this.children = new ArrayList<>(frame.size()); } + public FrameTree(String signature, long total, long self) { this.signature = signature; this.total = total; @@ -42,7 +44,8 @@ public FrameTree(String signature, long total, long self) { } public static FrameTree buildTree(Frame frame) { - if (frame == null) return null; + if (frame == null) + return null; FrameTree frameTree = new FrameTree(frame); // has children? diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java index c42f504ad8a2..43c7d47f64de 100644 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java @@ -19,15 +19,15 @@ package org.apache.skywalking.oap.server.library.pprof.type; import com.google.perftools.profiles.ProfileProto; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; @Data @NoArgsConstructor @@ -56,6 +56,7 @@ private FrameTree parseTree(RawFrameTree rawTree) { } return tree; } + private String getSignature(long locationId) { if (locationId == 0) { return "root"; diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/PprofModuleProvider.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/PprofModuleProvider.java index 0ff1aacad79e..9c8d4adb8b20 100644 --- a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/PprofModuleProvider.java +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/PprofModuleProvider.java @@ -66,10 +66,10 @@ public void prepare() throws ServiceNotProvidedException, ModuleStartException { @Override public void start() throws ServiceNotProvidedException, ModuleStartException { GRPCHandlerRegister grpcHandlerRegister = getManager().find(SharingServerModule.NAME) - .provider() - .getService(GRPCHandlerRegister.class); - PprofServiceHandler pprofServiceHandler = new PprofServiceHandler(getManager(), - config.getPprofMaxSize(), config.isMemoryParserEnabled()); + .provider() + .getService(GRPCHandlerRegister.class); + PprofServiceHandler pprofServiceHandler = new PprofServiceHandler( + getManager(), config.getPprofMaxSize(), config.isMemoryParserEnabled()); grpcHandlerRegister.addHandler(pprofServiceHandler); } @@ -79,10 +79,10 @@ public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleSta @Override public String[] requiredModules() { - return new String[]{ - CoreModule.NAME, - SharingServerModule.NAME + return new String[] { + CoreModule.NAME, + SharingServerModule.NAME }; } - + } diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java index 6fdf30538cd0..ab80395534eb 100644 --- a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java @@ -19,36 +19,31 @@ package org.apache.skywalking.oap.server.receiver.pprof.provider.handler; import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; import java.io.IOException; -import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; -import org.apache.skywalking.apm.network.pprof.v10.PprofData; -import org.apache.skywalking.apm.network.pprof.v10.PprofTaskGrpc; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.apm.network.common.v3.Commands; import org.apache.skywalking.apm.network.pprof.v10.PprofCollectionResponse; +import org.apache.skywalking.apm.network.pprof.v10.PprofData; import org.apache.skywalking.apm.network.pprof.v10.PprofMetaData; - -import org.apache.skywalking.apm.network.common.v3.Commands; import org.apache.skywalking.apm.network.pprof.v10.PprofTaskCommandQuery; - +import org.apache.skywalking.apm.network.pprof.v10.PprofTaskGrpc; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.cache.PprofTaskCache; import org.apache.skywalking.oap.server.core.command.CommandService; -import java.util.Objects; -import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskLogRecord; import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; import org.apache.skywalking.oap.server.core.source.SourceReceiver; -import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.StorageModule; -import org.apache.skywalking.oap.server.core.cache.PprofTaskCache; -import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskLogRecord; -import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; -import org.apache.skywalking.oap.server.core.analysis.TimeBucket; -import java.util.concurrent.TimeUnit; - -//import org.apache.skywalking.oap.server.core.storage.StorageModule; -//import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.apache.skywalking.oap.server.library.server.grpc.GRPCHandler; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; import org.apache.skywalking.oap.server.network.trace.component.command.PprofTaskCommand; import org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream.PprofByteBufCollectionObserver; import org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream.PprofCollectionMetaData; @@ -72,12 +67,12 @@ public PprofServiceHandler(ModuleManager moduleManager, int pprofMaxSize, boolea this.pprofMaxSize = pprofMaxSize; this.memoryParserEnabled = memoryParserEnabled; } - + @Override public StreamObserver collect(StreamObserver responseObserver) { - return memoryParserEnabled ? - new PprofByteBufCollectionObserver(taskDAO, responseObserver, sourceReceiver, pprofMaxSize) - : new PprofFileCollectionObserver(taskDAO, responseObserver, sourceReceiver, pprofMaxSize); + return memoryParserEnabled ? new PprofByteBufCollectionObserver( + taskDAO, responseObserver, sourceReceiver, pprofMaxSize) : new PprofFileCollectionObserver( + taskDAO, responseObserver, sourceReceiver, pprofMaxSize); } @Override @@ -86,8 +81,8 @@ public void getPprofTaskCommands(PprofTaskCommandQuery request, StreamObserver private PprofCollectionMetaData taskMetaData; private ByteBuffer buf; - public PprofByteBufCollectionObserver(IPprofTaskQueryDAO taskDAO, - StreamObserver responseObserver, - SourceReceiver sourceReceiver, int pprofMaxSize) { + public PprofByteBufCollectionObserver(IPprofTaskQueryDAO taskDAO, + StreamObserver responseObserver, + SourceReceiver sourceReceiver, int pprofMaxSize) { this.taskDAO = taskDAO; this.responseObserver = responseObserver; this.sourceReceiver = sourceReceiver; @@ -61,35 +62,45 @@ public void onNext(PprofData pprofData) { try { if (Objects.isNull(taskMetaData) && pprofData.hasMetadata()) { taskMetaData = parseMetaData(pprofData.getMetadata(), taskDAO); - if (PprofProfilingStatus.PPROF_PROFILING_SUCCESS.equals(taskMetaData.getType())) { - int size = taskMetaData.getContentSize(); - if (pprofMaxSize >= size) { - buf = ByteBuffer.allocate(size); - // Send success response to allow client to continue uploading - responseObserver.onNext(PprofCollectionResponse.newBuilder() - .setStatus(PprofProfilingStatus.PPROF_PROFILING_SUCCESS) - .build()); + if (PprofProfilingStatus.PPROF_PROFILING_SUCCESS.equals(taskMetaData.getType())) { + int size = taskMetaData.getContentSize(); + if (pprofMaxSize >= size) { + buf = ByteBuffer.allocate(size); + // Send success response to allow client to continue uploading + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus( + PprofProfilingStatus.PPROF_PROFILING_SUCCESS) + .build()); + } else { + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus( + PprofProfilingStatus.PPROF_TERMINATED_BY_OVERSIZE) + .build()); + recordPprofTaskLog( + taskMetaData.getTask(), taskMetaData.getInstanceId(), + PprofTaskLogOperationType.PPROF_UPLOAD_FILE_TOO_LARGE_ERROR + ); + } } else { responseObserver.onNext(PprofCollectionResponse.newBuilder() - .setStatus(PprofProfilingStatus.PPROF_TERMINATED_BY_OVERSIZE) - .build()); - recordPprofTaskLog(taskMetaData.getTask(), taskMetaData.getInstanceId(), PprofTaskLogOperationType.PPROF_UPLOAD_FILE_TOO_LARGE_ERROR); + .setStatus( + PprofProfilingStatus.PPROF_EXECUTION_TASK_ERROR) + .build()); + recordPprofTaskLog( + taskMetaData.getTask(), taskMetaData.getInstanceId(), + PprofTaskLogOperationType.EXECUTION_TASK_ERROR + ); + } + } else if (pprofData.hasContent()) { + if (buf != null) { + pprofData.getContent().copyTo(buf); + log.info("Received {} bytes of pprof data", pprofData.getContent().size()); } - } else { - responseObserver.onNext(PprofCollectionResponse.newBuilder() - .setStatus(PprofProfilingStatus.PPROF_EXECUTION_TASK_ERROR) - .build()); - recordPprofTaskLog(taskMetaData.getTask(), taskMetaData.getInstanceId(), PprofTaskLogOperationType.EXECUTION_TASK_ERROR); - } - } else if (pprofData.hasContent()) { - if (buf != null) { - pprofData.getContent().copyTo(buf); - log.info("Received {} bytes of pprof data", pprofData.getContent().size()); } - } } catch (IOException e) { log.error("Error processing pprof data", e); - responseObserver.onError(Status.INTERNAL.withDescription("Error processing pprof data: " + e.getMessage()).asRuntimeException()); + responseObserver.onError( + Status.INTERNAL.withDescription("Error processing pprof data: " + e.getMessage()).asRuntimeException()); } } @@ -103,7 +114,7 @@ public void onError(Throwable throwable) { return; } log.error("Error in receiving pprof profiling data", throwable); - + } @Override @@ -122,7 +133,10 @@ public void onCompleted() { private void parseAndStorageData(PprofCollectionMetaData taskMetaData, ByteBuffer buf) throws IOException { PprofTask task = taskMetaData.getTask(); if (task == null) { - log.error("Pprof instanceId:{} has not been assigned a task but still uploaded data", taskMetaData.getInstanceId()); + log.error( + "Pprof instanceId:{} has not been assigned a task but still uploaded data", + taskMetaData.getInstanceId() + ); return; } recordPprofTaskLog(task, taskMetaData.getInstanceId(), PprofTaskLogOperationType.EXECUTION_FINISHED); diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofCollectionMetaData.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofCollectionMetaData.java index b0986361d97c..8a5c0680ffda 100644 --- a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofCollectionMetaData.java +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofCollectionMetaData.java @@ -18,11 +18,10 @@ package org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream; -import org.apache.skywalking.oap.server.core.query.type.PprofTask; - -import org.apache.skywalking.apm.network.pprof.v10.PprofProfilingStatus; import lombok.Builder; import lombok.Data; +import org.apache.skywalking.apm.network.pprof.v10.PprofProfilingStatus; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; @Data @Builder diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java index 9ae09a6d97b3..f197d1af6f08 100644 --- a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java @@ -20,26 +20,27 @@ import io.grpc.Status; import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; import lombok.SneakyThrows; -import org.apache.skywalking.apm.network.pprof.v10.PprofData; +import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.apm.network.pprof.v10.PprofCollectionResponse; +import org.apache.skywalking.apm.network.pprof.v10.PprofData; import org.apache.skywalking.apm.network.pprof.v10.PprofProfilingStatus; -import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; import org.apache.skywalking.oap.server.core.query.type.PprofTask; -import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; import org.apache.skywalking.oap.server.core.source.PprofProfilingData; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; import org.apache.skywalking.oap.server.library.pprof.parser.PprofParser; import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; -import org.apache.skywalking.oap.server.core.query.type.PprofEventType; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Objects; -import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; -import static org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler.recordPprofTaskLog; + import static org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler.parseMetaData; +import static org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler.recordPprofTaskLog; @Slf4j public class PprofFileCollectionObserver implements StreamObserver { @@ -51,9 +52,9 @@ public class PprofFileCollectionObserver implements StreamObserver { private Path tempFile; private FileOutputStream fileOutputStream; - public PprofFileCollectionObserver(IPprofTaskQueryDAO taskDAO, - StreamObserver responseObserver, - SourceReceiver sourceReceiver, int pprofMaxSize) { + public PprofFileCollectionObserver(IPprofTaskQueryDAO taskDAO, + StreamObserver responseObserver, + SourceReceiver sourceReceiver, int pprofMaxSize) { this.taskDAO = taskDAO; this.responseObserver = responseObserver; this.sourceReceiver = sourceReceiver; @@ -65,34 +66,46 @@ public PprofFileCollectionObserver(IPprofTaskQueryDAO taskDAO, public void onNext(PprofData pprofData) { if (Objects.isNull(taskMetaData) && pprofData.hasMetadata()) { taskMetaData = parseMetaData(pprofData.getMetadata(), taskDAO); - + if (PprofProfilingStatus.PPROF_PROFILING_SUCCESS.equals(taskMetaData.getType())) { int size = taskMetaData.getContentSize(); if (pprofMaxSize >= size) { // Create temporary file for pprof data - tempFile = Files.createTempFile(taskMetaData.getTask().getId() + taskMetaData.getInstanceId() + System.currentTimeMillis(), ".pprof"); + tempFile = Files.createTempFile( + taskMetaData.getTask().getId() + taskMetaData.getInstanceId() + System.currentTimeMillis(), + ".pprof" + ); fileOutputStream = new FileOutputStream(tempFile.toFile()); - + // Send success response to allow client to continue uploading responseObserver.onNext(PprofCollectionResponse.newBuilder() - .setStatus(PprofProfilingStatus.PPROF_PROFILING_SUCCESS) - .build()); + .setStatus( + PprofProfilingStatus.PPROF_PROFILING_SUCCESS) + .build()); } else { responseObserver.onNext(PprofCollectionResponse.newBuilder() - .setStatus(PprofProfilingStatus.PPROF_TERMINATED_BY_OVERSIZE) - .build()); - recordPprofTaskLog(taskMetaData.getTask(), taskMetaData.getInstanceId(), PprofTaskLogOperationType.PPROF_UPLOAD_FILE_TOO_LARGE_ERROR); + .setStatus( + PprofProfilingStatus.PPROF_TERMINATED_BY_OVERSIZE) + .build()); + recordPprofTaskLog( + taskMetaData.getTask(), taskMetaData.getInstanceId(), + PprofTaskLogOperationType.PPROF_UPLOAD_FILE_TOO_LARGE_ERROR + ); } } else { responseObserver.onNext(PprofCollectionResponse.newBuilder() - .setStatus(PprofProfilingStatus.PPROF_EXECUTION_TASK_ERROR) - .build()); - recordPprofTaskLog(taskMetaData.getTask(), taskMetaData.getInstanceId(), PprofTaskLogOperationType.EXECUTION_TASK_ERROR); + .setStatus( + PprofProfilingStatus.PPROF_EXECUTION_TASK_ERROR) + .build()); + recordPprofTaskLog( + taskMetaData.getTask(), taskMetaData.getInstanceId(), + PprofTaskLogOperationType.EXECUTION_TASK_ERROR + ); } } else if (pprofData.hasContent()) { if (fileOutputStream != null) { fileOutputStream.write(pprofData.getContent().toByteArray()); - + if (log.isDebugEnabled()) { log.debug("Received {} bytes of pprof data", pprofData.getContent().size()); } @@ -110,7 +123,7 @@ public void onError(Throwable throwable) { } else { log.error("Error in receiving pprof profiling data", throwable); } - + // Clean up resources closeFileStream(); } @@ -119,7 +132,7 @@ public void onError(Throwable throwable) { @SneakyThrows public void onCompleted() { responseObserver.onCompleted(); - + if (Objects.nonNull(tempFile)) { closeFileStream(); parseAndStorageData(taskMetaData, tempFile.toAbsolutePath().toString()); @@ -141,23 +154,26 @@ private void closeFileStream() { private void parseAndStorageData(PprofCollectionMetaData taskMetaData, String fileName) { PprofTask task = taskMetaData.getTask(); if (task == null) { - log.error("Pprof instanceId:{} has not been assigned a task but still uploaded data", taskMetaData.getInstanceId()); + log.error( + "Pprof instanceId:{} has not been assigned a task but still uploaded data", + taskMetaData.getInstanceId() + ); return; } recordPprofTaskLog(task, taskMetaData.getInstanceId(), PprofTaskLogOperationType.EXECUTION_FINISHED); parsePprofAndStorage(taskMetaData, fileName); } - public void parsePprofAndStorage(PprofCollectionMetaData taskMetaData, + public void parsePprofAndStorage(PprofCollectionMetaData taskMetaData, String fileName) throws IOException { - PprofTask task = taskMetaData.getTask(); - FrameTree tree = PprofParser.dumpTree(fileName); - PprofProfilingData data = new PprofProfilingData(); - data.setEventType(PprofEventType.valueOfString(task.getEvents().name())); - data.setFrameTree(tree); - data.setTaskId(task.getId()); - data.setInstanceId(taskMetaData.getInstanceId()); - data.setUploadTime(taskMetaData.getUploadTime()); - sourceReceiver.receive(data); + PprofTask task = taskMetaData.getTask(); + FrameTree tree = PprofParser.dumpTree(fileName); + PprofProfilingData data = new PprofProfilingData(); + data.setEventType(PprofEventType.valueOfString(task.getEvents().name())); + data.setFrameTree(tree); + data.setTaskId(task.getId()); + data.setInstanceId(taskMetaData.getInstanceId()); + data.setUploadTime(taskMetaData.getUploadTime()); + sourceReceiver.receive(data); } } diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java index cdb573a9a8ef..3cd01466165c 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java @@ -19,26 +19,26 @@ package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream; import com.google.common.collect.ImmutableSet; -import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; -import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; -import java.util.Set; -import java.util.List; import java.io.IOException; import java.util.ArrayList; -import org.apache.skywalking.oap.server.library.util.StringUtil; -import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient; +import java.util.List; +import java.util.Set; +import org.apache.skywalking.banyandb.v1.client.RowEntity; import org.apache.skywalking.banyandb.v1.client.StreamQuery; import org.apache.skywalking.banyandb.v1.client.StreamQueryResponse; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; import org.apache.skywalking.oap.server.library.util.CollectionUtils; -import org.apache.skywalking.banyandb.v1.client.RowEntity; +import org.apache.skywalking.oap.server.library.util.StringUtil; import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBConverter; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient; public class BanyanDBPprofDataQueryDAO extends AbstractBanyanDBDAO implements IPprofDataQueryDAO { private static final Set TAGS = ImmutableSet.of( - PprofProfilingDataRecord.TASK_ID, - PprofProfilingDataRecord.INSTANCE_ID, - PprofProfilingDataRecord.UPLOAD_TIME, - PprofProfilingDataRecord.DATA_BINARY + PprofProfilingDataRecord.TASK_ID, + PprofProfilingDataRecord.INSTANCE_ID, + PprofProfilingDataRecord.UPLOAD_TIME, + PprofProfilingDataRecord.DATA_BINARY ); public BanyanDBPprofDataQueryDAO(BanyanDBStorageClient client) { @@ -46,20 +46,23 @@ public BanyanDBPprofDataQueryDAO(BanyanDBStorageClient client) { } @Override - public List getByTaskIdAndInstances(String taskId, List instanceIds) throws IOException { + public List getByTaskIdAndInstances(String taskId, + List instanceIds) throws IOException { if (StringUtil.isBlank(taskId)) { return new ArrayList<>(); } - StreamQueryResponse resp = query(false, PprofProfilingDataRecord.INDEX_NAME, TAGS, - new QueryBuilder() { - @Override - protected void apply(StreamQuery query) { - query.and(eq(PprofProfilingDataRecord.TASK_ID, taskId)); - if (CollectionUtils.isNotEmpty(instanceIds)) { - query.and(in(PprofProfilingDataRecord.INSTANCE_ID, instanceIds)); - } + StreamQueryResponse resp = query( + false, PprofProfilingDataRecord.INDEX_NAME, TAGS, + new QueryBuilder() { + @Override + protected void apply(StreamQuery query) { + query.and(eq(PprofProfilingDataRecord.TASK_ID, taskId)); + if (CollectionUtils.isNotEmpty(instanceIds)) { + query.and(in(PprofProfilingDataRecord.INSTANCE_ID, instanceIds)); } - }); + } + } + ); List records = new ArrayList<>(resp.size()); for (final RowEntity entity : resp.getElements()) { records.add(buildProfilingDataRecord(entity)); @@ -70,7 +73,8 @@ protected void apply(StreamQuery query) { private PprofProfilingDataRecord buildProfilingDataRecord(RowEntity entity) { final PprofProfilingDataRecord.Builder builder = new PprofProfilingDataRecord.Builder(); - BanyanDBConverter.StorageToStream storageToStream = new BanyanDBConverter.StorageToStream(PprofProfilingDataRecord.INDEX_NAME, entity); + BanyanDBConverter.StorageToStream storageToStream = new BanyanDBConverter.StorageToStream( + PprofProfilingDataRecord.INDEX_NAME, entity); return builder.storage2Entity(storageToStream); } } diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskLogQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskLogQueryDAO.java index cfd9d4d42daf..0f82af2f267c 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskLogQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskLogQueryDAO.java @@ -19,6 +19,10 @@ package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream; import com.google.common.collect.ImmutableSet; +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; import org.apache.skywalking.banyandb.v1.client.Element; import org.apache.skywalking.banyandb.v1.client.StreamQuery; import org.apache.skywalking.banyandb.v1.client.StreamQueryResponse; @@ -28,20 +32,15 @@ import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient; -import java.io.IOException; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; - /** * {@link PprofTaskLogRecord} is a stream */ public class BanyanDBPprofTaskLogQueryDAO extends AbstractBanyanDBDAO implements IPprofTaskLogQueryDAO { private static final Set TAGS = ImmutableSet.of( - PprofTaskLogRecord.OPERATION_TIME, - PprofTaskLogRecord.INSTANCE_ID, - PprofTaskLogRecord.TASK_ID, - PprofTaskLogRecord.OPERATION_TYPE + PprofTaskLogRecord.OPERATION_TIME, + PprofTaskLogRecord.INSTANCE_ID, + PprofTaskLogRecord.TASK_ID, + PprofTaskLogRecord.OPERATION_TYPE ); private final int queryMaxSize; @@ -54,13 +53,15 @@ public BanyanDBPprofTaskLogQueryDAO(BanyanDBStorageClient client, int taskQueryM @Override public List getTaskLogList() throws IOException { - StreamQueryResponse resp = query(false, PprofTaskLogRecord.INDEX_NAME, TAGS, - new QueryBuilder() { - @Override - public void apply(StreamQuery query) { - query.setLimit(BanyanDBPprofTaskLogQueryDAO.this.queryMaxSize); - } - }); + StreamQueryResponse resp = query( + false, PprofTaskLogRecord.INDEX_NAME, TAGS, + new QueryBuilder() { + @Override + public void apply(StreamQuery query) { + query.setLimit(BanyanDBPprofTaskLogQueryDAO.this.queryMaxSize); + } + } + ); final LinkedList tasks = new LinkedList<>(); for (final Element element : resp.getElements()) { @@ -73,10 +74,10 @@ private PprofTaskLog buildPprofTaskLog(Element data) { int operationTypeInt = ((Number) data.getTagValue(PprofTaskLogRecord.OPERATION_TYPE)).intValue(); PprofTaskLogOperationType operationType = PprofTaskLogOperationType.parse(operationTypeInt); return PprofTaskLog.builder() - .id(data.getTagValue(PprofTaskLogRecord.TASK_ID)) - .instanceId(data.getTagValue(PprofTaskLogRecord.INSTANCE_ID)) - .operationType(operationType) - .operationTime(((Number) data.getTagValue(PprofTaskLogRecord.OPERATION_TIME)).longValue()) - .build(); + .id(data.getTagValue(PprofTaskLogRecord.TASK_ID)) + .instanceId(data.getTagValue(PprofTaskLogRecord.INSTANCE_ID)) + .operationType(operationType) + .operationTime(((Number) data.getTagValue(PprofTaskLogRecord.OPERATION_TIME)).longValue()) + .build(); } } diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java index 58488382f6f4..6a2332faa345 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java @@ -21,6 +21,12 @@ import com.google.common.collect.ImmutableSet; import com.google.common.reflect.TypeToken; import com.google.gson.Gson; +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.banyandb.v1.client.AbstractQuery; import org.apache.skywalking.banyandb.v1.client.RowEntity; import org.apache.skywalking.banyandb.v1.client.StreamQuery; @@ -28,17 +34,11 @@ import org.apache.skywalking.banyandb.v1.client.TimestampRange; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; -import org.apache.skywalking.oap.server.core.query.type.PprofTask; import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; import org.apache.skywalking.oap.server.library.util.StringUtil; import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient; -import lombok.extern.slf4j.Slf4j; -import java.io.IOException; -import java.lang.reflect.Type; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; @Slf4j public class BanyanDBPprofTaskQueryDAO extends AbstractBanyanDBDAO implements IPprofTaskQueryDAO { diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java index 2dc2b03d7be1..77ea0c55bb7d 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java @@ -17,26 +17,26 @@ * under the License. */ - package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; -import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; -import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO; -import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; -import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; -import org.apache.skywalking.oap.server.library.util.StringUtil; -import org.apache.skywalking.oap.server.library.util.CollectionUtils; -import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; +import com.google.common.collect.Lists; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; import org.apache.skywalking.library.elasticsearch.requests.search.Query; -import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; import org.apache.skywalking.library.elasticsearch.requests.search.Search; +import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; -import com.google.common.collect.Lists; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.oap.server.library.util.StringUtil; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.ElasticSearchConverter; -import java.util.Map; -import java.util.ArrayList; -import java.util.List; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; public class PprofDataQueryEsDAO extends EsDAO implements IPprofDataQueryDAO { public PprofDataQueryEsDAO(ElasticSearchClient client) { @@ -48,10 +48,14 @@ public List getByTaskIdAndInstances(String taskId, Lis if (StringUtil.isBlank(taskId) || CollectionUtils.isEmpty(instanceIds)) { return new ArrayList<>(); } - final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofProfilingDataRecord.INDEX_NAME); + final String index = IndexController.LogicIndicesRegister.getPhysicalTableName( + PprofProfilingDataRecord.INDEX_NAME); final BoolQueryBuilder query = Query.bool(); if (IndexController.LogicIndicesRegister.isMergedTable(PprofProfilingDataRecord.INDEX_NAME)) { - query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofProfilingDataRecord.INDEX_NAME)); + query.must(Query.term( + IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, + PprofProfilingDataRecord.INDEX_NAME + )); } query.must(Query.term(PprofProfilingDataRecord.TASK_ID, taskId)); query.must(Query.terms(PprofProfilingDataRecord.INSTANCE_ID, instanceIds)); @@ -67,6 +71,7 @@ public List getByTaskIdAndInstances(String taskId, Lis private PprofProfilingDataRecord parseData(SearchHit data) { final Map sourceAsMap = data.getSource(); final PprofProfilingDataRecord.Builder builder = new PprofProfilingDataRecord.Builder(); - return builder.storage2Entity(new ElasticSearchConverter.ToEntity(PprofProfilingDataRecord.INDEX_NAME, sourceAsMap)); + return builder.storage2Entity( + new ElasticSearchConverter.ToEntity(PprofProfilingDataRecord.INDEX_NAME, sourceAsMap)); } } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java index 87f20d4498bf..355e8b73bc88 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java @@ -22,7 +22,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; - import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; import org.apache.skywalking.library.elasticsearch.requests.search.Query; import org.apache.skywalking.library.elasticsearch.requests.search.Search; diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java index 555e39cabe4e..719cd064d930 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java @@ -18,16 +18,14 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; import java.io.IOException; +import java.lang.reflect.Type; import java.util.LinkedList; import java.util.List; import java.util.Map; -import com.google.gson.Gson; import java.util.Objects; -import com.google.gson.reflect.TypeToken; -import java.lang.reflect.Type; - -import org.apache.skywalking.oap.server.library.util.StringUtil; import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; import org.apache.skywalking.library.elasticsearch.requests.search.Query; import org.apache.skywalking.library.elasticsearch.requests.search.Search; @@ -36,10 +34,11 @@ import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; -import org.apache.skywalking.oap.server.core.query.type.PprofTask; import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.library.util.StringUtil; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; @@ -54,7 +53,10 @@ public PprofTaskQueryEsDAO(ElasticSearchClient client, int queryMaxSize) { } @Override - public List getTaskList(String serviceId, Long startTimeBucket, Long endTimeBucket, Integer limit) throws IOException { + public List getTaskList(String serviceId, + Long startTimeBucket, + Long endTimeBucket, + Integer limit) throws IOException { String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofTaskRecord.INDEX_NAME); BoolQueryBuilder query = Query.bool(); if (IndexController.LogicIndicesRegister.isMergedTable(PprofTaskRecord.INDEX_NAME)) { @@ -116,13 +118,13 @@ private PprofTask parseTask(SearchHit data) { List instanceIdList = GSON.fromJson(serviceInstanceIds, listType); return PprofTask.builder() - .id((String) source.get(PprofTaskRecord.TASK_ID)) - .serviceId((String) source.get(PprofTaskRecord.SERVICE_ID)) - .serviceInstanceIds(instanceIdList) - .createTime(((Number) source.get(PprofTaskRecord.CREATE_TIME)).longValue()) - .events(PprofEventType.valueOfString((String) source.get(PprofTaskRecord.EVENT_TYPES))) - .duration(((Number) source.get(PprofTaskRecord.DURATION)).intValue()) - .dumpPeriod(((Number) source.get(PprofTaskRecord.DUMP_PERIOD)).intValue()) - .build(); + .id((String) source.get(PprofTaskRecord.TASK_ID)) + .serviceId((String) source.get(PprofTaskRecord.SERVICE_ID)) + .serviceInstanceIds(instanceIdList) + .createTime(((Number) source.get(PprofTaskRecord.CREATE_TIME)).longValue()) + .events(PprofEventType.valueOfString((String) source.get(PprofTaskRecord.EVENT_TYPES))) + .duration(((Number) source.get(PprofTaskRecord.DURATION)).intValue()) + .dumpPeriod(((Number) source.get(PprofTaskRecord.DUMP_PERIOD)).intValue()) + .build(); } } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java index 86899febe75b..f391f050f5cf 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java @@ -17,76 +17,77 @@ * under the License. */ - package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; - import lombok.RequiredArgsConstructor; - import lombok.SneakyThrows; - import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; - import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; - import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCClient; - import org.apache.skywalking.oap.server.library.util.CollectionUtils; - import org.apache.skywalking.oap.server.library.util.StringUtil; - import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCEntityConverters; - import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCTableInstaller; - import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.TableHelper; - - import java.io.IOException; - import java.sql.ResultSet; - import java.util.ArrayList; - import java.util.Base64; - import java.util.List; - - @RequiredArgsConstructor - public class JDBCPprofDataQueryDAO implements IPprofDataQueryDAO { - private final JDBCClient jdbcClient; - private final TableHelper tableHelper; - - @Override - @SneakyThrows - public List getByTaskIdAndInstances(String taskId, List instanceIds) throws IOException { - if (StringUtil.isBlank(taskId)) { - return new ArrayList<>(); - } - List tables = tableHelper.getTablesWithinTTL(PprofProfilingDataRecord.INDEX_NAME); - List results = new ArrayList<>(); - for (final var table : tables) { - List condition = new ArrayList<>(4); - StringBuilder sql = new StringBuilder() - .append("select * from ").append(table) - .append(" where ").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); - condition.add(PprofProfilingDataRecord.INDEX_NAME); - - if (CollectionUtils.isNotEmpty(instanceIds)) { - sql.append(" and ").append(PprofProfilingDataRecord.INSTANCE_ID).append(" in (?) "); - String joinedInstanceIds = String.join(",", instanceIds); - condition.add(joinedInstanceIds); - } - - results.addAll( - jdbcClient.executeQuery( - sql.toString(), - resultSet -> { - final var result = new ArrayList(); - while (resultSet.next()) { - result.add(parseData(resultSet)); - } - return result; - }, - condition.toArray(new Object[0])) - ); - } - return results; - } - - private PprofProfilingDataRecord parseData(ResultSet data) { - final PprofProfilingDataRecord.Builder builder = new PprofProfilingDataRecord.Builder(); - PprofProfilingDataRecord pprofProfilingDataRecord = builder.storage2Entity(JDBCEntityConverters.toEntity(data)); - byte[] dataBinary = pprofProfilingDataRecord.getDataBinary(); - if (dataBinary != null) { - byte[] decodeResult = Base64.getDecoder().decode(dataBinary); - pprofProfilingDataRecord.setDataBinary(decodeResult); - } - return pprofProfilingDataRecord; - } - } +import java.io.IOException; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; +import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCClient; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCEntityConverters; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCTableInstaller; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.TableHelper; + +@RequiredArgsConstructor +public class JDBCPprofDataQueryDAO implements IPprofDataQueryDAO { + private final JDBCClient jdbcClient; + private final TableHelper tableHelper; + + @Override + @SneakyThrows + public List getByTaskIdAndInstances(String taskId, + List instanceIds) throws IOException { + if (StringUtil.isBlank(taskId)) { + return new ArrayList<>(); + } + List tables = tableHelper.getTablesWithinTTL(PprofProfilingDataRecord.INDEX_NAME); + List results = new ArrayList<>(); + for (final var table : tables) { + List condition = new ArrayList<>(4); + StringBuilder sql = new StringBuilder() + .append("select * from ").append(table) + .append(" where ").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); + condition.add(PprofProfilingDataRecord.INDEX_NAME); + + if (CollectionUtils.isNotEmpty(instanceIds)) { + sql.append(" and ").append(PprofProfilingDataRecord.INSTANCE_ID).append(" in (?) "); + String joinedInstanceIds = String.join(",", instanceIds); + condition.add(joinedInstanceIds); + } + + results.addAll( + jdbcClient.executeQuery( + sql.toString(), + resultSet -> { + final var result = new ArrayList(); + while (resultSet.next()) { + result.add(parseData(resultSet)); + } + return result; + }, + condition.toArray(new Object[0]) + ) + ); + } + return results; + } + + private PprofProfilingDataRecord parseData(ResultSet data) { + final PprofProfilingDataRecord.Builder builder = new PprofProfilingDataRecord.Builder(); + PprofProfilingDataRecord pprofProfilingDataRecord = builder.storage2Entity(JDBCEntityConverters.toEntity(data)); + byte[] dataBinary = pprofProfilingDataRecord.getDataBinary(); + if (dataBinary != null) { + byte[] decodeResult = Base64.getDecoder().decode(dataBinary); + pprofProfilingDataRecord.setDataBinary(decodeResult); + } + return pprofProfilingDataRecord; + } +} \ No newline at end of file diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskLogQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskLogQueryDAO.java index ea6da7dd81d1..0db85eca5035 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskLogQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskLogQueryDAO.java @@ -17,8 +17,12 @@ * under the License. */ - package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskLogRecord; @@ -30,11 +34,6 @@ import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.SQLAndParameters; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.TableHelper; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - @RequiredArgsConstructor public class JDBCPprofTaskLogQueryDAO implements IPprofTaskLogQueryDAO { private final JDBCClient jdbcClient; @@ -48,15 +47,16 @@ public List getTaskLogList() { for (String table : tables) { SQLAndParameters sqlAndParameters = buildSQL(table); List logs = jdbcClient.executeQuery( - sqlAndParameters.sql(), - resultSet -> { - final List tasks = new ArrayList<>(); - while (resultSet.next()) { - tasks.add(parseLog(resultSet)); - } - return tasks; - }, - sqlAndParameters.parameters()); + sqlAndParameters.sql(), + resultSet -> { + final List tasks = new ArrayList<>(); + while (resultSet.next()) { + tasks.add(parseLog(resultSet)); + } + return tasks; + }, + sqlAndParameters.parameters() + ); results.addAll(logs); } return results; @@ -66,7 +66,7 @@ private SQLAndParameters buildSQL(String table) { StringBuilder sql = new StringBuilder(); List parameters = new ArrayList<>(2); sql.append("select * from ").append(table) - .append(" where ").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); + .append(" where ").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); parameters.add(PprofTaskLogRecord.INDEX_NAME); sql.append(" order by ").append(PprofTaskLogRecord.OPERATION_TIME).append(" desc"); return new SQLAndParameters(sql.toString(), parameters); @@ -74,10 +74,11 @@ private SQLAndParameters buildSQL(String table) { private PprofTaskLog parseLog(ResultSet data) throws SQLException { return PprofTaskLog.builder() - .id(data.getString(PprofTaskLogRecord.TASK_ID)) - .instanceId(data.getString(PprofTaskLogRecord.INSTANCE_ID)) - .operationType(PprofTaskLogOperationType.parse(data.getInt(PprofTaskLogRecord.OPERATION_TYPE))) - .operationTime(data.getLong(PprofTaskLogRecord.OPERATION_TIME)) - .build(); + .id(data.getString(PprofTaskLogRecord.TASK_ID)) + .instanceId(data.getString(PprofTaskLogRecord.INSTANCE_ID)) + .operationType( + PprofTaskLogOperationType.parse(data.getInt(PprofTaskLogRecord.OPERATION_TYPE))) + .operationTime(data.getLong(PprofTaskLogRecord.OPERATION_TIME)) + .build(); } } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskQueryDAO.java index 645accbb4d42..1dd991a25279 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskQueryDAO.java @@ -21,6 +21,12 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; +import java.io.IOException; +import java.lang.reflect.Type; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; @@ -32,13 +38,6 @@ import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCTableInstaller; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.TableHelper; -import java.io.IOException; -import java.lang.reflect.Type; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - import static java.util.stream.Collectors.toList; @RequiredArgsConstructor @@ -50,16 +49,19 @@ public class JDBCPprofTaskQueryDAO implements IPprofTaskQueryDAO { @Override @SneakyThrows - public List getTaskList(String serviceId, Long startTimeBucket, Long endTimeBucket, Integer limit) throws IOException { + public List getTaskList(String serviceId, + Long startTimeBucket, + Long endTimeBucket, + Integer limit) throws IOException { final var results = new ArrayList(); final var tables = startTimeBucket == null || endTimeBucket == null ? - tableHelper.getTablesWithinTTL(PprofTaskRecord.INDEX_NAME) : - tableHelper.getTablesForRead(PprofTaskRecord.INDEX_NAME, startTimeBucket, endTimeBucket); + tableHelper.getTablesWithinTTL(PprofTaskRecord.INDEX_NAME) : + tableHelper.getTablesForRead(PprofTaskRecord.INDEX_NAME, startTimeBucket, endTimeBucket); for (final var table : tables) { List condition = new ArrayList<>(4); StringBuilder sql = new StringBuilder() - .append("select * from ").append(table) - .append(" where ").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); + .append("select * from ").append(table) + .append(" where ").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); condition.add(PprofTaskRecord.INDEX_NAME); if (StringUtil.isNotEmpty(serviceId)) { @@ -84,24 +86,25 @@ public List getTaskList(String serviceId, Long startTimeBucket, Long } results.addAll( - jdbcClient.executeQuery( - sql.toString(), - resultSet -> { - final var tasks = new ArrayList(); - while (resultSet.next()) { - tasks.add(buildPprofTask(resultSet)); - } - return tasks; - }, - condition.toArray(new Object[0])) + jdbcClient.executeQuery( + sql.toString(), + resultSet -> { + final var tasks = new ArrayList(); + while (resultSet.next()) { + tasks.add(buildPprofTask(resultSet)); + } + return tasks; + }, + condition.toArray(new Object[0]) + ) ); } return limit == null ? - results : - results - .stream() - .limit(limit) - .collect(toList()); + results : + results + .stream() + .limit(limit) + .collect(toList()); } @Override @@ -112,22 +115,23 @@ public PprofTask getById(String id) throws IOException { final StringBuilder sql = new StringBuilder(); final List condition = new ArrayList<>(1); sql.append("select * from ").append(table) - .append(" where ") - .append(JDBCTableInstaller.TABLE_COLUMN).append(" = ? ") - .append(" and ") - .append(PprofTaskRecord.TASK_ID + "=? LIMIT 1"); + .append(" where ") + .append(JDBCTableInstaller.TABLE_COLUMN).append(" = ? ") + .append(" and ") + .append(PprofTaskRecord.TASK_ID + "=? LIMIT 1"); condition.add(PprofTaskRecord.INDEX_NAME); condition.add(id); final var r = jdbcClient.executeQuery( - sql.toString(), - resultSet -> { - if (resultSet.next()) { - return buildPprofTask(resultSet); - } - return null; - }, - condition.toArray(new Object[0])); + sql.toString(), + resultSet -> { + if (resultSet.next()) { + return buildPprofTask(resultSet); + } + return null; + }, + condition.toArray(new Object[0]) + ); if (r != null) { return r; } @@ -142,14 +146,14 @@ private PprofTask buildPprofTask(ResultSet data) throws SQLException { String serviceInstanceIds = data.getString(PprofTaskRecord.SERVICE_INSTANCE_IDS); List serviceInstanceIdList = GSON.fromJson(serviceInstanceIds, listType); return PprofTask.builder() - .id(data.getString(PprofTaskRecord.TASK_ID)) - .serviceId(data.getString(PprofTaskRecord.SERVICE_ID)) - .serviceInstanceIds(serviceInstanceIdList) - .createTime(data.getLong(PprofTaskRecord.CREATE_TIME)) - .duration(data.getInt(PprofTaskRecord.DURATION)) - .events(PprofEventType.valueOfString(events)) - .dumpPeriod(data.getInt(PprofTaskRecord.DUMP_PERIOD)) - .build(); + .id(data.getString(PprofTaskRecord.TASK_ID)) + .serviceId(data.getString(PprofTaskRecord.SERVICE_ID)) + .serviceInstanceIds(serviceInstanceIdList) + .createTime(data.getLong(PprofTaskRecord.CREATE_TIME)) + .duration(data.getInt(PprofTaskRecord.DURATION)) + .events(PprofEventType.valueOfString(events)) + .dumpPeriod(data.getInt(PprofTaskRecord.DUMP_PERIOD)) + .build(); } - + } From 3e4c261dfe0e53c2db3117c51d94983f9a85887f Mon Sep 17 00:00:00 2001 From: JophieQu Date: Tue, 21 Oct 2025 17:22:33 +0800 Subject: [PATCH 57/69] fix doc --- docs/en/setup/backend/backend-go-app-profiling.md | 3 +++ skywalking-ui | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/en/setup/backend/backend-go-app-profiling.md b/docs/en/setup/backend/backend-go-app-profiling.md index 1a00587d3041..271199281019 100644 --- a/docs/en/setup/backend/backend-go-app-profiling.md +++ b/docs/en/setup/backend/backend-go-app-profiling.md @@ -9,6 +9,9 @@ When service encounters performance issues (cpu usage, memory allocation, etc.), When the agent receives a task, it enables Pprof for sampling. After sampling is completed, the sampling results are analyzed by requesting the server to render a flame graph for performance analysis to determine the specific business code lines that cause performance problems. +Note, tracing profiling in the Go agent relies on the Go runtime’s global CPU sampling used by pprof. +Since only one CPU profiler can run at a time within the same instance, tracing and pprof CPU profiling cannot be enabled simultaneously. +If both are activated on the same instance, one task may fail to start. ## Activate Pprof in the OAP OAP and the agent use a brand-new protocol to exchange Pprof data, so it is necessary to start OAP with the following configuration: diff --git a/skywalking-ui b/skywalking-ui index 3cefbf1bd5d5..7111a2e764b6 160000 --- a/skywalking-ui +++ b/skywalking-ui @@ -1 +1 @@ -Subproject commit 3cefbf1bd5d588355829cbe9dd23cf2253af547e +Subproject commit 7111a2e764b66462d59bd6e92620ae0d5c545e21 From d3763bcff01ebf56e4cde937abe569b418838087 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Tue, 21 Oct 2025 17:24:17 +0800 Subject: [PATCH 58/69] roll back ui --- skywalking-ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skywalking-ui b/skywalking-ui index 7111a2e764b6..b710a0a589e2 160000 --- a/skywalking-ui +++ b/skywalking-ui @@ -1 +1 @@ -Subproject commit 7111a2e764b66462d59bd6e92620ae0d5c545e21 +Subproject commit b710a0a589e293b0028e3b525cbad51b2b09cf6b From e60e3d56dfd266504a3da497c786158ec16fd7ea Mon Sep 17 00:00:00 2001 From: JophieQu Date: Tue, 21 Oct 2025 17:28:10 +0800 Subject: [PATCH 59/69] roll back ci --- .github/workflows/skywalking.yaml | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index e50112abaf53..0bd11fca045c 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -17,9 +17,6 @@ name: CI on: - push: - branches: - - ci1 pull_request: schedule: - cron: "0 18 * * *" # TimeZone: UTC 0 @@ -35,7 +32,7 @@ env: jobs: license-header: - if: (github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || (github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') name: License header runs-on: ubuntu-latest timeout-minutes: 10 @@ -48,7 +45,7 @@ jobs: uses: apache/skywalking-eyes@5b7ee1731d036b5aac68f8bd3fc9e6f98ada082e code-style: - if: (github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || (github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') name: Code style runs-on: ubuntu-latest timeout-minutes: 10 @@ -63,7 +60,7 @@ jobs: dependency-license: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.pom == 'true' || needs.changes.outputs.ui == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.pom == 'true' || needs.changes.outputs.ui == 'true') name: Dependency licenses needs: [changes] runs-on: ubuntu-latest @@ -93,7 +90,7 @@ jobs: fi sanity-check: - if: ( always() && ! cancelled() ) && (github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || (github.event_name != 'schedule') + if: ( always() && ! cancelled() ) && (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') name: Sanity check results needs: [license-header, code-style, dependency-license] runs-on: ubuntu-latest @@ -162,7 +159,7 @@ jobs: dist-tar: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: Build dist tar needs: [changes] runs-on: ubuntu-latest @@ -194,7 +191,7 @@ jobs: docker: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: Docker images needs: [sanity-check, dist-tar, changes] runs-on: ubuntu-latest @@ -233,7 +230,7 @@ jobs: unit-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: Unit test needs: [sanity-check, changes] runs-on: ${{ matrix.os }} @@ -268,7 +265,7 @@ jobs: integration-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: Integration test needs: [sanity-check, changes] runs-on: ubuntu-latest @@ -301,7 +298,7 @@ jobs: slow-integration-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: Slow Integration Tests needs: [sanity-check, changes] runs-on: ubuntu-latest @@ -334,7 +331,7 @@ jobs: e2e-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker, dist-tar] runs-on: ${{ matrix.test.runs-on || 'ubuntu-latest' }} @@ -796,7 +793,7 @@ jobs: e2e-test-istio: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-24.04 @@ -864,7 +861,7 @@ jobs: e2e-test-istio-ambient: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-24.04 @@ -925,7 +922,7 @@ jobs: e2e-test-java-versions: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-latest @@ -975,7 +972,7 @@ jobs: # e2e-test-banyandb-stages: # if: | # ( always() && ! cancelled() ) && -# ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') +# ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') # name: E2E test # needs: [docker, dist-tar] # runs-on: ${{ matrix.test.runs-on || 'ubuntu-latest' }} From 5b6ffae4c6ce0e9ba140874dd40cae219c19b510 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Wed, 22 Oct 2025 01:51:11 +0800 Subject: [PATCH 60/69] fix pprof cache --- .../oap/server/core/CoreModuleConfig.java | 4 ++++ .../server/core/cache/CacheUpdateTimer.java | 11 ++++----- .../oap/server/core/cache/PprofTaskCache.java | 24 ++++++++++++------- .../provider/handler/PprofServiceHandler.java | 23 +++++++++++++----- skywalking-ui | 2 +- 5 files changed, 42 insertions(+), 22 deletions(-) diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java index cad853c86654..8052d8c44863 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java @@ -90,6 +90,10 @@ public class CoreModuleConfig extends ModuleConfig { * Following are cache setting for none stream(s) */ private long maxSizeOfProfileTask = 10_000L; + /** + * Following are cache setting for none stream(s) + */ + private long maxSizeOfPprofTask = 10_000L; /** * Analyze profile snapshots paging size. */ diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/CacheUpdateTimer.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/CacheUpdateTimer.java index 17f7cee530f7..1344cf7f22fe 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/CacheUpdateTimer.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/CacheUpdateTimer.java @@ -199,13 +199,12 @@ private void updatePprofTask(ModuleDefineHolder moduleDefineHolder) { List taskList = taskQueryDAO.getTaskList( null, taskCache.getCacheStartTimeBucket(), taskCache.getCacheEndTimeBucket(), null ); - if (CollectionUtils.isEmpty(taskList)) { - return; - } + taskList.stream().collect(Collectors.groupingBy(t -> t.getServiceId())).entrySet().stream().forEach(e -> { + final String serviceId = e.getKey(); + final List pprofTasks = e.getValue(); - for (PprofTask task : taskList) { - taskCache.saveTask(task.getServiceId(), task); - } + pprofTaskCache.saveTaskList(serviceId, pprofTasks); + }); } catch (IOException e) { log.warn("Unable to update pprof task cache", e); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java index 00aa57357068..ab4a52b66b63 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java @@ -21,6 +21,8 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import java.time.Duration; +import java.util.Collections; +import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.skywalking.oap.server.core.CoreModuleConfig; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; @@ -28,10 +30,10 @@ import org.apache.skywalking.oap.server.library.module.Service; public class PprofTaskCache implements Service { - private final Cache serviceId2taskCache; + private final Cache> serviceId2taskCache; public PprofTaskCache(CoreModuleConfig moduleConfig) { - long initialSize = moduleConfig.getMaxSizeOfProfileTask() / 10L; + long initialSize = moduleConfig.getMaxSizeOfPprofTask() / 10L; int initialCapacitySize = (int) (initialSize > Integer.MAX_VALUE ? Integer.MAX_VALUE : initialSize); serviceId2taskCache = CacheBuilder.newBuilder() @@ -42,17 +44,21 @@ public PprofTaskCache(CoreModuleConfig moduleConfig) { .build(); } - public PprofTask getPprofTask(String serviceId) { - PprofTask task = serviceId2taskCache.getIfPresent(serviceId); - return task; + public List getPprofTaskList(String serviceId) { + // read pprof task list from cache only, use cache update timer mechanism + List pprofTaskList = serviceId2taskCache.getIfPresent(serviceId); + return pprofTaskList; } - public void saveTask(String serviceId, PprofTask task) { - if (task == null) { - return; + /** + * save service task list + */ + public void saveTaskList(String serviceId, List taskList) { + if (taskList == null) { + taskList = Collections.emptyList(); } - serviceId2taskCache.put(serviceId, task); + serviceId2taskCache.put(serviceId, taskList); } /** diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java index ab80395534eb..63b906ac6b7c 100644 --- a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java @@ -20,7 +20,7 @@ import io.grpc.stub.StreamObserver; import java.io.IOException; -import java.util.Objects; +import java.util.List; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.apm.network.common.v3.Commands; @@ -79,20 +79,31 @@ public StreamObserver collect(StreamObserver public void getPprofTaskCommands(PprofTaskCommandQuery request, StreamObserver responseObserver) { String serviceId = IDManager.ServiceID.buildId(request.getService(), true); String serviceInstanceId = IDManager.ServiceInstanceID.buildId(serviceId, request.getServiceInstance()); - PprofTask task = taskCache.getPprofTask(serviceId); + List taskList = taskCache.getPprofTaskList(serviceId); // if task is null or createTime is less than lastCommandTime, return empty commands - if (Objects.isNull(task) || task.getCreateTime() <= request.getLastCommandTime() || (!CollectionUtils.isEmpty( - task.getServiceInstanceIds()) && !task.getServiceInstanceIds().contains(serviceInstanceId))) { + if (CollectionUtils.isEmpty(taskList)) { responseObserver.onNext(Commands.newBuilder().build()); responseObserver.onCompleted(); return; } - PprofTaskCommand pprofTaskCommand = commandService.newPprofTaskCommand(task); + final long lastCommandTime = request.getLastCommandTime(); + long minCreateTime = Long.MAX_VALUE; + PprofTask taskResult = null; + for (PprofTask task : taskList) { + if (task.getCreateTime() > lastCommandTime) { + if (task.getCreateTime() < minCreateTime) { + minCreateTime = task.getCreateTime(); + taskResult = task; + } + } + } + + PprofTaskCommand pprofTaskCommand = commandService.newPprofTaskCommand(taskResult); Commands commands = Commands.newBuilder().addCommands(pprofTaskCommand.serialize()).build(); responseObserver.onNext(commands); responseObserver.onCompleted(); - recordPprofTaskLog(task, serviceInstanceId, PprofTaskLogOperationType.NOTIFIED); + recordPprofTaskLog(taskResult, serviceInstanceId, PprofTaskLogOperationType.NOTIFIED); } public static void recordPprofTaskLog(PprofTask task, String instanceId, PprofTaskLogOperationType operationType) { diff --git a/skywalking-ui b/skywalking-ui index 3cefbf1bd5d5..b710a0a589e2 160000 --- a/skywalking-ui +++ b/skywalking-ui @@ -1 +1 @@ -Subproject commit 3cefbf1bd5d588355829cbe9dd23cf2253af547e +Subproject commit b710a0a589e293b0028e3b525cbad51b2b09cf6b From 6bb29d4c0a4339c7a26ae5a9be9868f3d8dbac99 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Wed, 22 Oct 2025 13:47:59 +0800 Subject: [PATCH 61/69] fix get pprof task --- .../provider/handler/PprofServiceHandler.java | 23 +++++++++++++++---- .../PprofByteBufCollectionObserver.java | 1 - 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java index 63b906ac6b7c..2a397b22f560 100644 --- a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java @@ -91,12 +91,25 @@ public void getPprofTaskCommands(PprofTaskCommandQuery request, StreamObserver lastCommandTime) { - if (task.getCreateTime() < minCreateTime) { - minCreateTime = task.getCreateTime(); - taskResult = task; - } + if (task.getCreateTime() <= lastCommandTime) { + continue; } + + if (!CollectionUtils.isEmpty(task.getServiceInstanceIds()) + && !task.getServiceInstanceIds().contains(serviceInstanceId)) { + continue; + } + + if (task.getCreateTime() < minCreateTime) { + minCreateTime = task.getCreateTime(); + taskResult = task; + } + } + + if (taskResult == null) { + responseObserver.onNext(Commands.newBuilder().build()); + responseObserver.onCompleted(); + return; } PprofTaskCommand pprofTaskCommand = commandService.newPprofTaskCommand(taskResult); diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java index ca401f740f9a..245c76811822 100644 --- a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java @@ -94,7 +94,6 @@ public void onNext(PprofData pprofData) { } else if (pprofData.hasContent()) { if (buf != null) { pprofData.getContent().copyTo(buf); - log.info("Received {} bytes of pprof data", pprofData.getContent().size()); } } } catch (IOException e) { From f613cbd8f181fcab58477e1ca39ee122df4bd48e Mon Sep 17 00:00:00 2001 From: JophieQu Date: Wed, 22 Oct 2025 19:28:03 +0800 Subject: [PATCH 62/69] delete blank lines --- .../query/PprofTaskLogQueryEsDAO.java | 25 +------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java index 355e8b73bc88..36e3f18f18a9 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java @@ -79,27 +79,4 @@ private PprofTaskLog buildPprofTaskLog(SearchHit data) { .operationTime(((Number) source.get(PprofTaskLogRecord.OPERATION_TIME)).longValue()) .build(); } -} - - - - - - - - - - - - - - - - - - - - - - - +} \ No newline at end of file From 02d2964661d1e5099ac2d13201ab453910236328 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Fri, 24 Oct 2025 16:23:26 +0800 Subject: [PATCH 63/69] fix doc & mock oap --- docs/en/changes/changes.md | 2 +- .../setup/backend/backend-go-app-profiling.md | 34 +++++++++---------- docs/menu.yml | 2 ++ .../profiling/pprof/PprofMutationService.java | 12 ++++--- .../provider/handler/PprofServiceHandler.java | 1 - .../src/main/resources/application.yml | 2 +- .../src/main/resources/log4j2.xml | 14 ++++---- .../profile/core/MockCoreModuleProvider.java | 9 +++++ test/e2e-v2/script/env | 2 +- 9 files changed, 46 insertions(+), 32 deletions(-) diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 344918512e2c..eb609cd1404d 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -53,7 +53,7 @@ * Self Observability: add `metrics_aggregation_queue_used_percentage` and `metrics_persistent_collection_cached_size` metrics for the OAP server. * Optimize metrics aggregate/persistent worker: separate `OAL` and `MAL` workers and consume pools. The dataflow signal drives the new MAL consumer, the following table shows the pool size，driven mode and queue size for each worker. -* Support pprof profiling feature. +* Support the go agent(0.7.0 release) bundled pprof profiling feature. | Worker | poolSize | isSignalDrivenMode | queueChannelSize | queueBufferSize | |-------------------------------|------------------------------------------|--------------------|------------------|-----------------| diff --git a/docs/en/setup/backend/backend-go-app-profiling.md b/docs/en/setup/backend/backend-go-app-profiling.md index 271199281019..dd791f0f0e0c 100644 --- a/docs/en/setup/backend/backend-go-app-profiling.md +++ b/docs/en/setup/backend/backend-go-app-profiling.md @@ -1,20 +1,20 @@ # Go App Profiling -Go App Profiling uses the Pprof for sampling +Go App Profiling uses the pprof for sampling -Pprof is bound within the auto-instrument agent and corresponds to [In-Process Profiling](../../concepts-and-designs/profiling.md#in-process-profiling). +pprof is bound within the auto-instrument agent and corresponds to [In-Process Profiling](../../concepts-and-designs/profiling.md#in-process-profiling). It is delivered to the agent in the form of a task, allowing it to be enabled or disabled dynamically. -When service encounters performance issues (cpu usage, memory allocation, etc.), Pprof task can be created. -When the agent receives a task, it enables Pprof for sampling. +When service encounters performance issues (CPU usage, memory allocation, etc.), pprof task can be created. +When the agent receives a task, it enables pprof for sampling. After sampling is completed, the sampling results are analyzed by requesting the server to render a flame graph for performance analysis to determine the specific business code lines that cause performance problems. Note, tracing profiling in the Go agent relies on the Go runtime’s global CPU sampling used by pprof. Since only one CPU profiler can run at a time within the same instance, tracing and pprof CPU profiling cannot be enabled simultaneously. If both are activated on the same instance, one task may fail to start. -## Activate Pprof in the OAP -OAP and the agent use a brand-new protocol to exchange Pprof data, so it is necessary to start OAP with the following configuration: +## Activate pprof in the OAP +OAP and the agent use a brand-new protocol to exchange pprof data, so it is necessary to start OAP with the following configuration: ```yaml receiver-pprof: @@ -32,18 +32,18 @@ receiver-pprof: memoryParserEnabled: ${SW_RECEIVER_PPROF_MEMORY_PARSER_ENABLED:true} ``` -## Pprof Task with Analysis +## pprof Task with Analysis -To use the Pprof feature, please follow these steps: +To use the pprof feature, please follow these steps: -1. **Create Pprof task**: Use the UI or CLI tool to create a task. -2. **Wait agent collect data and upload**: Wait for Pprof to collect pprof data and report. +1. **Create pprof task**: Use the UI or CLI tool to create a task. +2. **Wait agent collect data and upload**: Wait for pprof to collect pprof data and report. 3. **Query task progress**: Query the progress of tasks, including analyzing successful and failed instances and task logs. 4. **Analyze the data**: Analyze the pprof data to determine where performance bottlenecks exist in the service. -### Create an Pprof task +### Create an pprof task -Create an Pprof task to notify some go-agent instances in the execution service to start Pprof for data collection. +Create an pprof task to notify some go-agent instances in the execution service to start pprof for data collection. When creating a task, the following configuration fields are required: @@ -53,7 +53,7 @@ When creating a task, the following configuration fields are required: 4. **events**: Define which event types this task needs to collect. 5. **dumpPeriod**: Define the period of the pprof dump, required for BLOCK, MUTEX events. -When the Agent receives a Pprof task from OAP, it automatically generates a log to notify that the task has been acknowledged. The log contains the following field information: +When the Agent receives a pprof task from OAP, it automatically generates a log to notify that the task has been acknowledged. The log contains the following field information: 1. **Instance**: The name of the instance where the Agent is located. 2. **Type**: Supports "NOTIFIED" and "EXECUTION_FINISHED" and "PPROF_UPLOAD_FILE_TOO_LARGE_ERROR", "EXECUTION_TASK_ERROR", with the current log displaying "NOTIFIED". @@ -61,7 +61,7 @@ When the Agent receives a Pprof task from OAP, it automatically generates a log ### Wait the agent to collect data and upload -At this point, Pprof will trace the events you selected when you created the task: +At this point, pprof will trace the events you selected when you created the task: 1. CPU: samples CPU usage over time to show which functions consume the most processing time. 2. ALLOC, HEAP: @@ -74,12 +74,12 @@ At this point, Pprof will trace the events you selected when you created the tas - GOROUTINE: stack traces of all current goroutines. - THREADCREATE: stack traces that led to the creation of new OS threads. -Finally, the agent will upload the pprof file produced by Pprof to the oap server for online performance analysis. +Finally, the agent will upload the pprof file produced by pprof to the oap server for online performance analysis. ### Query the profiling task progresses -Wait for Pprof to complete data collection and upload successfully. -We can query the execution logs of the Pprof task and the task status, which includes the following information: +Wait for pprof to complete data collection and upload successfully. +We can query the execution logs of the pprof task and the task status, which includes the following information: 1. **successInstanceIds**: SuccessInstanceIds gives instances that have executed the task successfully. 2. **errorInstanceIds**: ErrorInstanceIds gives instances that failed to execute the task. diff --git a/docs/menu.yml b/docs/menu.yml index 79e32693040a..702fd1923453 100644 --- a/docs/menu.yml +++ b/docs/menu.yml @@ -270,6 +270,8 @@ catalog: path: "/en/setup/backend/backend-continuous-profiling" - name: "Java App Profiling" path: "/en/setup/backend/backend-java-app-profiling" + - name: "Go App Profiling" + path: "en/setup/backend/backend-go-app-profiling.md" - name: "Event" path: "/en/concepts-and-designs/event/" - name: "Extension" diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java index 527a24051eb1..a26dee399f29 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java @@ -97,7 +97,7 @@ private PprofTaskCreationResult checkDataSuccess(String serviceId, .errorReason(checkArgumentMessage) .build(); } - String checkTaskProfilingMessage = checkTaskProfiling(serviceId, createTime); + String checkTaskProfilingMessage = checkTaskProfiling(serviceId,events, createTime); if (checkTaskProfilingMessage != null) { return PprofTaskCreationResult.builder() .code(PprofTaskCreationType.ALREADY_PROFILING_ERROR) @@ -122,6 +122,9 @@ private String checkArgumentError(String serviceId, if (duration <= 0) { return "duration cannot be negative"; } + if (duration > 15) { + return "duration cannot be greater than 15 minutes"; + } } if (events == PprofEventType.BLOCK || events == PprofEventType.MUTEX) { if (dumpPeriod <= 0) { @@ -135,15 +138,16 @@ private String checkArgumentError(String serviceId, } private String checkTaskProfiling(String serviceId, + PprofEventType events, long createTime) throws IOException { // Each service can only enable one task at a time - long endTimeBucket = TimeBucket.getMinuteTimeBucket(createTime); + long endTimeBucket = TimeBucket.getRecordTimeBucket(createTime); final List alreadyHaveTaskList = getPprofTaskDAO().getTaskList( - serviceId, null, endTimeBucket, 1 + serviceId, null, endTimeBucket, null ); if (CollectionUtils.isNotEmpty(alreadyHaveTaskList)) { for (PprofTask task : alreadyHaveTaskList) { - if (task.getCreateTime() + TimeUnit.SECONDS.toMillis(task.getDuration()) >= createTime) { + if (task.getEvents().equals(events) && task.getCreateTime() + TimeUnit.MINUTES.toMillis(task.getDuration()) >= createTime) { // if the endTime is greater or equal than the createTime of the newly created task, i.e. there is overlap between two tasks, it is an invalid case, it will return an error return "current service already has monitor pprof task execute at this time"; } diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java index 2a397b22f560..30773ec2b8d2 100644 --- a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java @@ -80,7 +80,6 @@ public void getPprofTaskCommands(PprofTaskCommandQuery request, StreamObserver taskList = taskCache.getPprofTaskList(serviceId); - // if task is null or createTime is less than lastCommandTime, return empty commands if (CollectionUtils.isEmpty(taskList)) { responseObserver.onNext(Commands.newBuilder().build()); responseObserver.onCompleted(); diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index 7f082649425d..6c28501a39e3 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -145,7 +145,7 @@ core: # The long value of the max direct memory usage. The default max value is -1, representing no limit. The unit is in bytes. maxDirectMemoryUsage: ${SW_CORE_MAX_DIRECT_MEMORY_USAGE:-1} storage: - selector: ${SW_STORAGE:banyandb} + selector: ${SW_STORAGE:elasticsearch} banyandb: # Since 10.2.0, the banyandb configuration is separated to an independent configuration file: `bydb.yaml`. elasticsearch: diff --git a/oap-server/server-starter/src/main/resources/log4j2.xml b/oap-server/server-starter/src/main/resources/log4j2.xml index 6cbaa5c9dd0e..7614657400df 100644 --- a/oap-server/server-starter/src/main/resources/log4j2.xml +++ b/oap-server/server-starter/src/main/resources/log4j2.xml @@ -17,12 +17,12 @@ ~ --> - + - + @@ -32,15 +32,15 @@ - + - - + + - - + + diff --git a/oap-server/server-tools/profile-exporter/tool-profile-snapshot-server-mock/src/main/java/org/apache/skywalking/oap/server/tool/profile/core/MockCoreModuleProvider.java b/oap-server/server-tools/profile-exporter/tool-profile-snapshot-server-mock/src/main/java/org/apache/skywalking/oap/server/tool/profile/core/MockCoreModuleProvider.java index a5e98389984e..8e55a72df68d 100755 --- a/oap-server/server-tools/profile-exporter/tool-profile-snapshot-server-mock/src/main/java/org/apache/skywalking/oap/server/tool/profile/core/MockCoreModuleProvider.java +++ b/oap-server/server-tools/profile-exporter/tool-profile-snapshot-server-mock/src/main/java/org/apache/skywalking/oap/server/tool/profile/core/MockCoreModuleProvider.java @@ -25,6 +25,7 @@ import org.apache.skywalking.oap.server.core.annotation.AnnotationScan; import org.apache.skywalking.oap.server.core.cache.AsyncProfilerTaskCache; import org.apache.skywalking.oap.server.core.cache.NetworkAddressAliasCache; +import org.apache.skywalking.oap.server.core.cache.PprofTaskCache; import org.apache.skywalking.oap.server.core.cache.ProfileTaskCache; import org.apache.skywalking.oap.server.core.command.CommandService; import org.apache.skywalking.oap.server.core.config.ConfigService; @@ -44,6 +45,8 @@ import org.apache.skywalking.oap.server.core.profiling.continuous.ContinuousProfilingQueryService; import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingMutationService; import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingQueryService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofMutationService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofQueryService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskMutationService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskQueryService; import org.apache.skywalking.oap.server.core.query.AggregationQueryService; @@ -212,6 +215,12 @@ TTLStatusQuery.class, new TTLStatusQuery( AsyncProfilerQueryService.class, new AsyncProfilerQueryService(getManager())); this.registerServiceImplementation( AsyncProfilerTaskCache.class, new AsyncProfilerTaskCache(getManager(), moduleConfig)); + this.registerServiceImplementation( + PprofMutationService.class, new PprofMutationService(getManager())); + this.registerServiceImplementation( + PprofQueryService.class, new PprofQueryService(getManager())); + this.registerServiceImplementation( + PprofTaskCache.class, new PprofTaskCache(moduleConfig)); this.registerServiceImplementation( EBPFProfilingMutationService.class, new EBPFProfilingMutationService(getManager())); this.registerServiceImplementation( diff --git a/test/e2e-v2/script/env b/test/e2e-v2/script/env index a04eee45ffd3..343737522f92 100644 --- a/test/e2e-v2/script/env +++ b/test/e2e-v2/script/env @@ -17,7 +17,7 @@ SW_AGENT_JAVA_COMMIT=f0245864e4388a388fe7445b56b6ce7cedc94aaf SW_AGENT_SATELLITE_COMMIT=ea27a3f4e126a24775fe12e2aa2695bcb23d99c3 SW_AGENT_NGINX_LUA_COMMIT=c3cee4841798a147d83b96a10914d4ac0e11d0aa SW_AGENT_NODEJS_COMMIT=4f9a91dad3dfd8cfe5ba8f7bd06b39e11eb5e65e -SW_AGENT_GO_COMMIT=aa948377ecdb4724fad1cc365c13a1188021316f +SW_AGENT_GO_COMMIT=afa75a3cc8c31f142102443af6164b825d63d8fc SW_AGENT_PYTHON_COMMIT=c76a6ec51a478ac91abb20ec8f22a99b8d4d6a58 SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449 SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016 From 4f766981cc3766c5957349ab6e4a4c39c245938b Mon Sep 17 00:00:00 2001 From: JophieQu Date: Fri, 24 Oct 2025 16:27:43 +0800 Subject: [PATCH 64/69] roll back proto --- apm-protocol/apm-network/src/main/proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apm-protocol/apm-network/src/main/proto b/apm-protocol/apm-network/src/main/proto index 6ec6d71168c1..055d64b104b5 160000 --- a/apm-protocol/apm-network/src/main/proto +++ b/apm-protocol/apm-network/src/main/proto @@ -1 +1 @@ -Subproject commit 6ec6d71168c1068d1f09f19d57120dd344a1d585 +Subproject commit 055d64b104b5d84e15e27b74f5cbe712e7f9b0df From 49d84a014a49efc593494ec2a7c3fe7c9b218939 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Fri, 24 Oct 2025 17:23:45 +0800 Subject: [PATCH 65/69] roll back --- .../src/main/resources/application.yml | 2 +- .../server-starter/src/main/resources/log4j2.xml | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index 6c28501a39e3..7f082649425d 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -145,7 +145,7 @@ core: # The long value of the max direct memory usage. The default max value is -1, representing no limit. The unit is in bytes. maxDirectMemoryUsage: ${SW_CORE_MAX_DIRECT_MEMORY_USAGE:-1} storage: - selector: ${SW_STORAGE:elasticsearch} + selector: ${SW_STORAGE:banyandb} banyandb: # Since 10.2.0, the banyandb configuration is separated to an independent configuration file: `bydb.yaml`. elasticsearch: diff --git a/oap-server/server-starter/src/main/resources/log4j2.xml b/oap-server/server-starter/src/main/resources/log4j2.xml index 7614657400df..6cbaa5c9dd0e 100644 --- a/oap-server/server-starter/src/main/resources/log4j2.xml +++ b/oap-server/server-starter/src/main/resources/log4j2.xml @@ -17,12 +17,12 @@ ~ --> - + - + @@ -32,15 +32,15 @@ - + - - + + - - + + From ce5384bc3cc2a3c02f7e45b4327d9be236d107ec Mon Sep 17 00:00:00 2001 From: JophieQu Date: Fri, 24 Oct 2025 22:20:13 +0800 Subject: [PATCH 66/69] fix codestyle --- .../server/core/profiling/pprof/PprofMutationService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java index a26dee399f29..a220ba29304e 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java @@ -97,7 +97,7 @@ private PprofTaskCreationResult checkDataSuccess(String serviceId, .errorReason(checkArgumentMessage) .build(); } - String checkTaskProfilingMessage = checkTaskProfiling(serviceId,events, createTime); + String checkTaskProfilingMessage = checkTaskProfiling(serviceId, events, createTime); if (checkTaskProfilingMessage != null) { return PprofTaskCreationResult.builder() .code(PprofTaskCreationType.ALREADY_PROFILING_ERROR) @@ -147,7 +147,8 @@ private String checkTaskProfiling(String serviceId, ); if (CollectionUtils.isNotEmpty(alreadyHaveTaskList)) { for (PprofTask task : alreadyHaveTaskList) { - if (task.getEvents().equals(events) && task.getCreateTime() + TimeUnit.MINUTES.toMillis(task.getDuration()) >= createTime) { + if (task.getEvents().equals(events) && task.getCreateTime() + TimeUnit.MINUTES.toMillis( + task.getDuration()) >= createTime) { // if the endTime is greater or equal than the createTime of the newly created task, i.e. there is overlap between two tasks, it is an invalid case, it will return an error return "current service already has monitor pprof task execute at this time"; } From 4c47d09818eae6d0bed98bdab64f0b2c0f3842ca Mon Sep 17 00:00:00 2001 From: JophieQu Date: Fri, 24 Oct 2025 22:56:49 +0800 Subject: [PATCH 67/69] fix --- test/e2e-v2/cases/go/service/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e-v2/cases/go/service/go.mod b/test/e2e-v2/cases/go/service/go.mod index 6b5f043ed2a8..1e2b1db6e369 100644 --- a/test/e2e-v2/cases/go/service/go.mod +++ b/test/e2e-v2/cases/go/service/go.mod @@ -20,7 +20,7 @@ module sw-e2e go 1.19 require ( - github.com/apache/skywalking-go v0.6.1-0.20250924145416-aa948377ecdb + github.com/apache/skywalking-go v0.6.1-0.20251023090254-afa75a3cc8c3 github.com/apache/skywalking-go/toolkit v0.5.1-0.20250301084827-154de50628e8 github.com/gin-gonic/gin v1.10.0 ) From a895cbae9ae549179b0fe26ff28afae4c1c4103b Mon Sep 17 00:00:00 2001 From: JophieQu Date: Sat, 25 Oct 2025 00:10:10 +0800 Subject: [PATCH 68/69] fix doc --- .../network/trace/component/command/PprofTaskCommand.java | 8 +++++--- docs/en/changes/changes.md | 3 ++- docs/en/concepts-and-designs/profiling.md | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java index 87210f8685da..30954b263217 100644 --- a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java +++ b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java @@ -31,7 +31,9 @@ public class PprofTaskCommand extends BaseCommand implements Serializable, Deser * pprof taskId */ private String taskId; - // Type of profiling (CPU/Heap/Block/Mutex/Goroutine/Threadcreate/Allocs) + /** + * event type of profiling (CPU/Heap/Block/Mutex/Goroutine/Threadcreate/Allocs) + */ private String events; /** * run profiling for duration (minute) @@ -45,8 +47,8 @@ public class PprofTaskCommand extends BaseCommand implements Serializable, Deser * pprof dump period parameters. There are different dumpperiod configurations for different events. * Here is a table of parameters. * - *

For Block - sample an average of one blocking event per rate nanoseconds spent blocked. (default: 0)

- *

For Mutex - sample an average of 1/rate events are reported. (default: 0)

+ *

for Block - sample an average of one blocking event per rate nanoseconds spent blocked. (default: 0)

+ *

for Mutex - sample an average of 1/rate events are reported. (default: 0)

* details @see pprof argument */ private int dumpPeriod; diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index eb609cd1404d..e2bd5937b8bc 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -53,7 +53,6 @@ * Self Observability: add `metrics_aggregation_queue_used_percentage` and `metrics_persistent_collection_cached_size` metrics for the OAP server. * Optimize metrics aggregate/persistent worker: separate `OAL` and `MAL` workers and consume pools. The dataflow signal drives the new MAL consumer, the following table shows the pool size，driven mode and queue size for each worker. -* Support the go agent(0.7.0 release) bundled pprof profiling feature. | Worker | poolSize | isSignalDrivenMode | queueChannelSize | queueBufferSize | |-------------------------------|------------------------------------------|--------------------|------------------|-----------------| @@ -112,6 +111,8 @@ * Make MAL percentile align with OAL percentile calculation. * Update Grafana dashboards for OAP observability. * BanyanDB: fix query `getInstance` by instance ID. +* Support the go agent(0.7.0 release) bundled pprof profiling feature. + #### UI diff --git a/docs/en/concepts-and-designs/profiling.md b/docs/en/concepts-and-designs/profiling.md index 603adb1041ff..95992ef31b82 100644 --- a/docs/en/concepts-and-designs/profiling.md +++ b/docs/en/concepts-and-designs/profiling.md @@ -49,7 +49,7 @@ Only Java agent support this. ### Go App Profiling -Go App Profiling uses the [Pprof](https://github.com/google/pprof) for sampling. +Go App Profiling uses the [pprof](https://github.com/google/pprof) for sampling. pprof is a profiling tool by Google for visualizing and analyzing sampled performance data. It reads samples in profile.proto format and generates text or graphical reports (via the dot visualization) to highlight performance hotspots. From c2825e7c97ba5b4c9fc5990067f383c46bbc1e90 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Sat, 25 Oct 2025 00:11:16 +0800 Subject: [PATCH 69/69] fix doc --- docs/en/setup/backend/backend-go-app-profiling.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/setup/backend/backend-go-app-profiling.md b/docs/en/setup/backend/backend-go-app-profiling.md index dd791f0f0e0c..3cfbf0ba6d37 100644 --- a/docs/en/setup/backend/backend-go-app-profiling.md +++ b/docs/en/setup/backend/backend-go-app-profiling.md @@ -2,7 +2,7 @@ Go App Profiling uses the pprof for sampling -pprof is bound within the auto-instrument agent and corresponds to [In-Process Profiling](../../concepts-and-designs/profiling.md#in-process-profiling). +pprof is bundled within the auto-instrument agent and corresponds to [In-Process Profiling](../../concepts-and-designs/profiling.md#in-process-profiling). It is delivered to the agent in the form of a task, allowing it to be enabled or disabled dynamically. When service encounters performance issues (CPU usage, memory allocation, etc.), pprof task can be created.