diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index 217ccbe9f8b0..a953ec90f9af 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -17,6 +17,9 @@ name: CI on: + push: + branches: + - citest pull_request: schedule: - cron: "0 18 * * *" # TimeZone: UTC 0 @@ -32,7 +35,7 @@ env: jobs: license-header: - if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || (github.event_name != 'schedule') name: License header runs-on: ubuntu-latest timeout-minutes: 10 @@ -45,7 +48,7 @@ jobs: uses: apache/skywalking-eyes@5b7ee1731d036b5aac68f8bd3fc9e6f98ada082e code-style: - if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || (github.event_name != 'schedule') name: Code style runs-on: ubuntu-latest timeout-minutes: 10 @@ -60,7 +63,7 @@ jobs: dependency-license: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.pom == 'true' || needs.changes.outputs.ui == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.pom == 'true' || needs.changes.outputs.ui == 'true') name: Dependency licenses needs: [changes] runs-on: ubuntu-latest @@ -90,7 +93,7 @@ jobs: fi sanity-check: - if: ( always() && ! cancelled() ) && (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') + if: ( always() && ! cancelled() ) && (github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || (github.event_name != 'schedule') name: Sanity check results needs: [license-header, code-style, dependency-license] runs-on: ubuntu-latest @@ -159,7 +162,7 @@ jobs: dist-tar: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: Build dist tar needs: [changes] runs-on: ubuntu-latest @@ -191,7 +194,7 @@ jobs: docker: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: Docker images needs: [sanity-check, dist-tar, changes] runs-on: ubuntu-latest @@ -230,7 +233,7 @@ jobs: unit-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: Unit test needs: [sanity-check, changes] runs-on: ${{ matrix.os }} @@ -265,7 +268,7 @@ jobs: integration-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: Integration test needs: [sanity-check, changes] runs-on: ubuntu-latest @@ -298,7 +301,7 @@ jobs: slow-integration-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: Slow Integration Tests needs: [sanity-check, changes] runs-on: ubuntu-latest @@ -331,7 +334,7 @@ jobs: e2e-test: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker, dist-tar] runs-on: ${{ matrix.test.runs-on || 'ubuntu-latest' }} @@ -723,6 +726,13 @@ jobs: config: test/e2e-v2/cases/profiling/async-profiler/banyandb/e2e.yaml - name: Async Profiler MySQL config: test/e2e-v2/cases/profiling/async-profiler/mysql/e2e.yaml + + - name: Pprof ES + config: test/e2e-v2/cases/profiling/pprof/es/e2e.yaml + - name: Pprof BanyanDB + config: test/e2e-v2/cases/profiling/pprof/banyandb/e2e.yaml + - name: Pprof MySQL + config: test/e2e-v2/cases/profiling/pprof/mysql/e2e.yaml steps: - uses: actions/checkout@v4 with: @@ -786,7 +796,7 @@ jobs: e2e-test-istio: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-24.04 @@ -854,7 +864,7 @@ jobs: e2e-test-istio-ambient: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-24.04 @@ -915,7 +925,7 @@ jobs: e2e-test-java-versions: if: | ( always() && ! cancelled() ) && - ((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true') + ((github.event_name == 'schedule' && github.repository == 'JophieQu/skywalking') || needs.changes.outputs.oap == 'true') name: E2E test needs: [docker] runs-on: ubuntu-latest diff --git a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/CommandDeserializer.java b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/CommandDeserializer.java index 6323e6906a12..4bd94b8bc0e0 100644 --- a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/CommandDeserializer.java +++ b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/CommandDeserializer.java @@ -29,6 +29,8 @@ public static BaseCommand deserialize(final Command command) { return ConfigurationDiscoveryCommand.DESERIALIZER.deserialize(command); } else if (AsyncProfilerTaskCommand.NAME.equals(commandName)) { return AsyncProfilerTaskCommand.DESERIALIZER.deserialize(command); + } else if (PprofTaskCommand.NAME.equals(commandName)) { + return PprofTaskCommand.DESERIALIZER.deserialize(command); } throw new UnsupportedCommandException(command); } diff --git a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java new file mode 100644 index 000000000000..87210f8685da --- /dev/null +++ b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.network.trace.component.command; + +import org.apache.skywalking.apm.network.common.v3.Command; +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import java.util.List; +import lombok.Getter; + +@Getter +public class PprofTaskCommand extends BaseCommand implements Serializable, Deserializable { + public static final Deserializable DESERIALIZER = new PprofTaskCommand("", "", "", 0, 0, 0); + public static final String NAME = "PprofTaskQuery"; + /** + * pprof taskId + */ + private String taskId; + // Type of profiling (CPU/Heap/Block/Mutex/Goroutine/Threadcreate/Allocs) + private String events; + /** + * run profiling for duration (minute) + */ + private long duration; + /** + * task create time + */ + private long createTime; + /** + * pprof dump period parameters. There are different dumpperiod configurations for different events. + * Here is a table of parameters. + * + *

For Block - sample an average of one blocking event per rate nanoseconds spent blocked. (default: 0)

+ *

For Mutex - sample an average of 1/rate events are reported. (default: 0)

+ * details @see pprof argument + */ + private int dumpPeriod; + + public PprofTaskCommand(String serialNumber, String taskId, String events, + long duration, long createTime, int dumpPeriod) { + super(NAME, serialNumber); + this.taskId = taskId; + this.duration = duration; + this.createTime = createTime; + this.dumpPeriod = dumpPeriod; + this.events = events; + } + + @Override + public PprofTaskCommand deserialize(Command command) { + final List argsList = command.getArgsList(); + String taskId = null; + String events = null; + long duration = 0; + long createTime = 0; + int dumpPeriod = 0; + String serialNumber = null; + for (final KeyStringValuePair pair : argsList) { + if ("SerialNumber".equals(pair.getKey())) { + serialNumber = pair.getValue(); + } else if ("TaskId".equals(pair.getKey())) { + taskId = pair.getValue(); + } else if ("Events".equals(pair.getKey())) { + events = pair.getValue(); + } else if ("Duration".equals(pair.getKey())) { + duration = Long.parseLong(pair.getValue()); + } else if ("CreateTime".equals(pair.getKey())) { + createTime = Long.parseLong(pair.getValue()); + } else if ("DumpPeriod".equals(pair.getKey())) { + dumpPeriod = Integer.parseInt(pair.getValue()); + } + } + return new PprofTaskCommand(serialNumber, taskId, events, duration, createTime, dumpPeriod); + } + + @Override + public Command.Builder serialize() { + final Command.Builder builder = commandBuilder(); + builder.addArgs(KeyStringValuePair.newBuilder().setKey("TaskId").setValue(taskId)) + .addArgs(KeyStringValuePair.newBuilder().setKey("Events").setValue(events)) + .addArgs(KeyStringValuePair.newBuilder().setKey("Duration").setValue(String.valueOf(duration))) + .addArgs(KeyStringValuePair.newBuilder().setKey("CreateTime").setValue(String.valueOf(createTime))) + .addArgs(KeyStringValuePair.newBuilder().setKey("DumpPeriod").setValue(String.valueOf(dumpPeriod))); + return builder; + } +} \ No newline at end of file diff --git a/apm-protocol/apm-network/src/main/proto b/apm-protocol/apm-network/src/main/proto index 157baf3e7a97..fb3fb005650e 160000 --- a/apm-protocol/apm-network/src/main/proto +++ b/apm-protocol/apm-network/src/main/proto @@ -1 +1 @@ -Subproject commit 157baf3e7a9710d3041e9caec39eecab7e0c5a82 +Subproject commit fb3fb005650e2489164978b7804117c7ade1529a diff --git a/dist-material/release-docs/LICENSE b/dist-material/release-docs/LICENSE index 35777513907e..df015b932865 100644 --- a/dist-material/release-docs/LICENSE +++ b/dist-material/release-docs/LICENSE @@ -477,20 +477,21 @@ The text of each license is also included in licenses/LICENSE-[project].txt. https://npmjs.com/package/@floating-ui/core/v/1.6.9 1.6.9 MIT https://npmjs.com/package/@floating-ui/dom/v/1.6.13 1.6.13 MIT https://npmjs.com/package/@floating-ui/utils/v/0.2.9 0.2.9 MIT - https://npmjs.com/package/@interactjs/actions/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/auto-scroll/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/auto-start/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/core/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/dev-tools/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/inertia/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/interact/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/interactjs/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/modifiers/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/offset/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/pointer-events/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/reflow/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/snappers/v/1.10.27 1.10.27 MIT - https://npmjs.com/package/@interactjs/utils/v/1.10.27 1.10.27 MIT + https://npmjs.com/package/@interactjs/actions/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/auto-scroll/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/auto-start/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/core/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/dev-tools/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/inertia/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/interact/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/interactjs/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/modifiers/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/offset/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/pointer-events/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/reflow/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/snappers/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/types/v/1.10.17 1.10.17 MIT + https://npmjs.com/package/@interactjs/utils/v/1.10.17 1.10.17 MIT https://npmjs.com/package/@intlify/core-base/v/9.14.5 9.14.5 MIT https://npmjs.com/package/@intlify/message-compiler/v/9.14.5 9.14.5 MIT https://npmjs.com/package/@intlify/shared/v/9.14.5 9.14.5 MIT @@ -528,7 +529,7 @@ The text of each license is also included in licenses/LICENSE-[project].txt. https://npmjs.com/package/d3-dsv/node_modules/commander/v/7.2.0 7.2.0 MIT https://npmjs.com/package/d3-tip/v/0.9.1 0.9.1 MIT https://npmjs.com/package/dayjs/v/1.11.13 1.11.13 MIT - https://npmjs.com/package/element-plus/v/2.11.0 2.11.0 MIT + https://npmjs.com/package/element-plus/v/2.9.4 2.9.4 MIT https://npmjs.com/package/element-resize-detector/v/1.2.4 1.2.4 MIT https://npmjs.com/package/escape-html/v/1.0.3 1.0.3 MIT https://npmjs.com/package/estree-walker/v/2.0.2 2.0.2 MIT diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 48a0984adfff..212a059742a6 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -49,6 +49,7 @@ * Add UI dashboard for Ruby runtime metrics. * Tracing Query Execution HTTP APIs: make the argument `service layer` optional. * GraphQL API: metadata, topology, log and trace support query by name. +* Support pprof profiling feature * [Break Change] MQE function `sort_values` sorts according to the aggregation result and labels rather than the simple time series values. * Self Observability: add `metrics_aggregation_queue_used_percentage` and `metrics_persistent_collection_cached_size` metrics for the OAP server. * Optimize metrics aggregate/persistent worker: separate `OAL` and `MAL` workers and consume pools. The dataflow signal drives the new MAL consumer, diff --git a/docs/en/concepts-and-designs/profiling.md b/docs/en/concepts-and-designs/profiling.md index 5c1ae91e9603..603adb1041ff 100644 --- a/docs/en/concepts-and-designs/profiling.md +++ b/docs/en/concepts-and-designs/profiling.md @@ -47,6 +47,20 @@ Async Profiler can trace the following kinds of events: Only Java agent support this. +### Go App Profiling + +Go App Profiling uses the [Pprof](https://github.com/google/pprof) for sampling. + +pprof is a profiling tool by Google for visualizing and analyzing sampled performance data. +It reads samples in profile.proto format and generates text or graphical reports (via the dot visualization) to highlight performance hotspots. + +pprof supports profiling of: + +- CPU. +- Memory allocs / heap. +- Block / mutex. +- Gouroutine / threadcreate. + ## Out-of-process profiling Out-of-process profiling leverage [eBPF](https://ebpf.io/) technology with origins in the Linux kernel. diff --git a/oap-server/server-core/pom.xml b/oap-server/server-core/pom.xml index 784da5435c03..f6d233587ad2 100644 --- a/oap-server/server-core/pom.xml +++ b/oap-server/server-core/pom.xml @@ -44,6 +44,11 @@ library-async-profiler-jfr-parser ${project.version} + + org.apache.skywalking + library-pprof-parser + ${project.version} + org.apache.skywalking telemetry-api diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java index 91509759eab3..b6d1734d2f43 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java @@ -22,6 +22,7 @@ import java.util.List; import org.apache.skywalking.oap.server.core.analysis.meter.MeterSystem; import org.apache.skywalking.oap.server.core.cache.AsyncProfilerTaskCache; +import org.apache.skywalking.oap.server.core.cache.PprofTaskCache; import org.apache.skywalking.oap.server.core.cache.NetworkAddressAliasCache; import org.apache.skywalking.oap.server.core.cache.ProfileTaskCache; import org.apache.skywalking.oap.server.core.command.CommandService; @@ -41,6 +42,8 @@ import org.apache.skywalking.oap.server.core.profiling.continuous.ContinuousProfilingQueryService; import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingMutationService; import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingQueryService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofMutationService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofQueryService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskMutationService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskQueryService; import org.apache.skywalking.oap.server.core.query.AggregationQueryService; @@ -106,6 +109,7 @@ public Class[] services() { addManagementService(classes); addEBPFProfilingService(classes); addAsyncProfilerService(classes); + addPprofService(classes); classes.add(CommandService.class); classes.add(HierarchyService.class); @@ -137,6 +141,12 @@ private void addAsyncProfilerService(List classes) { classes.add(AsyncProfilerTaskCache.class); } + private void addPprofService(List classes) { + classes.add(PprofMutationService.class); + classes.add(PprofQueryService.class); + classes.add(PprofTaskCache.class); + } + private void addOALService(List classes) { classes.add(OALEngineLoaderService.class); } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java index ad9f7c4b61ad..d55dc78853ee 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java @@ -35,6 +35,7 @@ import org.apache.skywalking.oap.server.core.cache.AsyncProfilerTaskCache; import org.apache.skywalking.oap.server.core.cache.CacheUpdateTimer; import org.apache.skywalking.oap.server.core.cache.NetworkAddressAliasCache; +import org.apache.skywalking.oap.server.core.cache.PprofTaskCache; import org.apache.skywalking.oap.server.core.cache.ProfileTaskCache; import org.apache.skywalking.oap.server.core.cluster.ClusterCoordinator; import org.apache.skywalking.oap.server.core.cluster.ClusterModule; @@ -65,6 +66,8 @@ import org.apache.skywalking.oap.server.core.profiling.continuous.ContinuousProfilingQueryService; import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingMutationService; import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingQueryService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofMutationService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofQueryService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskMutationService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskQueryService; import org.apache.skywalking.oap.server.core.query.AggregationQueryService; @@ -330,6 +333,12 @@ TTLStatusQuery.class, new TTLStatusQuery( AsyncProfilerQueryService.class, new AsyncProfilerQueryService(getManager())); this.registerServiceImplementation( AsyncProfilerTaskCache.class, new AsyncProfilerTaskCache(getManager(), moduleConfig)); + this.registerServiceImplementation( + PprofMutationService.class, new PprofMutationService(getManager())); + this.registerServiceImplementation( + PprofQueryService.class, new PprofQueryService(getManager())); + this.registerServiceImplementation( + PprofTaskCache.class, new PprofTaskCache(moduleConfig)); this.registerServiceImplementation( EBPFProfilingMutationService.class, new EBPFProfilingMutationService(getManager())); this.registerServiceImplementation( diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/CacheUpdateTimer.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/CacheUpdateTimer.java index c38f90daccd0..17f7cee530f7 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/CacheUpdateTimer.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/CacheUpdateTimer.java @@ -27,12 +27,15 @@ import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.storage.AsyncProfilerTaskRecord; import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerTask; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; import org.apache.skywalking.oap.server.library.util.CollectionUtils; import org.apache.skywalking.oap.server.library.util.RunnableWithExceptionProtection; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.analysis.DisableRegister; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.manual.networkalias.NetworkAddressAlias; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskRecord; import org.apache.skywalking.oap.server.core.query.type.ProfileTask; import org.apache.skywalking.oap.server.core.storage.StorageModule; @@ -46,6 +49,8 @@ public enum CacheUpdateTimer { private AsyncProfilerTaskCache asyncProfilerTaskCache; private IAsyncProfilerTaskQueryDAO asyncProfilerTaskQueryDAO; + private PprofTaskCache pprofTaskCache; + private IPprofTaskQueryDAO pprofTaskQueryDAO; private int ttl = 10; @@ -72,6 +77,10 @@ private void update(ModuleDefineHolder moduleDefineHolder) { if (!DisableRegister.INSTANCE.include(AsyncProfilerTaskRecord.INDEX_NAME)) { updateAsyncProfilerTask(moduleDefineHolder); } + + if (!DisableRegister.INSTANCE.include(PprofTaskRecord.INDEX_NAME)) { + updatePprofTask(moduleDefineHolder); + } } /** @@ -163,4 +172,43 @@ private void updateAsyncProfilerTask(ModuleDefineHolder moduleDefineHolder) { return; } + + private PprofTaskCache getPprofTaskCache(ModuleDefineHolder moduleDefineHolder) { + if (pprofTaskCache == null) { + pprofTaskCache = moduleDefineHolder.find(CoreModule.NAME) + .provider() + .getService(PprofTaskCache.class); + } + return pprofTaskCache; + } + + private IPprofTaskQueryDAO getPprofTaskQueryDAO(ModuleDefineHolder moduleDefineHolder) { + if (pprofTaskQueryDAO == null) { + pprofTaskQueryDAO = moduleDefineHolder.find(StorageModule.NAME) + .provider() + .getService(IPprofTaskQueryDAO.class); + } + return pprofTaskQueryDAO; + } + + private void updatePprofTask(ModuleDefineHolder moduleDefineHolder) { + PprofTaskCache taskCache = getPprofTaskCache(moduleDefineHolder); + IPprofTaskQueryDAO taskQueryDAO = getPprofTaskQueryDAO(moduleDefineHolder); + + try { + List taskList = taskQueryDAO.getTaskList( + null, taskCache.getCacheStartTimeBucket(), taskCache.getCacheEndTimeBucket(), null + ); + if (CollectionUtils.isEmpty(taskList)) { + return; + } + + for (PprofTask task : taskList) { + taskCache.saveTask(task.getServiceId(), task); + } + + } catch (IOException e) { + log.warn("Unable to update pprof task cache", e); + } + } } \ No newline at end of file diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java new file mode 100644 index 000000000000..6a06227a5001 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.cache; + +import org.apache.skywalking.oap.server.library.module.Service; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.apache.skywalking.oap.server.core.CoreModuleConfig; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +public class PprofTaskCache implements Service { + private final Cache serviceId2taskCache; + + public PprofTaskCache(CoreModuleConfig moduleConfig) { + long initialSize = moduleConfig.getMaxSizeOfProfileTask() / 10L; + int initialCapacitySize = (int) (initialSize > Integer.MAX_VALUE ? Integer.MAX_VALUE : initialSize); + + serviceId2taskCache = CacheBuilder.newBuilder() + .initialCapacity(initialCapacitySize) + .maximumSize(moduleConfig.getMaxSizeOfProfileTask()) + // remove old pprof task data + .expireAfterWrite(Duration.ofMinutes(1)) + .build(); + } + + public PprofTask getPprofTask(String serviceId) { + PprofTask task = serviceId2taskCache.getIfPresent(serviceId); + return task; + } + + public void saveTask(String serviceId, PprofTask task) { + if (task == null) { + return ; + } + + serviceId2taskCache.put(serviceId, task); + } + + /** + * use for every db query, -5min start time + */ + public long getCacheStartTimeBucket() { + return TimeBucket.getRecordTimeBucket(System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(5)); + } + + /** + * use for every db query, +5min end time(because search through task's create time) + */ + public long getCacheEndTimeBucket() { + return TimeBucket.getRecordTimeBucket(System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(5)); + } + +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/command/CommandService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/command/CommandService.java index ac7ef401769a..40a8de67e06c 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/command/CommandService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/command/CommandService.java @@ -33,6 +33,7 @@ import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerEventType; import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerTask; import org.apache.skywalking.oap.server.core.query.type.EBPFProfilingTaskExtension; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; import org.apache.skywalking.oap.server.library.util.CollectionUtils; import org.apache.skywalking.oap.server.library.util.StringUtil; import org.apache.skywalking.oap.server.network.trace.component.command.AsyncProfilerTaskCommand; @@ -40,6 +41,7 @@ import org.apache.skywalking.oap.server.network.trace.component.command.ContinuousProfilingPolicyCommand; import org.apache.skywalking.oap.server.network.trace.component.command.EBPFProfilingTaskCommand; import org.apache.skywalking.oap.server.network.trace.component.command.EBPFProfilingTaskExtensionConfig; +import org.apache.skywalking.oap.server.network.trace.component.command.PprofTaskCommand; import org.apache.skywalking.oap.server.network.trace.component.command.ProfileTaskCommand; import org.apache.skywalking.oap.server.core.query.type.ProfileTask; import org.apache.skywalking.oap.server.library.module.ModuleManager; @@ -72,6 +74,19 @@ public AsyncProfilerTaskCommand newAsyncProfileTaskCommand(AsyncProfilerTask tas eventNames, task.getExecArgs(), task.getCreateTime()); } + /** + * Create a new pprof task command for Go agents + */ + public PprofTaskCommand newPprofTaskCommand(PprofTask task) { + final String serialNumber = UUID.randomUUID().toString(); + String events = ""; + if (task.getEvents() != null) { + events = task.getEvents().getName(); + } + return new PprofTaskCommand(serialNumber, task.getId(), events, + task.getDuration(), task.getCreateTime(), task.getDumpPeriod()); + } + /** * Used to notify the eBPF Profiling task to the eBPF agent side */ diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java new file mode 100644 index 000000000000..3d8264e90644 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof; + +import lombok.RequiredArgsConstructor; +import org.apache.skywalking.oap.server.library.module.Service; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import java.io.IOException; +import java.util.List; +import org.apache.skywalking.oap.server.core.analysis.worker.NoneStreamProcessor; +import java.util.concurrent.TimeUnit; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskCreationResult; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskCreationType; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; + +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RequiredArgsConstructor +public class PprofMutationService implements Service { + private final ModuleManager moduleManager; + + private IPprofTaskQueryDAO taskQueryDAO; + + private IPprofTaskQueryDAO getPprofTaskDAO() { + if (taskQueryDAO == null) { + this.taskQueryDAO = moduleManager.find(StorageModule.NAME) + .provider() + .getService(IPprofTaskQueryDAO.class); + } + return taskQueryDAO; + } + + public PprofTaskCreationResult createTask(String serviceId, + List serviceInstanceIds, + int duration, + PprofEventType events, + int dumpPeriod) throws IOException { + long createTime = System.currentTimeMillis(); + // check data + PprofTaskCreationResult checkResult = checkDataSuccess( + serviceId, serviceInstanceIds, duration, createTime, events, dumpPeriod + ); + if (checkResult != null) { + return checkResult; + } + // create task + PprofTaskRecord task = new PprofTaskRecord(); + String taskId = createTime + Const.ID_CONNECTOR + serviceId; + task.setTaskId(taskId); + task.setServiceId(serviceId); + task.setServiceInstanceIdsFromList(serviceInstanceIds); + task.setDuration(duration); + task.setEvents(events.toString()); + task.setDumpPeriod(dumpPeriod); + task.setCreateTime(createTime); + task.setTimeBucket(TimeBucket.getRecordTimeBucket(createTime)); + NoneStreamProcessor.getInstance().in(task); + return PprofTaskCreationResult.builder() + .id(task.id().build()) + .code(PprofTaskCreationType.SUCCESS) + .build(); + } + + private PprofTaskCreationResult checkDataSuccess(String serviceId, + List serviceInstanceIds, + int duration, + long createTime, + PprofEventType events, + int dumpPeriod) throws IOException { + String checkArgumentMessage = checkArgumentError(serviceId, serviceInstanceIds, duration, events, dumpPeriod); + if (checkArgumentMessage != null) { + return PprofTaskCreationResult.builder() + .code(PprofTaskCreationType.ARGUMENT_ERROR) + .errorReason(checkArgumentMessage) + .build(); + } + String checkTaskProfilingMessage = checkTaskProfiling(serviceId, createTime); + if (checkTaskProfilingMessage != null) { + return PprofTaskCreationResult.builder() + .code(PprofTaskCreationType.ALREADY_PROFILING_ERROR) + .errorReason(checkTaskProfilingMessage) + .build(); + } + return null; + } + + private String checkArgumentError(String serviceId, + List serviceInstanceIds, + int duration, + PprofEventType events, + int dumpPeriod) { + if (serviceId == null) { + return "service cannot be null"; + } + if (events == null) { + return "events cannot be empty"; + } + if (events == PprofEventType.CPU || events == PprofEventType.BLOCK || events == PprofEventType.MUTEX) { + if (duration <= 0) { + return "duration cannot be negative"; + } + } + if (events == PprofEventType.BLOCK || events == PprofEventType.MUTEX) { + if (dumpPeriod <= 0) { + return "dumpPeriod cannot be negative"; + } + } + if (CollectionUtils.isEmpty(serviceInstanceIds)) { + return "serviceInstanceIds cannot be empty"; + } + return null; + } + + private String checkTaskProfiling(String serviceId, + long createTime) throws IOException { + // Each service can only enable one task at a time + long endTimeBucket = TimeBucket.getMinuteTimeBucket(createTime); + final List alreadyHaveTaskList = getPprofTaskDAO().getTaskList( + serviceId, null, endTimeBucket, 1 + ); + if (CollectionUtils.isNotEmpty(alreadyHaveTaskList)) { + for (PprofTask task : alreadyHaveTaskList) { + if (task.getCreateTime() + TimeUnit.SECONDS.toMillis(task.getDuration()) >= createTime) { + // if the endTime is greater or equal than the createTime of the newly created task, i.e. there is overlap between two tasks, it is an invalid case, it will return an error + return "current service already has monitor pprof task execute at this time"; + } + } + } + return null; + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofQueryService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofQueryService.java new file mode 100644 index 000000000000..2d456996ac02 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofQueryService.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof; + +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.library.module.Service; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import java.util.Objects; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import java.io.IOException; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; +import org.apache.skywalking.oap.server.core.query.type.PprofStackTree; +import org.apache.skywalking.oap.server.library.pprof.parser.PprofMergeBuilder; +import java.util.List; +import com.google.gson.Gson; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RequiredArgsConstructor +public class PprofQueryService implements Service { + private static final Gson GSON = new Gson(); + + private final ModuleManager moduleManager; + + private IPprofTaskQueryDAO taskQueryDAO; + private IPprofDataQueryDAO dataQueryDAO; + private IPprofTaskLogQueryDAO logQueryDAO; + + private IPprofTaskQueryDAO getTaskQueryDAO() { + if (taskQueryDAO == null) { + this.taskQueryDAO = moduleManager.find(StorageModule.NAME) + .provider() + .getService(IPprofTaskQueryDAO.class); + } + return taskQueryDAO; + } + + private IPprofDataQueryDAO getPprofDataQueryDAO() { + if (dataQueryDAO == null) { + this.dataQueryDAO = moduleManager.find(StorageModule.NAME) + .provider() + .getService(IPprofDataQueryDAO.class); + } + return dataQueryDAO; + } + + private IPprofTaskLogQueryDAO getTaskLogQueryDAO() { + if (logQueryDAO == null) { + this.logQueryDAO = moduleManager.find(StorageModule.NAME) + .provider() + .getService(IPprofTaskLogQueryDAO.class); + } + return logQueryDAO; + } + + public List queryTask(String serviceId, Duration duration, Integer limit) throws IOException { + Long startTimeBucket = null; + Long endTimeBucket = null; + if (Objects.nonNull(duration)) { + startTimeBucket = duration.getStartTimeBucketInSec(); + endTimeBucket = duration.getEndTimeBucketInSec(); + } + List tasks = getTaskQueryDAO().getTaskList(serviceId, startTimeBucket, endTimeBucket, limit); + return tasks; + } + + public PprofStackTree queryPprofData(String taskId, List instanceIds) throws IOException { + List pprofDataList = getPprofDataQueryDAO().getByTaskIdAndInstances(taskId, instanceIds); + List trees = pprofDataList.stream() + .map(data -> GSON.fromJson(new String(data.getDataBinary()), FrameTree.class)) + .collect(Collectors.toList()); + FrameTree resultTree = new PprofMergeBuilder() + .merge(trees) + .build(); + return new PprofStackTree(resultTree); + } + + public List queryPprofTaskLogs(String taskId) throws IOException { + List taskLogList = getTaskLogQueryDAO().getTaskLogList(); + return findMatchedLogs(taskId, taskLogList); + } + + private List findMatchedLogs(final String taskID, final List allLogs) { + return allLogs.stream() + .filter(l -> Objects.equals(l.getId(), taskID)) + .map(this::extendTaskLog) + .collect(Collectors.toList()); + } + + private PprofTaskLog extendTaskLog(PprofTaskLog log) { + final IDManager.ServiceInstanceID.InstanceIDDefinition instanceIDDefinition = IDManager.ServiceInstanceID + .analysisId(log.getInstanceId()); + log.setInstanceName(instanceIDDefinition.getName()); + return log; + } + +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java new file mode 100644 index 000000000000..c35e197e9f18 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof.storage; + +import com.google.gson.Gson; +import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher; +import org.apache.skywalking.oap.server.core.source.PprofProfilingData; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; + +public class PprofProfilingDataDispatcher implements SourceDispatcher { + private static final Gson GSON = new Gson(); + + @Override + public void dispatch(PprofProfilingData source) { + PprofProfilingDataRecord record = new PprofProfilingDataRecord(); + record.setTaskId(source.getTaskId()); + record.setInstanceId(source.getInstanceId()); + record.setDataBinary(GSON.toJson(source.getFrameTree()).getBytes()); + record.setUploadTime(source.getUploadTime()); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(source.getUploadTime())); + RecordStreamProcessor.getInstance().in(record); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java new file mode 100644 index 000000000000..fd091c51b5c0 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof.storage; + +import lombok.Data; +import org.apache.skywalking.oap.server.core.analysis.record.Record; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB; +import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.core.analysis.Stream; +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.PPROF_PROFILING_DATA; +import org.apache.skywalking.oap.server.core.storage.StorageID; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; +import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; +import com.google.common.hash.Hashing; +import java.nio.charset.StandardCharsets; + +@Data +@Stream(name = PprofProfilingDataRecord.INDEX_NAME, scopeId = PPROF_PROFILING_DATA, + builder = PprofProfilingDataRecord.Builder.class, processor = RecordStreamProcessor.class) +@BanyanDB.TimestampColumn(PprofProfilingDataRecord.UPLOAD_TIME) +@BanyanDB.Group(streamGroup = BanyanDB.StreamGroup.RECORDS) +public class PprofProfilingDataRecord extends Record { + public static final String INDEX_NAME = "pprof_profiling_data"; + public static final String TASK_ID = "task_id"; + public static final String INSTANCE_ID = "instance_id"; + public static final String DATA_BINARY = "data_binary"; + public static final String UPLOAD_TIME = "upload_time"; + + @Column(name = TASK_ID) + private String taskId; + + @Column(name = INSTANCE_ID) + @BanyanDB.SeriesID(index = 0) + private String instanceId; + + @Column(name = UPLOAD_TIME) + private long uploadTime; + + @Column(name = DATA_BINARY, storageOnly = true) + private byte[] dataBinary; + + @Override + public StorageID id() { + return new StorageID().append( + Hashing.sha256().newHasher() + .putString(taskId, StandardCharsets.UTF_8) + .putString(instanceId, StandardCharsets.UTF_8) + .putLong(uploadTime) + .hash().toString() + ); + } + + public static class Builder implements StorageBuilder { + @Override + public PprofProfilingDataRecord storage2Entity(final Convert2Entity converter) { + final PprofProfilingDataRecord dataTraffic = new PprofProfilingDataRecord(); + dataTraffic.setTimeBucket(((Number) converter.get(TIME_BUCKET)).longValue()); + dataTraffic.setTaskId((String) converter.get(TASK_ID)); + dataTraffic.setInstanceId((String) converter.get(INSTANCE_ID)); + dataTraffic.setUploadTime(((Number) converter.get(UPLOAD_TIME)).longValue()); + dataTraffic.setDataBinary(converter.getBytes(DATA_BINARY)); + return dataTraffic; + } + + @Override + public void entity2Storage(final PprofProfilingDataRecord storageData, final Convert2Storage converter) { + converter.accept(TIME_BUCKET, storageData.getTimeBucket()); + converter.accept(TASK_ID, storageData.getTaskId()); + converter.accept(INSTANCE_ID, storageData.getInstanceId()); + converter.accept(UPLOAD_TIME, storageData.getUploadTime()); + converter.accept(DATA_BINARY, storageData.getDataBinary()); + } + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java new file mode 100644 index 000000000000..f26496c84f2b --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof.storage; + +import org.apache.skywalking.oap.server.core.analysis.Stream; +import org.apache.skywalking.oap.server.core.analysis.record.Record; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.source.ScopeDeclaration; +import org.apache.skywalking.oap.server.core.storage.StorageID; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB; +import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearch; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; +import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.PPROF_TASK_LOG; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +@ScopeDeclaration(id = PPROF_TASK_LOG, name = "PprofTaskLog") +@Stream(name = PprofTaskLogRecord.INDEX_NAME, scopeId = PPROF_TASK_LOG, builder = PprofTaskLogRecord.Builder.class, processor = RecordStreamProcessor.class) +@BanyanDB.TimestampColumn(PprofTaskLogRecord.TIMESTAMP) +@BanyanDB.Group(streamGroup = BanyanDB.StreamGroup.RECORDS) +public class PprofTaskLogRecord extends Record { + public static final String INDEX_NAME = "pprof_task_log"; + public static final String TASK_ID = "task_id"; + public static final String INSTANCE_ID = "instance_id"; + public static final String OPERATION_TYPE = "operation_type"; + public static final String OPERATION_TIME = "operation_time"; + public static final String TIMESTAMP = "timestamp"; + + @Column(name = TASK_ID) + private String taskId; + @Column(name = INSTANCE_ID) + @BanyanDB.SeriesID(index = 0) + private String instanceId; + @Column(name = OPERATION_TYPE, storageOnly = true) + private int operationType; + @ElasticSearch.EnableDocValues + @Column(name = OPERATION_TIME) + private long operationTime; + @Getter + @Setter + @ElasticSearch.EnableDocValues + @Column(name = TIMESTAMP) + private long timestamp; + + @Override + public StorageID id() { + return new StorageID() + .append(TASK_ID, getTaskId()) + .append(INSTANCE_ID, getInstanceId()) + .append(OPERATION_TYPE, getOperationType()) + .append(OPERATION_TIME, getOperationTime()); + } + + public static class Builder implements StorageBuilder { + @Override + public PprofTaskLogRecord storage2Entity(final Convert2Entity converter) { + final PprofTaskLogRecord log = new PprofTaskLogRecord(); + log.setTaskId((String) converter.get(TASK_ID)); + log.setInstanceId((String) converter.get(INSTANCE_ID)); + log.setOperationType(((Number) converter.get(OPERATION_TYPE)).intValue()); + log.setOperationTime(((Number) converter.get(OPERATION_TIME)).longValue()); + log.setTimestamp(((Number) converter.get(TIMESTAMP)).longValue()); + log.setTimeBucket(((Number) converter.get(TIME_BUCKET)).longValue()); + return log; + } + + @Override + public void entity2Storage(final PprofTaskLogRecord storageData, final Convert2Storage converter) { + converter.accept(TASK_ID, storageData.getTaskId()); + converter.accept(INSTANCE_ID, storageData.getInstanceId()); + converter.accept(OPERATION_TYPE, storageData.getOperationType()); + converter.accept(OPERATION_TIME, storageData.getOperationTime()); + converter.accept(TIME_BUCKET, storageData.getTimeBucket()); + converter.accept(TIMESTAMP, storageData.getTimestamp()); + } + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java new file mode 100644 index 000000000000..cf97a5c971c4 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof.storage; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.analysis.Stream; +import org.apache.skywalking.oap.server.core.analysis.config.NoneStream; +import org.apache.skywalking.oap.server.core.analysis.worker.NoneStreamProcessor; +import org.apache.skywalking.oap.server.core.source.ScopeDeclaration; +import org.apache.skywalking.oap.server.core.storage.StorageID; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB; +import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearch; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; +import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; +import java.util.List; + +import com.google.gson.Gson; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.PPROF_TASK; + +/** + * Pprof task database bean, use none stream + */ +@Getter +@Setter +@ScopeDeclaration(id = PPROF_TASK, name = "PprofTask") +@Stream(name = PprofTaskRecord.INDEX_NAME, scopeId = PPROF_TASK, builder = PprofTaskRecord.Builder.class, processor = NoneStreamProcessor.class) +@BanyanDB.TimestampColumn(PprofTaskRecord.CREATE_TIME) +@BanyanDB.Group(streamGroup = BanyanDB.StreamGroup.RECORDS) +public class PprofTaskRecord extends NoneStream { + private static final Gson GSON = new Gson(); + + public static final String INDEX_NAME = "pprof_task"; + public static final String TASK_ID = "task_id"; + public static final String SERVICE_ID = "service_id"; + public static final String SERVICE_INSTANCE_IDS = "service_instance_ids"; + public static final String CREATE_TIME = "create_time"; + public static final String EVENT_TYPES = "events"; + public static final String DURATION = "duration"; + public static final String DUMP_PERIOD = "dump_period"; + + @Override + public StorageID id() { + return new StorageID().append(TASK_ID, taskId); + } + + @Column(name = SERVICE_ID) + @BanyanDB.SeriesID(index = 0) + private String serviceId; + @Column(name = SERVICE_INSTANCE_IDS) + private String serviceInstanceIds; + @Column(name = TASK_ID) + private String taskId; + @ElasticSearch.EnableDocValues + @Column(name = CREATE_TIME) + private long createTime; + @ElasticSearch.EnableDocValues + @Column(name = EVENT_TYPES) + private String events; + @Column(name = DURATION) + private int duration; + @Column(name = DUMP_PERIOD) + private int dumpPeriod; + + public static class Builder implements StorageBuilder { + + @Override + public PprofTaskRecord storage2Entity(final Convert2Entity converter) { + PprofTaskRecord record = new PprofTaskRecord(); + record.setServiceId((String) converter.get(SERVICE_ID)); + record.setServiceInstanceIds((String) converter.get(SERVICE_INSTANCE_IDS)); + record.setTaskId((String) converter.get(TASK_ID)); + record.setCreateTime(((Number) converter.get(CREATE_TIME)).longValue()); + record.setEvents((String) converter.get(EVENT_TYPES)); + record.setDuration(((Number) converter.get(DURATION)).intValue()); + record.setDumpPeriod(((Number) converter.get(DUMP_PERIOD)).intValue()); + record.setTimeBucket(((Number) converter.get(TIME_BUCKET)).longValue()); + return record; + } + + @Override + public void entity2Storage(final PprofTaskRecord storageData, final Convert2Storage converter) { + converter.accept(SERVICE_ID, storageData.getServiceId()); + converter.accept(SERVICE_INSTANCE_IDS, storageData.getServiceInstanceIds()); + converter.accept(TASK_ID, storageData.getTaskId()); + converter.accept(CREATE_TIME, storageData.getCreateTime()); + converter.accept(EVENT_TYPES, storageData.getEvents()); + converter.accept(DURATION, storageData.getDuration()); + converter.accept(DUMP_PERIOD, storageData.getDumpPeriod()); + converter.accept(TIME_BUCKET, storageData.getTimeBucket()); + } + } + + public void setServiceInstanceIdsFromList(List serviceInstanceIds) { + this.serviceInstanceIds = GSON.toJson(serviceInstanceIds); + } + +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PprofTaskLog.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PprofTaskLog.java new file mode 100644 index 000000000000..471bde9cdfaa --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PprofTaskLog.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; + +@Setter +@Getter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class PprofTaskLog { + // task id + private String id; + + // instance + private String instanceId; + private String instanceName; + + // operation + private PprofTaskLogOperationType operationType; + private long operationTime; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofAnalyzationRequest.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofAnalyzationRequest.java new file mode 100644 index 000000000000..cb62845d2bee --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofAnalyzationRequest.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.input; + +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +@Getter +@Setter +public class PprofAnalyzationRequest { + private String taskId; + private List instanceIds; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskCreationRequest.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskCreationRequest.java new file mode 100644 index 000000000000..af19a9466d1e --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskCreationRequest.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.input; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; + +import java.util.List; + +@Getter +@Setter +public class PprofTaskCreationRequest { + private String serviceId; + private List serviceInstanceIds; + private int duration; + private PprofEventType events; + private int dumpPeriod; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java new file mode 100644 index 000000000000..a2304a04ba95 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.input; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class PprofTaskListRequest { + private String serviceId; + private Duration queryDuration; + private Integer limit; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/AlarmMessage.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/AlarmMessage.java index 7ff8e5306ccd..67dd14f234a8 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/AlarmMessage.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/AlarmMessage.java @@ -35,6 +35,7 @@ public class AlarmMessage { private String name; private String message; private Long startTime; + private Long recoveryTime; private transient String id1; private final List tags; private List events = new ArrayList<>(2); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofAnalyzation.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofAnalyzation.java new file mode 100644 index 000000000000..ef56f268acad --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofAnalyzation.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class PprofAnalyzation { + private PprofStackTree tree; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofEventType.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofEventType.java new file mode 100644 index 000000000000..333b0b061dd9 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofEventType.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public enum PprofEventType { + CPU(0, "cpu"), + HEAP(1, "heap"), + BLOCK(2, "block"), + MUTEX(3, "mutex"), + GOROUTINE(4, "goroutine"), + THREADCREATE(5, "threadcreate"), + ALLOCS(6, "allocs"); + + + private final int code; + private final String name; + + public static PprofEventType valueOfString(String event) { + return PprofEventType.valueOf(event); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackElement.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackElement.java new file mode 100644 index 000000000000..b82f3e1ec670 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackElement.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter + +public class PprofStackElement { + // work for tree building, id matches multiple parentId + private int id; + private int parentId; + + // stack code signature + private String codeSignature; + + private long total; + private long self; + +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackTree.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackTree.java new file mode 100644 index 000000000000..2600133dd98b --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackTree.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import com.google.common.collect.Lists; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; +import java.util.List; +import java.util.Objects; + +@Setter +@Getter +@NoArgsConstructor +public class PprofStackTree { + private List elements; + + private int idGen = 0; + + public PprofStackTree(FrameTree tree) { + this.elements = convertTree(-1, tree); + } + + private List convertTree(int parentId, FrameTree tree) { + PprofStackElement pprofStackElement = new PprofStackElement(); + pprofStackElement.setId(idGen++); + pprofStackElement.setParentId(parentId); + pprofStackElement.setCodeSignature(tree.getSignature()); + pprofStackElement.setTotal(tree.getTotal()); + pprofStackElement.setSelf(tree.getSelf()); + + List children = tree.getChildren(); + List result = Lists.newArrayList(pprofStackElement); + if (Objects.isNull(children) || children.isEmpty()) { + return result; + } + + for (FrameTree child : children) { + List childElements = convertTree(pprofStackElement.getId(), child); + result.addAll(childElements); + } + + return result; + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTask.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTask.java new file mode 100644 index 000000000000..3d8e4e3ba5bc --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTask.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Setter +@Getter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class PprofTask { + + private String id; + private String serviceId; + private List serviceInstanceIds; + private PprofEventType events; + private long createTime; + private int duration; + private int dumpPeriod; + +} \ No newline at end of file diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java new file mode 100644 index 000000000000..10f7653e9aa6 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +/** + * create pprof task result + */ +@Setter +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class PprofTaskCreationResult { + // ErrorReason gives detailed reason for the exception, if the code returned represents a kind of failure. + private String errorReason; + // Code defines the status of the response, i.e. success or failure. + private PprofTaskCreationType code; + // Task id, if code is SUCCESS. + private String id; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java new file mode 100644 index 000000000000..14519f62d3c0 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +public enum PprofTaskCreationType { + SUCCESS, + ARGUMENT_ERROR, + ALREADY_PROFILING_ERROR, + ; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskListResult.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskListResult.java new file mode 100644 index 000000000000..e3ee7a75a99d --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskListResult.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.util.List; + +@Data +@AllArgsConstructor +public class PprofTaskListResult { + private String errorReason; + private List tasks; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskLogOperationType.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskLogOperationType.java new file mode 100644 index 000000000000..e220ceb23d37 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskLogOperationType.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import java.util.HashMap; +import java.util.Map; + +public enum PprofTaskLogOperationType { + NOTIFIED(1), // when sniffer has execution finished to report + EXECUTION_FINISHED(2), // when sniffer has execution finished to report + PPROF_UPLOAD_FILE_TOO_LARGE_ERROR(3), // when sniffer finished task but jfr file is to large that oap server can not receive + EXECUTION_TASK_ERROR(4) // when sniffer fails to execute its task + ; + + private final int code; + private static final Map CACHE = new HashMap(); + + static { + for (PprofTaskLogOperationType val : PprofTaskLogOperationType.values()) { + CACHE.put(val.getCode(), val); + } + } + + /** + * Parse operation type by code + */ + public static PprofTaskLogOperationType parse(int code) { + return CACHE.get(code); + } + + PprofTaskLogOperationType(int code) { + this.code = code; + } + + public int getCode() { + return this.code; + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskProgress.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskProgress.java new file mode 100644 index 000000000000..991e6e4f6d95 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskProgress.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.Data; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; + +import java.util.List; + +@Data +public class PprofTaskProgress { + private List logs; + private List errorInstanceIds; + private List successInstanceIds; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java index 71fe59b4e0cc..359910b05a89 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java @@ -152,6 +152,9 @@ public class DefaultScopeDefine { public static final int BROWSER_APP_WEB_INTERACTION_PAGE_PERF = 89; public static final int SW_SPAN_ATTACHED_EVENT = 90; public static final int SERVICE_DATABASE_SLOW_STATEMENT = 91; + public static final int PPROF_TASK = 92; + public static final int PPROF_PROFILING_DATA = 93; + public static final int PPROF_TASK_LOG = 94; /** * Catalog of scope, the metrics processor could use this to group all generated metrics by oal rt. diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/PprofProfilingData.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/PprofProfilingData.java new file mode 100644 index 000000000000..d9c23835f86c --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/PprofProfilingData.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.source; + +import lombok.Data; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.PPROF_PROFILING_DATA; + +@Data +@ScopeDeclaration(id = PPROF_PROFILING_DATA, name = "PprofProfilingData") +@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class) +public class PprofProfilingData extends Source { + private volatile String entityId; + + @Override + public int scope() { + return PPROF_PROFILING_DATA; + } + + @Override + public String getEntityId() { + if (entityId == null) { + return taskId + instanceId + eventType.name() + uploadTime; + } + return entityId; + } + + private String taskId; + private String instanceId; + private long uploadTime; + private PprofEventType eventType; + private Object frameTree; +} \ No newline at end of file diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java index 89ac015cd0b4..cd711dedeaa7 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java @@ -24,6 +24,9 @@ import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IJFRDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.continuous.IContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IServiceLabelDAO; import org.apache.skywalking.oap.server.core.storage.profiling.trace.IProfileTaskLogQueryDAO; @@ -96,6 +99,9 @@ public Class[] services() { IAsyncProfilerTaskQueryDAO.class, IAsyncProfilerTaskLogQueryDAO.class, IJFRDataQueryDAO.class, + IPprofTaskQueryDAO.class, + IPprofTaskLogQueryDAO.class, + IPprofDataQueryDAO.class, StorageTTLStatusQuery.class }; } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofDataQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofDataQueryDAO.java new file mode 100644 index 000000000000..523fa73d419b --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofDataQueryDAO.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.oap.server.core.storage.profiling.pprof; + +import org.apache.skywalking.oap.server.library.module.Service; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import java.io.IOException; +import java.util.List; + +public interface IPprofDataQueryDAO extends Service { + /** + * get pprof data record + * + * @param taskId taskId + * @param instanceIds instances of successfully uploaded file and parsed + * @return record list + */ + List getByTaskIdAndInstances(final String taskId, List instanceIds) throws IOException; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java new file mode 100644 index 000000000000..ba6b652dc396 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.storage.profiling.pprof; + +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.storage.DAO; +import java.io.IOException; +import java.util.List; + +public interface IPprofTaskLogQueryDAO extends DAO { + /** + * search all task log list in appoint task id + */ + List getTaskLogList() throws IOException; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskQueryDAO.java new file mode 100644 index 000000000000..4c4945654512 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskQueryDAO.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.storage.profiling.pprof; + +import org.apache.skywalking.oap.server.core.storage.DAO; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import java.io.IOException; +import java.util.List; + +public interface IPprofTaskQueryDAO extends DAO { + + /** + * search task list in appoint time bucket + * + * @param serviceId monitor service id, maybe null + * @param startTimeBucket time bucket bigger than or equals, nullable + * @param endTimeBucket time bucket smaller than or equals, nullable + * @param limit limit count, if null means query all + */ + List getTaskList(final String serviceId, final Long startTimeBucket, + final Long endTimeBucket, final Integer limit) throws IOException; + + /** + * query profile task by id + * + * @param id taskId + * @return task data + */ + PprofTask getById(final String id) throws IOException; + +} diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/CoreModuleTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/CoreModuleTest.java index a8e0098a72e2..cd1edcc1cac9 100644 --- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/CoreModuleTest.java +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/CoreModuleTest.java @@ -26,6 +26,6 @@ public class CoreModuleTest { public void testOpenServiceList() { CoreModule coreModule = new CoreModule(); - Assertions.assertEquals(49, coreModule.services().length); + Assertions.assertEquals(52, coreModule.services().length); } } diff --git a/oap-server/server-library/library-pprof-parser/pom.xml b/oap-server/server-library/library-pprof-parser/pom.xml new file mode 100755 index 000000000000..25a549747ea2 --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/pom.xml @@ -0,0 +1,104 @@ + + + + + 4.0.0 + + + org.apache.skywalking + server-library + ${revision} + + + library-pprof-parser + + + 11 + 11 + UTF-8 + true + + + + + com.google.protobuf + protobuf-java + 3.25.5 + + + com.google.code.gson + gson + 2.10.1 + + + org.projectlombok + lombok + 1.18.30 + provided + + + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.6.1 + + com.google.protobuf:protoc:3.25.5:exe:${os.detected.classifier} + + + + + compile + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.3.0 + + + add-generated-sources + generate-sources + + add-source + + + + target/generated-sources/protobuf/java + + + + + + + + + kr.motd.maven + os-maven-plugin + 1.7.0 + + + + \ No newline at end of file diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java new file mode 100644 index 000000000000..3e4e0f9f0082 --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.pprof.parser; + +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; +import org.apache.skywalking.oap.server.library.pprof.type.Frame; +import java.util.List; + +public class PprofMergeBuilder { + private final Frame root = new Frame("root"); + + public PprofMergeBuilder merge(List trees) { + if (trees == null || trees.isEmpty()) { + return this; + } + for (FrameTree tree : trees) { + merge0(root, tree); + } + return this; + } + + public PprofMergeBuilder merge(FrameTree tree) { + merge0(root, tree); + return this; + } + + private void merge0(Frame frame, FrameTree tree) { + if (tree == null) { + return; + } + if (tree.getChildren() != null) { + for (FrameTree childTree : tree.getChildren()) { + Frame child = addChild(frame, childTree.getSignature()); + merge0(child, childTree); + } + } + frame.setTotal(frame.getTotal() + tree.getTotal()); + frame.setSelf(frame.getSelf() + tree.getSelf()); + } + + private Frame addChild(Frame parent, String signature) { + return parent.getChild(signature); + } + + public FrameTree build() { + return FrameTree.buildTree(root); + } + +} diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java new file mode 100644 index 000000000000..9fe8ab776b3f --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.pprof.parser; + +import com.google.perftools.profiles.ProfileProto; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.zip.GZIPInputStream; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTreeBuilder; + +/** + * Parses pprof protobuf format files and converts them to frame trees. + */ +public class PprofParser { + + public static FrameTree dumpTree(ByteBuffer buf) throws IOException { + byte[] bytes = new byte[buf.remaining()]; + buf.get(bytes); + InputStream stream = new java.io.ByteArrayInputStream(bytes); + InputStream inputStream = new GZIPInputStream(stream); + ProfileProto.Profile profile = ProfileProto.Profile.parseFrom(inputStream); + FrameTree tree = new FrameTreeBuilder(profile).build(); + return tree; + + } + + public static FrameTree dumpTree(String filePath) throws IOException { + File file = new File(filePath); + if (!file.exists()) { + throw new IOException("Pprof file not found: " + filePath); + } + InputStream fileStream = new FileInputStream(file); + InputStream stream = new GZIPInputStream(fileStream); + ProfileProto.Profile profile = ProfileProto.Profile.parseFrom(stream); + FrameTree tree = new FrameTreeBuilder(profile).build(); + return tree; + } +} diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java new file mode 100644 index 000000000000..194829d9262b --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.pprof.type; + +import java.util.HashMap; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class Frame extends HashMap { + final String signature; + long total; + long self; + + + public Frame(String signature) { + this.signature = signature; + } + + public Frame getChild(String signature) { + return super.computeIfAbsent(signature.hashCode(), k -> new Frame(signature)); + } +} diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java new file mode 100755 index 000000000000..9a2f240ccdd3 --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.pprof.type; + +import java.util.ArrayList; +import java.util.List; +import lombok.Getter; +@Getter +public class FrameTree { + private String signature; + private long total; + private long self; + private List children; + + public FrameTree(Frame frame) { + this.signature = frame.getSignature(); + this.total = frame.getTotal(); + this.self = frame.getSelf(); + this.children = new ArrayList<>(frame.size()); + } + public FrameTree(String signature, long total, long self) { + this.signature = signature; + this.total = total; + this.self = self; + this.children = new ArrayList<>(); + } + + public static FrameTree buildTree(Frame frame) { + if (frame == null) return null; + + FrameTree frameTree = new FrameTree(frame); + // has children? + if (!frame.isEmpty()) { + frameTree.children = new ArrayList<>(frame.size()); + // build tree + for (Frame childFrame : frame.values()) { + FrameTree childFrameTree = buildTree(childFrame); + frameTree.children.add(childFrameTree); + } + } + return frameTree; + } +} diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java new file mode 100644 index 000000000000..c42f504ad8a2 --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.pprof.type; + +import com.google.perftools.profiles.ProfileProto; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Data +@NoArgsConstructor +@AllArgsConstructor +class RawFrameTree { + private long locationId; + private long total; + private long self; + private final Map children = new HashMap<>(); +} + +public class FrameTreeBuilder { + private final ProfileProto.Profile profile; + private final RawFrameTree root; + + public FrameTreeBuilder(ProfileProto.Profile profile) { + this.profile = profile; + this.root = new RawFrameTree(0, 0, 0); + } + + private FrameTree parseTree(RawFrameTree rawTree) { + FrameTree tree = new FrameTree(getSignature(rawTree.getLocationId()), rawTree.getTotal(), rawTree.getSelf()); + for (RawFrameTree rawChild : rawTree.getChildren().values()) { + FrameTree child = parseTree(rawChild); + tree.getChildren().add(child); + } + return tree; + } + private String getSignature(long locationId) { + if (locationId == 0) { + return "root"; + } + ProfileProto.Location location = profile.getLocation((int) locationId - 1); + return location.getLineList().stream().map((line) -> { + ProfileProto.Function function = profile.getFunction((int) line.getFunctionId() - 1); + String functionName = profile.getStringTable((int) function.getName()); + return functionName + ":" + line.getLine(); + }).collect(Collectors.joining(";")); + } + + public FrameTree build() { + for (ProfileProto.Sample sample : profile.getSampleList()) { + mergeSample(sample); + } + return parseTree(this.root); + } + + private void mergeSample(ProfileProto.Sample sample) { + // merge sample data + Map children = root.getChildren(); + List locationIdList = new ArrayList<>(sample.getLocationIdList()); + // from root to leaf + Collections.reverse(locationIdList); + int size = locationIdList.size(); + for (int i = 0; i < size; i++) { + boolean isEnd = i == size - 1; + long locationId = locationIdList.get(i); + if (children.containsKey(locationId)) { + // if the child exists, merge the sample data + RawFrameTree child = children.get(locationId); + child.setTotal(child.getTotal() + 1); + child.setSelf(child.getSelf() + (isEnd ? 1 : 0)); + children = child.getChildren(); + } else { + // if the child does not exist, create a new child + RawFrameTree child = new RawFrameTree(locationId, 1, (isEnd ? 1 : 0)); + children.put(locationId, child); + children = child.getChildren(); + } + } + root.setTotal(root.getTotal() + 1); + } +} \ No newline at end of file diff --git a/oap-server/server-library/library-pprof-parser/src/main/proto/profile.proto b/oap-server/server-library/library-pprof-parser/src/main/proto/profile.proto new file mode 100755 index 000000000000..d1cb0a5a4040 --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/proto/profile.proto @@ -0,0 +1,250 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Profile is a common stacktrace profile format. +// +// Measurements represented with this format should follow the +// following conventions: +// +// - Consumers should treat unset optional fields as if they had been +// set with their default value. +// +// - When possible, measurements should be stored in "unsampled" form +// that is most useful to humans. There should be enough +// information present to determine the original sampled values. +// +// - On-disk, the serialized proto must be gzip-compressed. +// +// - The profile is represented as a set of samples, where each sample +// references a sequence of locations, and where each location belongs +// to a mapping. +// - There is a N->1 relationship from sample.location_id entries to +// locations. For every sample.location_id entry there must be a +// unique Location with that id. +// - There is an optional N->1 relationship from locations to +// mappings. For every nonzero Location.mapping_id there must be a +// unique Mapping with that id. + +syntax = "proto3"; + +package perftools.profiles; + +option java_package = "com.google.perftools.profiles"; +option java_outer_classname = "ProfileProto"; + +message Profile { + // A description of the samples associated with each Sample.value. + // For a cpu profile this might be: + // [["cpu","nanoseconds"]] or [["wall","seconds"]] or [["syscall","count"]] + // For a heap profile, this might be: + // [["allocations","count"], ["space","bytes"]], + // If one of the values represents the number of events represented + // by the sample, by convention it should be at index 0 and use + // sample_type.unit == "count". + repeated ValueType sample_type = 1; + // The set of samples recorded in this profile. + repeated Sample sample = 2; + // Mapping from address ranges to the image/binary/library mapped + // into that address range. mapping[0] will be the main binary. + repeated Mapping mapping = 3; + // Locations referenced by samples. + repeated Location location = 4; + // Functions referenced by locations. + repeated Function function = 5; + // A common table for strings referenced by various messages. + // string_table[0] must always be "". + repeated string string_table = 6; + // frames with Function.function_name fully matching the following + // regexp will be dropped from the samples, along with their successors. + int64 drop_frames = 7; // Index into string table. + // frames with Function.function_name fully matching the following + // regexp will be kept, even if it matches drop_frames. + int64 keep_frames = 8; // Index into string table. + + // The following fields are informational, do not affect + // interpretation of results. + + // Time of collection (UTC) represented as nanoseconds past the epoch. + int64 time_nanos = 9; + // Duration of the profile, if a duration makes sense. + int64 duration_nanos = 10; + // The kind of events between sampled occurrences. + // e.g [ "cpu","cycles" ] or [ "heap","bytes" ] + ValueType period_type = 11; + // The number of events between sampled occurrences. + int64 period = 12; + // Free-form text associated with the profile. The text is displayed as is + // to the user by the tools that read profiles (e.g. by pprof). This field + // should not be used to store any machine-readable information, it is only + // for human-friendly content. The profile must stay functional if this field + // is cleaned. + repeated int64 comment = 13; // Indices into string table. + // Index into the string table of the type of the preferred sample + // value. If unset, clients should default to the last sample value. + int64 default_sample_type = 14; + // Documentation link for this profile. The URL must be absolute, + // e.g., http://pprof.example.com/cpu-profile.html + // + // The URL may be missing if the profile was generated by older code or code + // that did not bother to supply a link. + int64 doc_url = 15; // Index into string table. +} + +// ValueType describes the semantics and measurement units of a value. +message ValueType { + int64 type = 1; // Index into string table. + int64 unit = 2; // Index into string table. +} + +// Each Sample records values encountered in some program +// context. The program context is typically a stack trace, perhaps +// augmented with auxiliary information like the thread-id, some +// indicator of a higher level request being handled etc. +message Sample { + // The ids recorded here correspond to a Profile.location.id. + // The leaf is at location_id[0]. + repeated uint64 location_id = 1; + // The type and unit of each value is defined by the corresponding + // entry in Profile.sample_type. All samples must have the same + // number of values, the same as the length of Profile.sample_type. + // When aggregating multiple samples into a single sample, the + // result has a list of values that is the element-wise sum of the + // lists of the originals. + repeated int64 value = 2; + // label includes additional context for this sample. It can include + // things like a thread id, allocation size, etc. + // + // NOTE: While possible, having multiple values for the same label key is + // strongly discouraged and should never be used. Most tools (e.g. pprof) do + // not have good (or any) support for multi-value labels. And an even more + // discouraged case is having a string label and a numeric label of the same + // name on a sample. Again, possible to express, but should not be used. + repeated Label label = 3; +} + +message Label { + // Index into string table. An annotation for a sample (e.g. + // "allocation_size") with an associated value. + // Keys with "pprof::" prefix are reserved for internal use by pprof. + int64 key = 1; + + // At most one of the following must be present + int64 str = 2; // Index into string table + int64 num = 3; + + // Should only be present when num is present. + // Specifies the units of num. + // Use arbitrary string (for example, "requests") as a custom count unit. + // If no unit is specified, consumer may apply heuristic to deduce the unit. + // Consumers may also interpret units like "bytes" and "kilobytes" as memory + // units and units like "seconds" and "nanoseconds" as time units, + // and apply appropriate unit conversions to these. + int64 num_unit = 4; // Index into string table +} + +message Mapping { + // Unique nonzero id for the mapping. + uint64 id = 1; + // Address at which the binary (or DLL) is loaded into memory. + uint64 memory_start = 2; + // The limit of the address range occupied by this mapping. + uint64 memory_limit = 3; + // Offset in the binary that corresponds to the first mapped address. + uint64 file_offset = 4; + // The object this entry is loaded from. This can be a filename on + // disk for the main binary and shared libraries, or virtual + // abstractions like "[vdso]". + int64 filename = 5; // Index into string table + // A string that uniquely identifies a particular program version + // with high probability. E.g., for binaries generated by GNU tools, + // it could be the contents of the .note.gnu.build-id field. + int64 build_id = 6; // Index into string table + + // The following fields indicate the resolution of symbolic info. + bool has_functions = 7; + bool has_filenames = 8; + bool has_line_numbers = 9; + bool has_inline_frames = 10; +} + +// Describes function and line table debug information. +message Location { + // Unique nonzero id for the location. A profile could use + // instruction addresses or any integer sequence as ids. + uint64 id = 1; + // The id of the corresponding profile.Mapping for this location. + // It can be unset if the mapping is unknown or not applicable for + // this profile type. + uint64 mapping_id = 2; + // The instruction address for this location, if available. It + // should be within [Mapping.memory_start...Mapping.memory_limit] + // for the corresponding mapping. A non-leaf address may be in the + // middle of a call instruction. It is up to display tools to find + // the beginning of the instruction if necessary. + uint64 address = 3; + // Multiple line indicates this location has inlined functions, + // where the last entry represents the caller into which the + // preceding entries were inlined. + // + // E.g., if memcpy() is inlined into printf: + // line[0].function_name == "memcpy" + // line[1].function_name == "printf" + repeated Line line = 4; + // Provides an indication that multiple symbols map to this location's + // address, for example due to identical code folding by the linker. In that + // case the line information above represents one of the multiple + // symbols. This field must be recomputed when the symbolization state of the + // profile changes. + bool is_folded = 5; +} + +message Line { + // The id of the corresponding profile.Function for this line. + uint64 function_id = 1; + // Line number in source code. + int64 line = 2; + // Column number in source code. + int64 column = 3; +} + +message Function { + // Unique nonzero id for the function. + uint64 id = 1; + // Name of the function, in human-readable form if available. + int64 name = 2; // Index into string table + // Name of the function, as identified by the system. + // For instance, it can be a C++ mangled name. + int64 system_name = 3; // Index into string table + // Source file containing the function. + int64 filename = 4; // Index into string table + // Line number in source file. + int64 start_line = 5; +} \ No newline at end of file diff --git a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/CollectionUtils.java b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/CollectionUtils.java index d41bce7f2b49..110a4f341e8b 100644 --- a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/CollectionUtils.java +++ b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/CollectionUtils.java @@ -32,6 +32,10 @@ public static boolean isEmpty(List list) { return list == null || list.size() == 0; } + public static boolean isEmpty(String str) { + return str == null || str.length() == 0; + } + public static boolean isEmpty(Set set) { return set == null || set.size() == 0; } diff --git a/oap-server/server-library/pom.xml b/oap-server/server-library/pom.xml index 3e92ba599bae..93481db0742a 100644 --- a/oap-server/server-library/pom.xml +++ b/oap-server/server-library/pom.xml @@ -36,6 +36,7 @@ library-datacarrier-queue library-kubernetes-support library-async-profiler-jfr-parser + library-pprof-parser library-integration-test diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java index c2f2cfcfde82..a07954aa79d6 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java @@ -47,6 +47,8 @@ import org.apache.skywalking.oap.query.graphql.resolver.OndemandLogQuery; import org.apache.skywalking.oap.query.graphql.resolver.ProfileMutation; import org.apache.skywalking.oap.query.graphql.resolver.ProfileQuery; +import org.apache.skywalking.oap.query.graphql.resolver.PprofMutation; +import org.apache.skywalking.oap.query.graphql.resolver.PprofQuery; import org.apache.skywalking.oap.query.graphql.resolver.Query; import org.apache.skywalking.oap.query.graphql.resolver.RecordsQuery; import org.apache.skywalking.oap.query.graphql.resolver.TopNRecordsQuery; @@ -153,7 +155,9 @@ public void prepare() throws ServiceNotProvidedException { .resolvers(new RecordsQuery(getManager())) .file("query-protocol/hierarchy.graphqls").resolvers(new HierarchyQuery(getManager())) .file("query-protocol/async-profiler.graphqls") - .resolvers(new AsyncProfilerQuery(getManager()), new AsyncProfilerMutation(getManager())); + .resolvers(new AsyncProfilerQuery(getManager()), new AsyncProfilerMutation(getManager())) + .file("query-protocol/pprof.graphqls") + .resolvers(new PprofQuery(getManager()), new PprofMutation(getManager())); if (config.isEnableOnDemandPodLog()) { schemaBuilder diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofMutation.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofMutation.java new file mode 100644 index 000000000000..eb98f8d7825a --- /dev/null +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofMutation.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.query.graphql.resolver; + +import graphql.kickstart.tools.GraphQLMutationResolver; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofMutationService; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskCreationResult; +import org.apache.skywalking.oap.server.core.query.input.PprofTaskCreationRequest; + +import java.io.IOException; + +@Slf4j +public class PprofMutation implements GraphQLMutationResolver { + private final ModuleManager moduleManager; + + private PprofMutationService mutationService; + + public PprofMutation(ModuleManager moduleManager) { + this.moduleManager = moduleManager; + } + + private PprofMutationService getPprofMutationService() { + if (mutationService == null) { + this.mutationService = moduleManager.find(CoreModule.NAME) + .provider() + .getService(PprofMutationService.class); + } + return mutationService; + } + + public PprofTaskCreationResult createPprofTask(PprofTaskCreationRequest request) throws IOException { + PprofMutationService pprofMutationService = getPprofMutationService(); + return pprofMutationService.createTask(request.getServiceId(), request.getServiceInstanceIds(), + request.getDuration(), request.getEvents(), request.getDumpPeriod()); + } +} \ No newline at end of file diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofQuery.java new file mode 100644 index 000000000000..9b1096cfbdec --- /dev/null +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofQuery.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.query.graphql.resolver; + +import org.apache.skywalking.oap.server.core.CoreModule; +import groovy.util.logging.Slf4j; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import graphql.kickstart.tools.GraphQLQueryResolver; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofQueryService; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskListResult; +import org.apache.skywalking.oap.server.core.query.input.PprofTaskListRequest; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskProgress; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.oap.server.core.query.type.PprofAnalyzation; +import org.apache.skywalking.oap.server.core.query.type.PprofStackTree; +import org.apache.skywalking.oap.server.core.query.input.PprofAnalyzationRequest; +import java.util.ArrayList; +import java.io.IOException; +import java.util.List; + +@Slf4j +public class PprofQuery implements GraphQLQueryResolver { + private final ModuleManager moduleManager; + + private PprofQueryService queryService; + + public PprofQuery(ModuleManager moduleManager) { + this.moduleManager = moduleManager; + } + + private PprofQueryService getPprofQueryService() { + if (queryService == null) { + this.queryService = moduleManager.find(CoreModule.NAME).provider().getService(PprofQueryService.class); + } + return queryService; + } + + public PprofTaskListResult queryPprofTaskList(PprofTaskListRequest request) throws IOException { + List tasks = getPprofQueryService().queryTask( + request.getServiceId(), request.getQueryDuration(), request.getLimit() + ); + return new PprofTaskListResult(null, tasks); + } + + public PprofAnalyzation queryPprofAnalyze(PprofAnalyzationRequest request) throws IOException { + PprofStackTree eventFrameTrees = getPprofQueryService().queryPprofData( + request.getTaskId(), request.getInstanceIds() + ); + return new PprofAnalyzation(eventFrameTrees); + } + + public PprofTaskProgress queryPprofTaskProgress(String taskId) throws IOException { + PprofTaskProgress pprofTaskProgress = new PprofTaskProgress(); + List logs = getPprofQueryService().queryPprofTaskLogs(taskId); + pprofTaskProgress.setLogs(logs); + List errorInstances = new ArrayList<>(); + List successInstances = new ArrayList<>(); + logs.forEach(log -> { + if (PprofTaskLogOperationType.EXECUTION_FINISHED.equals(log.getOperationType())) { + successInstances.add(log.getInstanceId()); + } else if (PprofTaskLogOperationType.EXECUTION_TASK_ERROR.equals(log.getOperationType()) + || PprofTaskLogOperationType.PPROF_UPLOAD_FILE_TOO_LARGE_ERROR.equals(log.getOperationType())) { + errorInstances.add(log.getInstanceId()); + } + }); + pprofTaskProgress.setErrorInstanceIds(errorInstances); + pprofTaskProgress.setSuccessInstanceIds(successInstances); + return pprofTaskProgress; + } +} diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol index 97f9bfc9dbf9..4fc10625ba72 160000 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol @@ -1 +1 @@ -Subproject commit 97f9bfc9dbf9313a951a5fa3791e92186cfede05 +Subproject commit 4fc10625ba72ef4788972b4f7991a535065d609b diff --git a/oap-server/server-receiver-plugin/pom.xml b/oap-server/server-receiver-plugin/pom.xml index 17892a71a48e..068a86167c62 100644 --- a/oap-server/server-receiver-plugin/pom.xml +++ b/oap-server/server-receiver-plugin/pom.xml @@ -49,6 +49,7 @@ skywalking-telegraf-receiver-plugin aws-firehose-receiver skywalking-async-profiler-receiver-plugin + skywalking-pprof-receiver-plugin diff --git a/oap-server/server-receiver-plugin/receiver-proto/pom.xml b/oap-server/server-receiver-plugin/receiver-proto/pom.xml index 0c057f133cef..aaff44d8c39a 100644 --- a/oap-server/server-receiver-plugin/receiver-proto/pom.xml +++ b/oap-server/server-receiver-plugin/receiver-proto/pom.xml @@ -41,6 +41,14 @@ flatbuffers-java provided + + com.google.protobuf + protobuf-java + + + com.google.protobuf + protobuf-java-util + diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/pom.xml b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/pom.xml new file mode 100644 index 000000000000..985a5b30702f --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/pom.xml @@ -0,0 +1,53 @@ + + + + + + server-receiver-plugin + org.apache.skywalking + ${revision} + + 4.0.0 + + skywalking-pprof-receiver-plugin + + + + org.apache.skywalking + library-module + ${project.version} + + + org.apache.skywalking + skywalking-sharing-server-plugin + ${project.version} + + + org.apache.skywalking + apm-network + ${project.version} + + + org.apache.skywalking + library-pprof-parser + ${project.version} + + + diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModule.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModule.java new file mode 100644 index 000000000000..f768f1a7ec91 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModule.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.module; + +import org.apache.skywalking.oap.server.library.module.ModuleDefine; + +public class PprofModule extends ModuleDefine { + public static final String NAME = "receiver-pprof"; + + public PprofModule() { + super(NAME); + } + + @Override + public Class[] services() { + return new Class[] {}; + } +} \ No newline at end of file diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java new file mode 100644 index 000000000000..acb92227d020 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.module; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; + +@Getter +@Setter +public class PprofModuleConfig extends ModuleConfig { + /** + * Used to manage the maximum size of the pprof file that can be received, the unit is Byte + * default is 30M + */ + private int pprofMaxSize = 30 * 1024 * 1024; + /** + * default is true + *

+ * If memoryParserEnabled is true, then PprofByteBufCollectionObserver will be enabled + * will use memory to receive pprof files without writing files (this is currently used). + * This can prevent the oap server from crashing due to no volume mounting. + *

+ * If memoryParserEnabled is false, then PprofFileCollectionObserver will be enabled + * which uses createTemp to write files and then reads the files for parsing. + * The advantage of this is that it reduces memory and prevents the oap server from crashing due to + * insufficient memory, but it may report an error due to no volume mounting. + */ + private boolean memoryParserEnabled = true; +} diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/PprofModuleProvider.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/PprofModuleProvider.java new file mode 100644 index 000000000000..0ff1aacad79e --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/PprofModuleProvider.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.provider; + +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.server.GRPCHandlerRegister; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; +import org.apache.skywalking.oap.server.library.module.ModuleDefine; +import org.apache.skywalking.oap.server.library.module.ModuleProvider; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; +import org.apache.skywalking.oap.server.receiver.pprof.module.PprofModule; +import org.apache.skywalking.oap.server.receiver.pprof.module.PprofModuleConfig; +import org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler; +import org.apache.skywalking.oap.server.receiver.sharing.server.SharingServerModule; + +public class PprofModuleProvider extends ModuleProvider { + private PprofModuleConfig config; + + @Override + public String name() { + return "default"; + } + + @Override + public Class module() { + return PprofModule.class; + } + + @Override + public ConfigCreator newConfigCreator() { + return new ConfigCreator() { + @Override + public Class type() { + return PprofModuleConfig.class; + } + + @Override + public void onInitialized(final PprofModuleConfig initialized) { + config = initialized; + } + }; + } + + @Override + public void prepare() throws ServiceNotProvidedException, ModuleStartException { + + } + + @Override + public void start() throws ServiceNotProvidedException, ModuleStartException { + GRPCHandlerRegister grpcHandlerRegister = getManager().find(SharingServerModule.NAME) + .provider() + .getService(GRPCHandlerRegister.class); + PprofServiceHandler pprofServiceHandler = new PprofServiceHandler(getManager(), + config.getPprofMaxSize(), config.isMemoryParserEnabled()); + grpcHandlerRegister.addHandler(pprofServiceHandler); + } + + @Override + public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException { + } + + @Override + public String[] requiredModules() { + return new String[]{ + CoreModule.NAME, + SharingServerModule.NAME + }; + } + +} diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java new file mode 100644 index 000000000000..6fdf30538cd0 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.provider.handler; + +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; +import java.io.IOException; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.apm.network.pprof.v10.PprofData; +import org.apache.skywalking.apm.network.pprof.v10.PprofTaskGrpc; +import org.apache.skywalking.apm.network.pprof.v10.PprofCollectionResponse; +import org.apache.skywalking.apm.network.pprof.v10.PprofMetaData; + +import org.apache.skywalking.apm.network.common.v3.Commands; +import org.apache.skywalking.apm.network.pprof.v10.PprofTaskCommandQuery; + +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.command.CommandService; +import java.util.Objects; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.cache.PprofTaskCache; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskLogRecord; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import java.util.concurrent.TimeUnit; + +//import org.apache.skywalking.oap.server.core.storage.StorageModule; +//import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.server.grpc.GRPCHandler; +import org.apache.skywalking.oap.server.network.trace.component.command.PprofTaskCommand; +import org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream.PprofByteBufCollectionObserver; +import org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream.PprofCollectionMetaData; +import org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream.PprofFileCollectionObserver; + +@Slf4j +public class PprofServiceHandler extends PprofTaskGrpc.PprofTaskImplBase implements GRPCHandler { + + private final IPprofTaskQueryDAO taskDAO; + private final SourceReceiver sourceReceiver; + private final CommandService commandService; + private final PprofTaskCache taskCache; + private final int pprofMaxSize; + private final boolean memoryParserEnabled; + + public PprofServiceHandler(ModuleManager moduleManager, int pprofMaxSize, boolean memoryParserEnabled) { + this.taskDAO = moduleManager.find(StorageModule.NAME).provider().getService(IPprofTaskQueryDAO.class); + this.commandService = moduleManager.find(CoreModule.NAME).provider().getService(CommandService.class); + this.sourceReceiver = moduleManager.find(CoreModule.NAME).provider().getService(SourceReceiver.class); + this.taskCache = moduleManager.find(CoreModule.NAME).provider().getService(PprofTaskCache.class); + this.pprofMaxSize = pprofMaxSize; + this.memoryParserEnabled = memoryParserEnabled; + } + + @Override + public StreamObserver collect(StreamObserver responseObserver) { + return memoryParserEnabled ? + new PprofByteBufCollectionObserver(taskDAO, responseObserver, sourceReceiver, pprofMaxSize) + : new PprofFileCollectionObserver(taskDAO, responseObserver, sourceReceiver, pprofMaxSize); + } + + @Override + public void getPprofTaskCommands(PprofTaskCommandQuery request, StreamObserver responseObserver) { + String serviceId = IDManager.ServiceID.buildId(request.getService(), true); + String serviceInstanceId = IDManager.ServiceInstanceID.buildId(serviceId, request.getServiceInstance()); + PprofTask task = taskCache.getPprofTask(serviceId); + // if task is null or createTime is less than lastCommandTime, return empty commands + if (Objects.isNull(task) || task.getCreateTime() <= request.getLastCommandTime() || + (!CollectionUtils.isEmpty(task.getServiceInstanceIds()) && !task.getServiceInstanceIds().contains(serviceInstanceId))) { + responseObserver.onNext(Commands.newBuilder().build()); + responseObserver.onCompleted(); + return; + } + + PprofTaskCommand pprofTaskCommand = commandService.newPprofTaskCommand(task); + Commands commands = Commands.newBuilder().addCommands(pprofTaskCommand.serialize()).build(); + responseObserver.onNext(commands); + responseObserver.onCompleted(); + recordPprofTaskLog(task, serviceInstanceId, PprofTaskLogOperationType.NOTIFIED); + } + + public static void recordPprofTaskLog(PprofTask task, String instanceId, PprofTaskLogOperationType operationType) { + PprofTaskLogRecord logRecord = new PprofTaskLogRecord(); + logRecord.setTaskId(task.getId()); + logRecord.setInstanceId(instanceId); + logRecord.setOperationType(operationType.getCode()); + logRecord.setOperationTime(System.currentTimeMillis()); + long timestamp = task.getCreateTime() + TimeUnit.SECONDS.toMillis(task.getDuration()); + logRecord.setTimestamp(timestamp); + logRecord.setTimeBucket(TimeBucket.getRecordTimeBucket(timestamp)); + RecordStreamProcessor.getInstance().in(logRecord); + } + + public static PprofCollectionMetaData parseMetaData(PprofMetaData metaData, IPprofTaskQueryDAO taskDAO) throws IOException { + String taskId = metaData.getTaskId(); + PprofTask task = taskDAO.getById(taskId); + String serviceId = IDManager.ServiceID.buildId(metaData.getService(), true); + String serviceInstanceId = IDManager.ServiceInstanceID.buildId(serviceId, metaData.getServiceInstance()); + + return PprofCollectionMetaData.builder() + .task(task) + .serviceId(serviceId) + .instanceId(serviceInstanceId) + .type(metaData.getType()) + .contentSize(metaData.getContentSize()) + .uploadTime(System.currentTimeMillis()) + .build(); + } +} \ No newline at end of file diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java new file mode 100644 index 000000000000..833305e8f0b9 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream; + +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.library.pprof.parser.PprofParser; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; +import org.apache.skywalking.apm.network.pprof.v10.PprofData; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.apm.network.pprof.v10.PprofCollectionResponse; +import org.apache.skywalking.apm.network.pprof.v10.PprofProfilingStatus; +import org.apache.skywalking.oap.server.core.source.PprofProfilingData; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import static org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler.parseMetaData; +import static org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler.recordPprofTaskLog; + +@Slf4j +public class PprofByteBufCollectionObserver implements StreamObserver { + private final IPprofTaskQueryDAO taskDAO; + private final StreamObserver responseObserver; + private final SourceReceiver sourceReceiver; + private final int pprofMaxSize; + private PprofCollectionMetaData taskMetaData; + private ByteBuffer buf; + + public PprofByteBufCollectionObserver(IPprofTaskQueryDAO taskDAO, + StreamObserver responseObserver, + SourceReceiver sourceReceiver, int pprofMaxSize) { + this.taskDAO = taskDAO; + this.responseObserver = responseObserver; + this.sourceReceiver = sourceReceiver; + this.pprofMaxSize = pprofMaxSize; + } + + @Override + public void onNext(PprofData pprofData) { + try { + if (Objects.isNull(taskMetaData) && pprofData.hasMetadata()) { + taskMetaData = parseMetaData(pprofData.getMetadata(), taskDAO); + if (PprofProfilingStatus.PPROF_PROFILING_SUCCESS.equals(taskMetaData.getType())) { + int size = taskMetaData.getContentSize(); + if (pprofMaxSize >= size) { + buf = ByteBuffer.allocate(size); + // Send success response to allow client to continue uploading + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus(PprofProfilingStatus.PPROF_PROFILING_SUCCESS) + .build()); + } else { + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus(PprofProfilingStatus.PPROF_TERMINATED_BY_OVERSIZE) + .build()); + recordPprofTaskLog(taskMetaData.getTask(), taskMetaData.getInstanceId(), PprofTaskLogOperationType.PPROF_UPLOAD_FILE_TOO_LARGE_ERROR); + } + } else { + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus(PprofProfilingStatus.PPROF_EXECUTION_TASK_ERROR) + .build()); + recordPprofTaskLog(taskMetaData.getTask(), taskMetaData.getInstanceId(), PprofTaskLogOperationType.EXECUTION_TASK_ERROR); + } + } else if (pprofData.hasContent()) { + if (buf != null) { + pprofData.getContent().copyTo(buf); + log.info("Received {} bytes of pprof data", pprofData.getContent().size()); + } + } + } catch (IOException e) { + log.error("Error processing pprof data", e); + responseObserver.onError(Status.INTERNAL.withDescription("Error processing pprof data: " + e.getMessage()).asRuntimeException()); + } + } + + @Override + public void onError(Throwable throwable) { + Status status = Status.fromThrowable(throwable); + if (Status.CANCELLED.getCode() == status.getCode()) { + if (log.isDebugEnabled()) { + log.debug(throwable.getMessage(), throwable); + } + return; + } + log.error("Error in receiving pprof profiling data", throwable); + + } + + @Override + public void onCompleted() { + responseObserver.onCompleted(); + if (Objects.nonNull(buf)) { + buf.flip(); + try { + parseAndStorageData(taskMetaData, buf); + } catch (IOException e) { + log.error("Failed to parse and store pprof data", e); + } + } + } + + private void parseAndStorageData(PprofCollectionMetaData taskMetaData, ByteBuffer buf) throws IOException { + PprofTask task = taskMetaData.getTask(); + if (task == null) { + log.error("Pprof instanceId:{} has not been assigned a task but still uploaded data", taskMetaData.getInstanceId()); + return; + } + recordPprofTaskLog(task, taskMetaData.getInstanceId(), PprofTaskLogOperationType.EXECUTION_FINISHED); + parsePprofAndStorage(taskMetaData, buf); + } + + public void parsePprofAndStorage(PprofCollectionMetaData taskMetaData, ByteBuffer buf) throws IOException { + log.info("Parsing pprof file for service: {}, instance: {}", + taskMetaData.getServiceId(), taskMetaData.getInstanceId()); + PprofTask task = taskMetaData.getTask(); + FrameTree tree = PprofParser.dumpTree(buf); + PprofProfilingData data = new PprofProfilingData(); + data.setEventType(PprofEventType.valueOfString(task.getEvents().name())); + data.setFrameTree(tree); + data.setTaskId(task.getId()); + data.setInstanceId(taskMetaData.getInstanceId()); + data.setUploadTime(taskMetaData.getUploadTime()); + log.info("data eventType: {}", data.getEventType()); + log.info("data frameTree: {}", tree); + log.info("data taskId: {}", task.getId()); + log.info("data instanceId: {}", taskMetaData.getInstanceId()); + log.info("data uploadTime: {}", taskMetaData.getUploadTime()); + sourceReceiver.receive(data); + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofCollectionMetaData.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofCollectionMetaData.java new file mode 100644 index 000000000000..b0986361d97c --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofCollectionMetaData.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream; + +import org.apache.skywalking.oap.server.core.query.type.PprofTask; + +import org.apache.skywalking.apm.network.pprof.v10.PprofProfilingStatus; +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class PprofCollectionMetaData { + private PprofTask task; + private String serviceId; + private String instanceId; + private int contentSize; + private PprofProfilingStatus type; + private long uploadTime; +} diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java new file mode 100644 index 000000000000..a5cd831e9685 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream; + +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; +import lombok.SneakyThrows; +import org.apache.skywalking.apm.network.pprof.v10.PprofData; +import org.apache.skywalking.apm.network.pprof.v10.PprofCollectionResponse; +import org.apache.skywalking.apm.network.pprof.v10.PprofProfilingStatus; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.source.PprofProfilingData; +import org.apache.skywalking.oap.server.library.pprof.parser.PprofParser; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import static org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler.recordPprofTaskLog; +import static org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler.parseMetaData; + +@Slf4j +public class PprofFileCollectionObserver implements StreamObserver { + private final IPprofTaskQueryDAO taskDAO; + private final StreamObserver responseObserver; + private final SourceReceiver sourceReceiver; + private final int pprofMaxSize; + private PprofCollectionMetaData taskMetaData; + private Path tempFile; + private FileOutputStream fileOutputStream; + + public PprofFileCollectionObserver(IPprofTaskQueryDAO taskDAO, + StreamObserver responseObserver, + SourceReceiver sourceReceiver, int pprofMaxSize) { + this.taskDAO = taskDAO; + this.responseObserver = responseObserver; + this.sourceReceiver = sourceReceiver; + this.pprofMaxSize = pprofMaxSize; + } + + @SneakyThrows + @Override + public void onNext(PprofData pprofData) { + if (Objects.isNull(taskMetaData) && pprofData.hasMetadata()) { + taskMetaData = parseMetaData(pprofData.getMetadata(), taskDAO); + + if (PprofProfilingStatus.PPROF_PROFILING_SUCCESS.equals(taskMetaData.getType())) { + int size = taskMetaData.getContentSize(); + if (pprofMaxSize >= size) { + // Create temporary file for pprof data + tempFile = Files.createTempFile(taskMetaData.getTask().getId() + taskMetaData.getInstanceId() + System.currentTimeMillis(), ".pprof"); + fileOutputStream = new FileOutputStream(tempFile.toFile()); + + // Send success response to allow client to continue uploading + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus(PprofProfilingStatus.PPROF_PROFILING_SUCCESS) + .build()); + } else { + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus(PprofProfilingStatus.PPROF_TERMINATED_BY_OVERSIZE) + .build()); + recordPprofTaskLog(taskMetaData.getTask(), taskMetaData.getInstanceId(), PprofTaskLogOperationType.PPROF_UPLOAD_FILE_TOO_LARGE_ERROR); + } + } else { + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus(PprofProfilingStatus.PPROF_EXECUTION_TASK_ERROR) + .build()); + recordPprofTaskLog(taskMetaData.getTask(), taskMetaData.getInstanceId(), PprofTaskLogOperationType.EXECUTION_TASK_ERROR); + } + } else if (pprofData.hasContent()) { + if (fileOutputStream != null) { + fileOutputStream.write(pprofData.getContent().toByteArray()); + + if (log.isDebugEnabled()) { + log.debug("Received {} bytes of pprof data", pprofData.getContent().size()); + } + } + } + } + + @Override + public void onError(Throwable throwable) { + Status status = Status.fromThrowable(throwable); + if (Status.CANCELLED.getCode() == status.getCode()) { + if (log.isDebugEnabled()) { + log.debug("Pprof data collection cancelled: {}", throwable.getMessage()); + } + } else { + log.error("Error in receiving pprof profiling data", throwable); + } + + // Clean up resources + closeFileStream(); + } + + @Override + @SneakyThrows + public void onCompleted() { + responseObserver.onCompleted(); + + if (Objects.nonNull(tempFile)) { + closeFileStream(); + parseAndStorageData(taskMetaData, tempFile.toAbsolutePath().toString()); + } + } + + private void closeFileStream() { + if (fileOutputStream != null) { + try { + fileOutputStream.close(); + fileOutputStream = null; + } catch (IOException e) { + log.error("Failed to close file output stream", e); + } + } + } + + @SneakyThrows + private void parseAndStorageData(PprofCollectionMetaData taskMetaData, String fileName) { + PprofTask task = taskMetaData.getTask(); + if (task == null) { + log.error("Pprof instanceId:{} has not been assigned a task but still uploaded data", taskMetaData.getInstanceId()); + return; + } + recordPprofTaskLog(task, taskMetaData.getInstanceId(), PprofTaskLogOperationType.EXECUTION_FINISHED); + parsePprofAndStorage(taskMetaData, fileName); + } + + public void parsePprofAndStorage(PprofCollectionMetaData taskMetaData, + String fileName) throws IOException { + log.info("Parsing pprof file for service: {}, instance: {}", + taskMetaData.getServiceId(), taskMetaData.getInstanceId()); + PprofTask task = taskMetaData.getTask(); + FrameTree tree = PprofParser.dumpTree(fileName); + PprofProfilingData data = new PprofProfilingData(); + data.setEventType(PprofEventType.valueOfString(task.getEvents().name())); + data.setFrameTree(tree); + data.setTaskId(task.getId()); + data.setInstanceId(taskMetaData.getInstanceId()); + data.setUploadTime(taskMetaData.getUploadTime()); + log.info("data eventType: {}", data.getEventType()); + log.info("data frameTree: {}", tree); + log.info("data taskId: {}", task.getId()); + log.info("data instanceId: {}", taskMetaData.getInstanceId()); + log.info("data uploadTime: {}", taskMetaData.getUploadTime()); + sourceReceiver.receive(data); + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine new file mode 100644 index 000000000000..3ef9c8a560ef --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.oap.server.receiver.pprof.module.PprofModule diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider new file mode 100644 index 000000000000..e4e90929ba62 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.oap.server.receiver.pprof.provider.PprofModuleProvider diff --git a/oap-server/server-receiver-plugin/skywalking-sharing-server-plugin/pom.xml b/oap-server/server-receiver-plugin/skywalking-sharing-server-plugin/pom.xml index 5cf914ba25c2..19b45143d0ea 100644 --- a/oap-server/server-receiver-plugin/skywalking-sharing-server-plugin/pom.xml +++ b/oap-server/server-receiver-plugin/skywalking-sharing-server-plugin/pom.xml @@ -27,4 +27,12 @@ skywalking-sharing-server-plugin jar + + + + org.apache.skywalking + library-server + ${project.version} + + \ No newline at end of file diff --git a/oap-server/server-starter/pom.xml b/oap-server/server-starter/pom.xml index d1971572aa90..7d44453471ae 100644 --- a/oap-server/server-starter/pom.xml +++ b/oap-server/server-starter/pom.xml @@ -176,6 +176,11 @@ skywalking-async-profiler-receiver-plugin ${project.version} + + org.apache.skywalking + skywalking-pprof-receiver-plugin + ${project.version} + org.apache.skywalking skywalking-telegraf-receiver-plugin diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index 281321788e3f..7f082649425d 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -183,6 +183,7 @@ storage: segmentQueryMaxSize: ${SW_STORAGE_ES_QUERY_SEGMENT_SIZE:200} profileTaskQueryMaxSize: ${SW_STORAGE_ES_QUERY_PROFILE_TASK_SIZE:200} asyncProfilerTaskQueryMaxSize: ${SW_STORAGE_ES_QUERY_ASYNC_PROFILER_TASK_SIZE:200} + pprofTaskQueryMaxSize: ${SW_STORAGE_ES_QUERY_PPROF_TASK_SIZE:200} profileDataQueryBatchSize: ${SW_STORAGE_ES_QUERY_PROFILE_DATA_BATCH_SIZE:100} oapAnalyzer: ${SW_STORAGE_ES_OAP_ANALYZER:"{\"analyzer\":{\"oap_analyzer\":{\"type\":\"stop\"}}}"} # the oap analyzer. oapLogAnalyzer: ${SW_STORAGE_ES_OAP_LOG_ANALYZER:"{\"analyzer\":{\"oap_log_analyzer\":{\"type\":\"standard\"}}}"} # the oap log analyzer. It could be customized by the ES analyzer configuration to support more language log formats, such as Chinese log, Japanese log and etc. @@ -296,6 +297,20 @@ receiver-async-profiler: # It is recommended to use physical file mode when volume mounting is used or the tmp directory has sufficient storage. memoryParserEnabled: ${SW_RECEIVER_ASYNC_PROFILER_MEMORY_PARSER_ENABLED:true} +receiver-pprof: + selector: ${SW_RECEIVER_PPROF:default} + default: + # Used to manage the maximum size of the pprof file that can be received, the unit is Byte, default is 30M + pprofMaxSize: ${SW_RECEIVER_PPROF_MAX_SIZE:31457280} + # Used to determine whether to receive pprof in memory file or physical file mode + # + # The memory file mode have fewer local file system limitations, so they are by default. But it costs more memory. + # + # The physical file mode will use less memory when parsing and is more friendly to parsing large files. + # However, if the storage of the tmp directory in the container is insufficient, the oap server instance may crash. + # It is recommended to use physical file mode when volume mounting is used or the tmp directory has sufficient storage. + memoryParserEnabled: ${SW_RECEIVER_PPROF_MEMORY_PARSER_ENABLED:true} + receiver-zabbix: selector: ${SW_RECEIVER_ZABBIX:-} default: diff --git a/oap-server/server-starter/src/main/resources/bydb.yml b/oap-server/server-starter/src/main/resources/bydb.yml index 1260ffa34d0f..496d6314c543 100644 --- a/oap-server/server-starter/src/main/resources/bydb.yml +++ b/oap-server/server-starter/src/main/resources/bydb.yml @@ -43,6 +43,7 @@ global: # The batch size for querying profile data. profileDataQueryBatchSize: ${SW_STORAGE_BANYANDB_QUERY_PROFILE_DATA_BATCH_SIZE:100} asyncProfilerTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_ASYNC_PROFILER_TASK_QUERY_MAX_SIZE:200} + pprofTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_PPROF_TASK_QUERY_MAX_SIZE:200} user: ${SW_STORAGE_BANYANDB_USER:""} password: ${SW_STORAGE_BANYANDB_PASSWORD:""} # If the BanyanDB server is configured with TLS, configure the TLS cert file path and enable TLS connection. diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java index 41d9bc2c412c..97da501178ce 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java @@ -100,7 +100,12 @@ public static class Global { * single request. */ private int asyncProfilerTaskQueryMaxSize; - + /** + * Max size of {@link org.apache.skywalking.oap.server.core.query.type.PprofTask} to be fetched in a + * single request. + */ + private int pprofTaskQueryMaxSize; + private int resultWindowMaxSize = 10000; private int metadataQueryMaxSize = 5000; private int segmentQueryMaxSize = 200; diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java index 5322d9305a07..47e81cfc1d37 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java @@ -40,6 +40,9 @@ import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IJFRDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.continuous.IContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingDataDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingScheduleDAO; @@ -81,6 +84,9 @@ import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBAlarmQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBAsyncProfilerTaskQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBPprofDataQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBPprofTaskQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBBrowserLogQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBEBPFProfilingDataDAO; @@ -203,6 +209,17 @@ IAsyncProfilerTaskLogQueryDAO.class, new BanyanDBAsyncProfilerTaskLogQueryDAO(cl this.config.getGlobal().getAsyncProfilerTaskQueryMaxSize() )); this.registerServiceImplementation(IJFRDataQueryDAO.class, new BanyanDBJFRDataQueryDAO(client)); + this.registerServiceImplementation( + IPprofTaskQueryDAO.class, new BanyanDBPprofTaskQueryDAO(client, + this.config.getGlobal().getPprofTaskQueryMaxSize() + )); + this.registerServiceImplementation( + IPprofTaskLogQueryDAO.class, new BanyanDBPprofTaskLogQueryDAO(client, + this.config.getGlobal().getPprofTaskQueryMaxSize() + )); + this.registerServiceImplementation( + IPprofDataQueryDAO.class, new BanyanDBPprofDataQueryDAO(client) + ); this.registerServiceImplementation( StorageTTLStatusQuery.class, new BanyanDBTTLStatusQuery(config) diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java new file mode 100644 index 000000000000..cdb573a9a8ef --- /dev/null +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream; + +import com.google.common.collect.ImmutableSet; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; +import java.util.Set; +import java.util.List; +import java.io.IOException; +import java.util.ArrayList; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient; +import org.apache.skywalking.banyandb.v1.client.StreamQuery; +import org.apache.skywalking.banyandb.v1.client.StreamQueryResponse; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.banyandb.v1.client.RowEntity; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBConverter; + +public class BanyanDBPprofDataQueryDAO extends AbstractBanyanDBDAO implements IPprofDataQueryDAO { + private static final Set TAGS = ImmutableSet.of( + PprofProfilingDataRecord.TASK_ID, + PprofProfilingDataRecord.INSTANCE_ID, + PprofProfilingDataRecord.UPLOAD_TIME, + PprofProfilingDataRecord.DATA_BINARY + ); + + public BanyanDBPprofDataQueryDAO(BanyanDBStorageClient client) { + super(client); + } + + @Override + public List getByTaskIdAndInstances(String taskId, List instanceIds) throws IOException { + if (StringUtil.isBlank(taskId)) { + return new ArrayList<>(); + } + StreamQueryResponse resp = query(false, PprofProfilingDataRecord.INDEX_NAME, TAGS, + new QueryBuilder() { + @Override + protected void apply(StreamQuery query) { + query.and(eq(PprofProfilingDataRecord.TASK_ID, taskId)); + if (CollectionUtils.isNotEmpty(instanceIds)) { + query.and(in(PprofProfilingDataRecord.INSTANCE_ID, instanceIds)); + } + } + }); + List records = new ArrayList<>(resp.size()); + for (final RowEntity entity : resp.getElements()) { + records.add(buildProfilingDataRecord(entity)); + } + + return records; + } + + private PprofProfilingDataRecord buildProfilingDataRecord(RowEntity entity) { + final PprofProfilingDataRecord.Builder builder = new PprofProfilingDataRecord.Builder(); + BanyanDBConverter.StorageToStream storageToStream = new BanyanDBConverter.StorageToStream(PprofProfilingDataRecord.INDEX_NAME, entity); + return builder.storage2Entity(storageToStream); + } +} diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskLogQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskLogQueryDAO.java new file mode 100644 index 000000000000..cfd9d4d42daf --- /dev/null +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskLogQueryDAO.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream; + +import com.google.common.collect.ImmutableSet; +import org.apache.skywalking.banyandb.v1.client.Element; +import org.apache.skywalking.banyandb.v1.client.StreamQuery; +import org.apache.skywalking.banyandb.v1.client.StreamQueryResponse; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskLogRecord; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +/** + * {@link PprofTaskLogRecord} is a stream + */ +public class BanyanDBPprofTaskLogQueryDAO extends AbstractBanyanDBDAO implements IPprofTaskLogQueryDAO { + private static final Set TAGS = ImmutableSet.of( + PprofTaskLogRecord.OPERATION_TIME, + PprofTaskLogRecord.INSTANCE_ID, + PprofTaskLogRecord.TASK_ID, + PprofTaskLogRecord.OPERATION_TYPE + ); + + private final int queryMaxSize; + + public BanyanDBPprofTaskLogQueryDAO(BanyanDBStorageClient client, int taskQueryMaxSize) { + super(client); + // query log size use pprof task query max size * per log count + this.queryMaxSize = taskQueryMaxSize * 50; + } + + @Override + public List getTaskLogList() throws IOException { + StreamQueryResponse resp = query(false, PprofTaskLogRecord.INDEX_NAME, TAGS, + new QueryBuilder() { + @Override + public void apply(StreamQuery query) { + query.setLimit(BanyanDBPprofTaskLogQueryDAO.this.queryMaxSize); + } + }); + + final LinkedList tasks = new LinkedList<>(); + for (final Element element : resp.getElements()) { + tasks.add(buildPprofTaskLog(element)); + } + return tasks; + } + + private PprofTaskLog buildPprofTaskLog(Element data) { + int operationTypeInt = ((Number) data.getTagValue(PprofTaskLogRecord.OPERATION_TYPE)).intValue(); + PprofTaskLogOperationType operationType = PprofTaskLogOperationType.parse(operationTypeInt); + return PprofTaskLog.builder() + .id(data.getTagValue(PprofTaskLogRecord.TASK_ID)) + .instanceId(data.getTagValue(PprofTaskLogRecord.INSTANCE_ID)) + .operationType(operationType) + .operationTime(((Number) data.getTagValue(PprofTaskLogRecord.OPERATION_TIME)).longValue()) + .build(); + } +} diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java new file mode 100644 index 000000000000..58488382f6f4 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream; + +import com.google.common.collect.ImmutableSet; +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import org.apache.skywalking.banyandb.v1.client.AbstractQuery; +import org.apache.skywalking.banyandb.v1.client.RowEntity; +import org.apache.skywalking.banyandb.v1.client.StreamQuery; +import org.apache.skywalking.banyandb.v1.client.StreamQueryResponse; +import org.apache.skywalking.banyandb.v1.client.TimestampRange; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient; +import lombok.extern.slf4j.Slf4j; +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +@Slf4j +public class BanyanDBPprofTaskQueryDAO extends AbstractBanyanDBDAO implements IPprofTaskQueryDAO { + private static final Gson GSON = new Gson(); + private static final Set TAGS = ImmutableSet.of( + PprofTaskRecord.SERVICE_ID, + PprofTaskRecord.SERVICE_INSTANCE_IDS, + PprofTaskRecord.TASK_ID, + PprofTaskRecord.CREATE_TIME, + PprofTaskRecord.EVENT_TYPES, + PprofTaskRecord.DURATION, + PprofTaskRecord.DUMP_PERIOD + ); + + private final int queryMaxSize; + + public BanyanDBPprofTaskQueryDAO(BanyanDBStorageClient client, int queryMaxSize) { + super(client); + this.queryMaxSize = queryMaxSize; + } + + @Override + public List getTaskList(String serviceId, Long startTimeBucket, Long endTimeBucket, Integer limit) throws IOException { + long startTS = LOWER_BOUND_TIME; + long endTS = UPPER_BOUND_TIME; + if (startTimeBucket != null) { + startTS = TimeBucket.getTimestamp(startTimeBucket); + } + if (endTimeBucket != null) { + endTS = TimeBucket.getTimestamp(endTimeBucket); + } + StreamQueryResponse resp = query(false, PprofTaskRecord.INDEX_NAME, TAGS, new TimestampRange(startTS, endTS), + new QueryBuilder() { + @Override + protected void apply(StreamQuery query) { + if (StringUtil.isNotEmpty(serviceId)) { + query.and(eq(PprofTaskRecord.SERVICE_ID, serviceId)); + } + + if (limit != null) { + query.setLimit(limit); + } else { + query.setLimit(BanyanDBPprofTaskQueryDAO.this.queryMaxSize); + } + query.setOrderBy(new AbstractQuery.OrderBy(AbstractQuery.Sort.DESC)); + } + }); + + List tasks = new ArrayList<>(resp.size()); + for (final RowEntity entity : resp.getElements()) { + tasks.add(buildPprofTask(entity)); + } + return tasks; + } + + @Override + public PprofTask getById(String id) throws IOException { + StreamQueryResponse resp = query(false, PprofTaskRecord.INDEX_NAME, TAGS, + new QueryBuilder() { + @Override + protected void apply(StreamQuery query) { + if (StringUtil.isNotEmpty(id)) { + query.and(eq(PprofTaskRecord.TASK_ID, id)); + } + query.setLimit(1); + } + }); + + if (resp.size() == 0) { + return null; + } + + return buildPprofTask(resp.getElements().get(0)); + } + + private PprofTask buildPprofTask(RowEntity data) { + Type listType = new TypeToken>() { + }.getType(); + + String serviceInstanceIds = data.getTagValue(PprofTaskRecord.SERVICE_INSTANCE_IDS); + List serviceInstanceIdList = GSON.fromJson(serviceInstanceIds, listType); + + // Convert string events to PprofEventType enum + String eventsStr = data.getTagValue(PprofTaskRecord.EVENT_TYPES); + PprofEventType eventType = null; + if (StringUtil.isNotEmpty(eventsStr)) { + try { + eventType = PprofEventType.valueOfString(eventsStr); + } catch (Exception e) { + // Default to CPU if conversion fails + eventType = PprofEventType.CPU; + log.warn("Failed to parse pprof event type: {}, using CPU as default", eventsStr, e); + } + } + + return PprofTask.builder() + .id(data.getTagValue(PprofTaskRecord.TASK_ID)) + .serviceId(data.getTagValue(PprofTaskRecord.SERVICE_ID)) + .serviceInstanceIds(serviceInstanceIdList) + .createTime(((Number) data.getTagValue(PprofTaskRecord.CREATE_TIME)).longValue()) + .events(eventType) + .duration(((Number) data.getTagValue(PprofTaskRecord.DURATION)).intValue()) + .dumpPeriod(((Number) data.getTagValue(PprofTaskRecord.DUMP_PERIOD)).intValue()) + .build(); + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java index b275acd65014..d8dc848be5c3 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java @@ -167,4 +167,9 @@ public class StorageModuleElasticsearchConfig extends ModuleConfig { * in a single request. */ private int asyncProfilerTaskQueryMaxSize; + /** + * Max size of {@link org.apache.skywalking.oap.server.core.query.type.PprofTask} to be fetched + * in a single request. + */ + private int pprofTaskQueryMaxSize; } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java index c5dc8d018ef6..3cfd5e014c6b 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java @@ -39,6 +39,9 @@ import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IJFRDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.continuous.IContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingDataDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingScheduleDAO; @@ -81,6 +84,7 @@ import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.AlarmQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.AsyncProfilerTaskLogQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.AsyncProfilerTaskQueryEsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.PprofTaskLogQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.BrowserLogQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ContinuousProfilingPolicyEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.EBPFProfilingDataEsDAO; @@ -89,12 +93,14 @@ import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ESEventQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.HierarchyQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.JFRDataQueryEsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.PprofDataQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.LogQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.MetadataQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.MetricsQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ProfileTaskLogEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ProfileTaskQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ProfileThreadSnapshotQueryEsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.PprofTaskQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.RecordsQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ServiceLabelEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.SpanAttachedEventEsDAO; @@ -276,6 +282,18 @@ IProfileThreadSnapshotQueryDAO.class, new ProfileThreadSnapshotQueryEsDAO(elasti IJFRDataQueryDAO.class, new JFRDataQueryEsDAO(elasticSearchClient) ); + this.registerServiceImplementation( + IPprofTaskQueryDAO.class, + new PprofTaskQueryEsDAO(elasticSearchClient, config.getPprofTaskQueryMaxSize()) + ); + this.registerServiceImplementation( + IPprofTaskLogQueryDAO.class, + new PprofTaskLogQueryEsDAO(elasticSearchClient, config.getPprofTaskQueryMaxSize()) + ); + this.registerServiceImplementation( + IPprofDataQueryDAO.class, + new PprofDataQueryEsDAO(elasticSearchClient) + ); this.registerServiceImplementation( StorageTTLStatusQuery.class, new DefaultStorageTTLStatusQuery() diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java new file mode 100644 index 000000000000..2dc2b03d7be1 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; +import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Query; +import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Search; +import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; +import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; +import com.google.common.collect.Lists; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.ElasticSearchConverter; +import java.util.Map; +import java.util.ArrayList; +import java.util.List; + +public class PprofDataQueryEsDAO extends EsDAO implements IPprofDataQueryDAO { + public PprofDataQueryEsDAO(ElasticSearchClient client) { + super(client); + } + + @Override + public List getByTaskIdAndInstances(String taskId, List instanceIds) { + if (StringUtil.isBlank(taskId) || CollectionUtils.isEmpty(instanceIds)) { + return new ArrayList<>(); + } + final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofProfilingDataRecord.INDEX_NAME); + final BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isMergedTable(PprofProfilingDataRecord.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofProfilingDataRecord.INDEX_NAME)); + } + query.must(Query.term(PprofProfilingDataRecord.TASK_ID, taskId)); + query.must(Query.terms(PprofProfilingDataRecord.INSTANCE_ID, instanceIds)); + final SearchBuilder search = Search.builder().query(query); + final SearchResponse response = getClient().search(index, search.build()); + List dataRecords = Lists.newArrayList(); + for (SearchHit searchHit : response.getHits().getHits()) { + dataRecords.add(parseData(searchHit)); + } + return dataRecords; + } + + private PprofProfilingDataRecord parseData(SearchHit data) { + final Map sourceAsMap = data.getSource(); + final PprofProfilingDataRecord.Builder builder = new PprofProfilingDataRecord.Builder(); + return builder.storage2Entity(new ElasticSearchConverter.ToEntity(PprofProfilingDataRecord.INDEX_NAME, sourceAsMap)); + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java new file mode 100644 index 000000000000..87f20d4498bf --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Query; +import org.apache.skywalking.library.elasticsearch.requests.search.Search; +import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Sort; +import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; +import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskLogRecord; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; + +public class PprofTaskLogQueryEsDAO extends EsDAO implements IPprofTaskLogQueryDAO { + + private final int queryMaxSize; + + public PprofTaskLogQueryEsDAO(ElasticSearchClient client, int profileTaskQueryMaxSize) { + super(client); + // query log size use pprof task query max size * per log count + this.queryMaxSize = profileTaskQueryMaxSize * 50; + } + + @Override + public List getTaskLogList() throws IOException { + final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofTaskLogRecord.INDEX_NAME); + final BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isMergedTable(PprofTaskLogRecord.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofTaskLogRecord.INDEX_NAME)); + } + + final SearchBuilder search = + Search.builder().query(query) + .sort(PprofTaskLogRecord.OPERATION_TIME, Sort.Order.DESC) + .size(queryMaxSize); + final SearchResponse response = getClient().search(index, search.build()); + + List tasks = new LinkedList<>(); + for (SearchHit searchHit : response.getHits().getHits()) { + tasks.add(buildPprofTaskLog(searchHit)); + } + return tasks; + } + + private PprofTaskLog buildPprofTaskLog(SearchHit data) { + Map source = data.getSource(); + int operationTypeInt = ((Number) source.get(PprofTaskLogRecord.OPERATION_TYPE)).intValue(); + PprofTaskLogOperationType operationType = PprofTaskLogOperationType.parse(operationTypeInt); + return PprofTaskLog.builder() + .id((String) source.get(PprofTaskLogRecord.TASK_ID)) + .instanceId((String) source.get(PprofTaskLogRecord.INSTANCE_ID)) + .operationType(operationType) + .operationTime(((Number) source.get(PprofTaskLogRecord.OPERATION_TIME)).longValue()) + .build(); + } +} + + + + + + + + + + + + + + + + + + + + + + + diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java new file mode 100644 index 000000000000..555e39cabe4e --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import com.google.gson.Gson; +import java.util.Objects; +import com.google.gson.reflect.TypeToken; +import java.lang.reflect.Type; + +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Query; +import org.apache.skywalking.library.elasticsearch.requests.search.Search; +import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Sort; +import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; +import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; + +public class PprofTaskQueryEsDAO extends EsDAO implements IPprofTaskQueryDAO { + private static final Gson GSON = new Gson(); + + private final int queryMaxSize; + + public PprofTaskQueryEsDAO(ElasticSearchClient client, int queryMaxSize) { + super(client); + this.queryMaxSize = queryMaxSize; + } + + @Override + public List getTaskList(String serviceId, Long startTimeBucket, Long endTimeBucket, Integer limit) throws IOException { + String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofTaskRecord.INDEX_NAME); + BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isMergedTable(PprofTaskRecord.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofTaskRecord.INDEX_NAME)); + } + + if (StringUtil.isNotEmpty(serviceId)) { + query.must(Query.term(PprofTaskRecord.SERVICE_ID, serviceId)); + } + + if (startTimeBucket != null) { + query.must(Query.range(PprofTaskRecord.TIME_BUCKET).gte(startTimeBucket)); + } + + if (endTimeBucket != null) { + query.must(Query.range(PprofTaskRecord.TIME_BUCKET).lte(endTimeBucket)); + } + SearchBuilder search = Search.builder().query(query); + search.size(Objects.requireNonNullElse(limit, queryMaxSize)); + + search.sort(PprofTaskRecord.CREATE_TIME, Sort.Order.DESC); + + final SearchResponse response = getClient().search(index, search.build()); + + List tasks = new LinkedList<>(); + for (SearchHit hit : response.getHits().getHits()) { + tasks.add(parseTask(hit)); + } + return tasks; + } + + @Override + public PprofTask getById(String id) { + if (StringUtil.isEmpty(id)) { + return null; + } + final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofTaskRecord.INDEX_NAME); + final BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isMergedTable(PprofTaskRecord.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofTaskRecord.INDEX_NAME)); + } + query.must(Query.term(PprofTaskRecord.TASK_ID, id)); + + final SearchBuilder search = Search.builder().query(query).size(1); + final SearchResponse response = getClient().search(index, search.build()); + + if (!response.getHits().getHits().isEmpty()) { + return parseTask(response.getHits().getHits().iterator().next()); + } + return null; + } + + private PprofTask parseTask(SearchHit data) { + Map source = data.getSource(); + Type listType = new TypeToken>() { + }.getType(); + + String serviceInstanceIds = (String) source.get(PprofTaskRecord.SERVICE_INSTANCE_IDS); + + List instanceIdList = GSON.fromJson(serviceInstanceIds, listType); + return PprofTask.builder() + .id((String) source.get(PprofTaskRecord.TASK_ID)) + .serviceId((String) source.get(PprofTaskRecord.SERVICE_ID)) + .serviceInstanceIds(instanceIdList) + .createTime(((Number) source.get(PprofTaskRecord.CREATE_TIME)).longValue()) + .events(PprofEventType.valueOfString((String) source.get(PprofTaskRecord.EVENT_TYPES))) + .duration(((Number) source.get(PprofTaskRecord.DURATION)).intValue()) + .dumpPeriod(((Number) source.get(PprofTaskRecord.DUMP_PERIOD)).intValue()) + .build(); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml index 3285d110a4e1..025c702d12a8 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml @@ -48,6 +48,12 @@ org.postgresql postgresql + + mysql + mysql-connector-java + 8.0.13 + provided + org.testcontainers @@ -55,4 +61,44 @@ test + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + default-test + test + + test + + + + **/PreventRedistributionMySQLDriverTest.java + + + + + + test-without-mysql + test + + test + + + + **/PreventRedistributionMySQLDriverTest.java + + + mysql:mysql-connector-java + + + + + + + diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCStorageProvider.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCStorageProvider.java index 2fd5cbcb4927..7891c1c7b122 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCStorageProvider.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCStorageProvider.java @@ -33,6 +33,9 @@ import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IJFRDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.continuous.IContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingDataDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingScheduleDAO; @@ -66,6 +69,9 @@ import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCAlarmQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCAsyncProfilerTaskQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCPprofDataQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCPprofTaskQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCBatchDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCBrowserLogQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCContinuousProfilingPolicyDAO; @@ -255,6 +261,18 @@ public void prepare() throws ServiceNotProvidedException, ModuleStartException { IJFRDataQueryDAO.class, new JDBCJFRDataQueryDAO(jdbcClient, tableHelper) ); + this.registerServiceImplementation( + IPprofTaskQueryDAO.class, + new JDBCPprofTaskQueryDAO(jdbcClient, tableHelper) + ); + this.registerServiceImplementation( + IPprofTaskLogQueryDAO.class, + new JDBCPprofTaskLogQueryDAO(jdbcClient, tableHelper) + ); + this.registerServiceImplementation( + IPprofDataQueryDAO.class, + new JDBCPprofDataQueryDAO(jdbcClient, tableHelper) + ); this.registerServiceImplementation( StorageTTLStatusQuery.class, new DefaultStorageTTLStatusQuery() diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java new file mode 100644 index 000000000000..86899febe75b --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + + import lombok.RequiredArgsConstructor; + import lombok.SneakyThrows; + import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; + import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; + import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCClient; + import org.apache.skywalking.oap.server.library.util.CollectionUtils; + import org.apache.skywalking.oap.server.library.util.StringUtil; + import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCEntityConverters; + import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCTableInstaller; + import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.TableHelper; + + import java.io.IOException; + import java.sql.ResultSet; + import java.util.ArrayList; + import java.util.Base64; + import java.util.List; + + @RequiredArgsConstructor + public class JDBCPprofDataQueryDAO implements IPprofDataQueryDAO { + private final JDBCClient jdbcClient; + private final TableHelper tableHelper; + + @Override + @SneakyThrows + public List getByTaskIdAndInstances(String taskId, List instanceIds) throws IOException { + if (StringUtil.isBlank(taskId)) { + return new ArrayList<>(); + } + List tables = tableHelper.getTablesWithinTTL(PprofProfilingDataRecord.INDEX_NAME); + List results = new ArrayList<>(); + for (final var table : tables) { + List condition = new ArrayList<>(4); + StringBuilder sql = new StringBuilder() + .append("select * from ").append(table) + .append(" where ").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); + condition.add(PprofProfilingDataRecord.INDEX_NAME); + + if (CollectionUtils.isNotEmpty(instanceIds)) { + sql.append(" and ").append(PprofProfilingDataRecord.INSTANCE_ID).append(" in (?) "); + String joinedInstanceIds = String.join(",", instanceIds); + condition.add(joinedInstanceIds); + } + + results.addAll( + jdbcClient.executeQuery( + sql.toString(), + resultSet -> { + final var result = new ArrayList(); + while (resultSet.next()) { + result.add(parseData(resultSet)); + } + return result; + }, + condition.toArray(new Object[0])) + ); + } + return results; + } + + private PprofProfilingDataRecord parseData(ResultSet data) { + final PprofProfilingDataRecord.Builder builder = new PprofProfilingDataRecord.Builder(); + PprofProfilingDataRecord pprofProfilingDataRecord = builder.storage2Entity(JDBCEntityConverters.toEntity(data)); + byte[] dataBinary = pprofProfilingDataRecord.getDataBinary(); + if (dataBinary != null) { + byte[] decodeResult = Base64.getDecoder().decode(dataBinary); + pprofProfilingDataRecord.setDataBinary(decodeResult); + } + return pprofProfilingDataRecord; + } + } + \ No newline at end of file diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskLogQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskLogQueryDAO.java new file mode 100644 index 000000000000..ea6da7dd81d1 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskLogQueryDAO.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskLogRecord; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCClient; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCTableInstaller; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.SQLAndParameters; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.TableHelper; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +@RequiredArgsConstructor +public class JDBCPprofTaskLogQueryDAO implements IPprofTaskLogQueryDAO { + private final JDBCClient jdbcClient; + private final TableHelper tableHelper; + + @Override + @SneakyThrows + public List getTaskLogList() { + List tables = tableHelper.getTablesWithinTTL(PprofTaskLogRecord.INDEX_NAME); + final List results = new ArrayList(); + for (String table : tables) { + SQLAndParameters sqlAndParameters = buildSQL(table); + List logs = jdbcClient.executeQuery( + sqlAndParameters.sql(), + resultSet -> { + final List tasks = new ArrayList<>(); + while (resultSet.next()) { + tasks.add(parseLog(resultSet)); + } + return tasks; + }, + sqlAndParameters.parameters()); + results.addAll(logs); + } + return results; + } + + private SQLAndParameters buildSQL(String table) { + StringBuilder sql = new StringBuilder(); + List parameters = new ArrayList<>(2); + sql.append("select * from ").append(table) + .append(" where ").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); + parameters.add(PprofTaskLogRecord.INDEX_NAME); + sql.append(" order by ").append(PprofTaskLogRecord.OPERATION_TIME).append(" desc"); + return new SQLAndParameters(sql.toString(), parameters); + } + + private PprofTaskLog parseLog(ResultSet data) throws SQLException { + return PprofTaskLog.builder() + .id(data.getString(PprofTaskLogRecord.TASK_ID)) + .instanceId(data.getString(PprofTaskLogRecord.INSTANCE_ID)) + .operationType(PprofTaskLogOperationType.parse(data.getInt(PprofTaskLogRecord.OPERATION_TYPE))) + .operationTime(data.getLong(PprofTaskLogRecord.OPERATION_TIME)) + .build(); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskQueryDAO.java new file mode 100644 index 000000000000..645accbb4d42 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskQueryDAO.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCClient; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCTableInstaller; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.TableHelper; + +import java.io.IOException; +import java.lang.reflect.Type; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import static java.util.stream.Collectors.toList; + +@RequiredArgsConstructor +public class JDBCPprofTaskQueryDAO implements IPprofTaskQueryDAO { + private static final Gson GSON = new Gson(); + + private final JDBCClient jdbcClient; + private final TableHelper tableHelper; + + @Override + @SneakyThrows + public List getTaskList(String serviceId, Long startTimeBucket, Long endTimeBucket, Integer limit) throws IOException { + final var results = new ArrayList(); + final var tables = startTimeBucket == null || endTimeBucket == null ? + tableHelper.getTablesWithinTTL(PprofTaskRecord.INDEX_NAME) : + tableHelper.getTablesForRead(PprofTaskRecord.INDEX_NAME, startTimeBucket, endTimeBucket); + for (final var table : tables) { + List condition = new ArrayList<>(4); + StringBuilder sql = new StringBuilder() + .append("select * from ").append(table) + .append(" where ").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); + condition.add(PprofTaskRecord.INDEX_NAME); + + if (StringUtil.isNotEmpty(serviceId)) { + sql.append(" and ").append(PprofTaskRecord.SERVICE_ID).append("=? "); + condition.add(serviceId); + } + + if (startTimeBucket != null) { + sql.append(" and ").append(PprofTaskRecord.TIME_BUCKET).append(" >= ? "); + condition.add(startTimeBucket); + } + + if (endTimeBucket != null) { + sql.append(" and ").append(PprofTaskRecord.TIME_BUCKET).append(" <= ? "); + condition.add(endTimeBucket); + } + + sql.append(" ORDER BY ").append(PprofTaskRecord.CREATE_TIME).append(" DESC "); + + if (limit != null) { + sql.append(" LIMIT ").append(limit); + } + + results.addAll( + jdbcClient.executeQuery( + sql.toString(), + resultSet -> { + final var tasks = new ArrayList(); + while (resultSet.next()) { + tasks.add(buildPprofTask(resultSet)); + } + return tasks; + }, + condition.toArray(new Object[0])) + ); + } + return limit == null ? + results : + results + .stream() + .limit(limit) + .collect(toList()); + } + + @Override + @SneakyThrows + public PprofTask getById(String id) throws IOException { + final var tables = tableHelper.getTablesWithinTTL(PprofTaskRecord.INDEX_NAME); + for (String table : tables) { + final StringBuilder sql = new StringBuilder(); + final List condition = new ArrayList<>(1); + sql.append("select * from ").append(table) + .append(" where ") + .append(JDBCTableInstaller.TABLE_COLUMN).append(" = ? ") + .append(" and ") + .append(PprofTaskRecord.TASK_ID + "=? LIMIT 1"); + condition.add(PprofTaskRecord.INDEX_NAME); + condition.add(id); + + final var r = jdbcClient.executeQuery( + sql.toString(), + resultSet -> { + if (resultSet.next()) { + return buildPprofTask(resultSet); + } + return null; + }, + condition.toArray(new Object[0])); + if (r != null) { + return r; + } + } + return null; + } + + private PprofTask buildPprofTask(ResultSet data) throws SQLException { + Type listType = new TypeToken>() { + }.getType(); + String events = data.getString(PprofTaskRecord.EVENT_TYPES); + String serviceInstanceIds = data.getString(PprofTaskRecord.SERVICE_INSTANCE_IDS); + List serviceInstanceIdList = GSON.fromJson(serviceInstanceIds, listType); + return PprofTask.builder() + .id(data.getString(PprofTaskRecord.TASK_ID)) + .serviceId(data.getString(PprofTaskRecord.SERVICE_ID)) + .serviceInstanceIds(serviceInstanceIdList) + .createTime(data.getLong(PprofTaskRecord.CREATE_TIME)) + .duration(data.getInt(PprofTaskRecord.DURATION)) + .events(PprofEventType.valueOfString(events)) + .dumpPeriod(data.getInt(PprofTaskRecord.DUMP_PERIOD)) + .build(); + } + +} diff --git a/skywalking-ui b/skywalking-ui index 3cefbf1bd5d5..ad4b0639cd9b 160000 --- a/skywalking-ui +++ b/skywalking-ui @@ -1 +1 @@ -Subproject commit 3cefbf1bd5d588355829cbe9dd23cf2253af547e +Subproject commit ad4b0639cd9bef691a12f894d95d385fb44e889c diff --git a/test/e2e-v2/cases/go/service/go.mod b/test/e2e-v2/cases/go/service/go.mod index 2c412645299f..6b5f043ed2a8 100644 --- a/test/e2e-v2/cases/go/service/go.mod +++ b/test/e2e-v2/cases/go/service/go.mod @@ -20,7 +20,7 @@ module sw-e2e go 1.19 require ( - github.com/apache/skywalking-go v0.5.1-0.20250301084827-154de50628e8 + github.com/apache/skywalking-go v0.6.1-0.20250924145416-aa948377ecdb github.com/apache/skywalking-go/toolkit v0.5.1-0.20250301084827-154de50628e8 github.com/gin-gonic/gin v1.10.0 ) @@ -40,13 +40,16 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/uuid v1.3.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.15.9 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/segmentio/kafka-go v0.4.43 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect golang.org/x/arch v0.8.0 // indirect diff --git a/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml new file mode 100644 index 000000000000..eba20b929672 --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '3.8' + +services: + banyandb: + extends: + file: ../../../../script/docker-compose/base-compose.yml + service: banyandb + networks: + - e2e + + go-service: + build: + context: ../../../go/service + dockerfile: Dockerfile + args: + - SW_AGENT_GO_COMMIT=${SW_AGENT_GO_COMMIT} + networks: + - e2e + expose: + - 8080 + environment: + SW_AGENT_NAME: go-service + SW_AGENT_INSTANCE_NAME: provider1 + SW_AGENT_REPORTER_GRPC_BACKEND_SERVICE: oap:11800 + UPSTREAM_URL: http://localhost:8080/ignored.html + privileged: true + stop_grace_period: 30s + depends_on: + oap: + condition: service_healthy + healthcheck: + test: ["CMD", "sh", "-c", "nc -z 127.0.0.1 8080 || (echo '[HEALTHCHECK] nc command failed' >&2; which nc >&2 || echo '[HEALTHCHECK] nc not found' >&2; netstat -tuln 2>&1 | grep 8080 || echo '[HEALTHCHECK] Port 8080 not listening' >&2; exit 1)"] + interval: 5s + timeout: 60s + retries: 120 + ports: + - 8080 + + oap: + extends: + file: ../../../../script/docker-compose/base-compose.yml + service: oap + environment: + SW_STORAGE: banyandb + depends_on: + banyandb: + condition: service_healthy + ports: + - 12800 + +networks: + e2e: diff --git a/test/e2e-v2/cases/profiling/pprof/banyandb/e2e.yaml b/test/e2e-v2/cases/profiling/pprof/banyandb/e2e.yaml new file mode 100644 index 000000000000..956e87c2820e --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/banyandb/e2e.yaml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 20m + init-system-environment: ../../../../script/env + steps: + - name: set PATH + command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + +verify: + retry: + count: 20 + interval: 10s + cases: + - includes: + - ../profiling-cases.yaml diff --git a/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml new file mode 100644 index 000000000000..c7d2e9519de4 --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '3.8' + +services: + es: + image: elastic/elasticsearch:7.15.0 + expose: + - 9200 + networks: + - e2e + environment: + - discovery.type=single-node + - cluster.routing.allocation.disk.threshold_enabled=false + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"] + interval: 5s + timeout: 60s + retries: 120 + + go-service: + build: + context: ../../../go/service + dockerfile: Dockerfile + args: + - SW_AGENT_GO_COMMIT=${SW_AGENT_GO_COMMIT} + networks: + - e2e + expose: + - 8080 + environment: + SW_AGENT_NAME: go-service + SW_AGENT_INSTANCE_NAME: provider1 + SW_AGENT_REPORTER_GRPC_BACKEND_SERVICE: oap:11800 + UPSTREAM_URL: http://localhost:8080/ignored.html + privileged: true + stop_grace_period: 30s + depends_on: + oap: + condition: service_healthy + healthcheck: + test: ["CMD", "sh", "-c", "nc -z 127.0.0.1 8080"] + interval: 5s + timeout: 60s + retries: 120 + ports: + - 8080 + + oap: + extends: + file: ../../../../script/docker-compose/base-compose.yml + service: oap + environment: + SW_STORAGE: elasticsearch + depends_on: + es: + condition: service_healthy + ports: + - 12800 + +networks: + e2e: diff --git a/test/e2e-v2/cases/profiling/pprof/es/e2e.yaml b/test/e2e-v2/cases/profiling/pprof/es/e2e.yaml new file mode 100644 index 000000000000..956e87c2820e --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/es/e2e.yaml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 20m + init-system-environment: ../../../../script/env + steps: + - name: set PATH + command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + +verify: + retry: + count: 20 + interval: 10s + cases: + - includes: + - ../profiling-cases.yaml diff --git a/test/e2e-v2/cases/profiling/pprof/expected/analysis.yml b/test/e2e-v2/cases/profiling/pprof/expected/analysis.yml new file mode 100644 index 000000000000..31d6b8056290 --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/expected/analysis.yml @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tree: + elements: + {{- contains .tree.elements }} + - id: {{ notEmpty .id }} + parentid: {{ notEmpty .parentid }} + codesignature: {{ notEmpty .codesignature }} + total: {{ gt .total -1 }} + self: {{ gt .self -1 }} + {{- end }} \ No newline at end of file diff --git a/test/e2e-v2/cases/profiling/pprof/expected/create.yml b/test/e2e-v2/cases/profiling/pprof/expected/create.yml new file mode 100644 index 000000000000..85bd8870434e --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/expected/create.yml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +errorreason: null +code: SUCCESS +id: {{ notEmpty .id }} \ No newline at end of file diff --git a/test/e2e-v2/cases/profiling/pprof/expected/list.yml b/test/e2e-v2/cases/profiling/pprof/expected/list.yml new file mode 100644 index 000000000000..823d93eebdbe --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/expected/list.yml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +errorreason: null +tasks: + {{- contains .tasks }} +- id: {{ notEmpty .id }} + serviceid: {{ b64enc "go-service" }}.1 + serviceinstanceids: + - {{ b64enc "go-service" }}.1_{{ b64enc "provider1" }} + createtime: {{ gt .createtime 0 }} + dumpperiod: {{ ge .dumpperiod 0 }} + events: HEAP + duration: {{ ge .duration 0 }} + {{- end }} \ No newline at end of file diff --git a/test/e2e-v2/cases/profiling/pprof/expected/progress.yml b/test/e2e-v2/cases/profiling/pprof/expected/progress.yml new file mode 100644 index 000000000000..9c285f08c530 --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/expected/progress.yml @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +logs: + {{- contains .logs }} +- id: {{ notEmpty .id}} + instanceid: {{ b64enc "go-service" }}.1_{{ b64enc "provider1" }} + instancename: {{ notEmpty .instancename}} + operationtype: {{ notEmpty .operationtype}} + operationtime: {{ ge .operationtime 0 }} + {{- end }} +errorinstanceids: [] +successinstanceids: + - {{ b64enc "go-service" }}.1_{{ b64enc "provider1" }} diff --git a/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml b/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml new file mode 100644 index 000000000000..630d58ce03c4 --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml @@ -0,0 +1,28 @@ +# Licensed to Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Apache Software Foundation (ASF) licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{{- contains . }} +- id: {{ notEmpty .id }} + name: {{ notEmpty .name }} + attributes: + {{- contains .attributes }} + - name: ipv4s + value: {{ notEmpty .value }} + {{- end}} + language: GO + instanceuuid: {{ notEmpty .instanceuuid }} +{{- end}} diff --git a/test/e2e-v2/cases/profiling/pprof/expected/service.yml b/test/e2e-v2/cases/profiling/pprof/expected/service.yml new file mode 100644 index 000000000000..5744ee6c74b4 --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/expected/service.yml @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- contains . }} +- id: {{ b64enc "go-service" }}.1 + name: go-service + group: "" + shortname: go-service + normal: true + layers: + - GENERAL + - SO11Y_GO_AGENT +{{- end }} diff --git a/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml new file mode 100644 index 000000000000..828163d3eeda --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + mysql: + image: mysql/mysql-server:8.0.13 + networks: + - e2e + expose: + - 3306 + environment: + - MYSQL_ROOT_PASSWORD=root@1234 + - MYSQL_DATABASE=swtest + - MYSQL_ROOT_HOST=% + healthcheck: + test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/3306" ] + interval: 5s + timeout: 60s + retries: 120 + go-service: + build: + context: ../../../go/service + dockerfile: Dockerfile + args: + - SW_AGENT_GO_COMMIT=${SW_AGENT_GO_COMMIT} + networks: + - e2e + expose: + - 8080 + environment: + SW_AGENT_NAME: go-service + SW_AGENT_INSTANCE_NAME: provider1 + SW_AGENT_REPORTER_GRPC_BACKEND_SERVICE: oap:11800 + UPSTREAM_URL: http://localhost:8080/ignored.html + privileged: true + stop_grace_period: 30s + depends_on: + oap: + condition: service_healthy + healthcheck: + test: ["CMD", "sh", "-c", "nc -z 127.0.0.1 8080"] + interval: 5s + timeout: 60s + retries: 120 + ports: + - 8080 + + oap: + extends: + file: ../../../../script/docker-compose/base-compose.yml + service: oap + environment: + SW_STORAGE: mysql + depends_on: + mysql: + condition: service_healthy + entrypoint: ['sh', '-c', '/download-mysql.sh /skywalking/oap-libs && /skywalking/docker-entrypoint.sh'] + ports: + - 12800 + +networks: + e2e: diff --git a/test/e2e-v2/cases/profiling/pprof/mysql/e2e.yaml b/test/e2e-v2/cases/profiling/pprof/mysql/e2e.yaml new file mode 100644 index 000000000000..956e87c2820e --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/mysql/e2e.yaml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 20m + init-system-environment: ../../../../script/env + steps: + - name: set PATH + command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + +verify: + retry: + count: 20 + interval: 10s + cases: + - includes: + - ../profiling-cases.yaml diff --git a/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml b/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml new file mode 100644 index 000000000000..8256a642241a --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +cases: + # service list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls + expected: expected/service.yml + # service instance list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=go-service + expected: expected/service-instance.yml + # create task + - query: | + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ + profiling pprof create --service-name=go-service \ + --dump-period=1 --events=HEAP \ + --instance-name-list=provider1 + expected: expected/create.yml + # list task + - query: | + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ + profiling pprof list --service-name=go-service + expected: expected/list.yml + # get task progress finished + - query: | + sleep 15 && swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ + profiling pprof progress --task-id=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling pprof list --service-name=go-service | yq e '.tasks[0].id') + expected: expected/progress.yml + # get task analysis + - query: | + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ + profiling pprof analysis --service-name=go-service \ + --instance-name-list=provider1 \ + --task-id=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling pprof list --service-name=go-service | yq e '.tasks[0].id') + expected: expected/analysis.yml + diff --git a/test/e2e-v2/script/env b/test/e2e-v2/script/env index 480d3081fef1..a04eee45ffd3 100644 --- a/test/e2e-v2/script/env +++ b/test/e2e-v2/script/env @@ -17,7 +17,7 @@ SW_AGENT_JAVA_COMMIT=f0245864e4388a388fe7445b56b6ce7cedc94aaf SW_AGENT_SATELLITE_COMMIT=ea27a3f4e126a24775fe12e2aa2695bcb23d99c3 SW_AGENT_NGINX_LUA_COMMIT=c3cee4841798a147d83b96a10914d4ac0e11d0aa SW_AGENT_NODEJS_COMMIT=4f9a91dad3dfd8cfe5ba8f7bd06b39e11eb5e65e -SW_AGENT_GO_COMMIT=154de50628e82e590941585411299459e352317d +SW_AGENT_GO_COMMIT=aa948377ecdb4724fad1cc365c13a1188021316f SW_AGENT_PYTHON_COMMIT=c76a6ec51a478ac91abb20ec8f22a99b8d4d6a58 SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449 SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016 @@ -27,4 +27,4 @@ SW_BANYANDB_COMMIT=7466afd2a2120f7fa311983be5a3077cb15d07e7 SW_AGENT_PHP_COMMIT=d1114e7be5d89881eec76e5b56e69ff844691e35 SW_PREDICTOR_COMMIT=54a0197654a3781a6f73ce35146c712af297c994 -SW_CTL_COMMIT=3b675df73824bbb80e6aabf6a95d110feb37b6b1 +SW_CTL_COMMIT=9a1beab08413ce415a00a8547a238a14691c5655