diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index 217ccbe9f8b0..0bd11fca045c 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -723,6 +723,13 @@ jobs: config: test/e2e-v2/cases/profiling/async-profiler/banyandb/e2e.yaml - name: Async Profiler MySQL config: test/e2e-v2/cases/profiling/async-profiler/mysql/e2e.yaml + + - name: Pprof ES + config: test/e2e-v2/cases/profiling/pprof/es/e2e.yaml + - name: Pprof BanyanDB + config: test/e2e-v2/cases/profiling/pprof/banyandb/e2e.yaml + - name: Pprof MySQL + config: test/e2e-v2/cases/profiling/pprof/mysql/e2e.yaml steps: - uses: actions/checkout@v4 with: diff --git a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/CommandDeserializer.java b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/CommandDeserializer.java index 6323e6906a12..4bd94b8bc0e0 100644 --- a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/CommandDeserializer.java +++ b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/CommandDeserializer.java @@ -29,6 +29,8 @@ public static BaseCommand deserialize(final Command command) { return ConfigurationDiscoveryCommand.DESERIALIZER.deserialize(command); } else if (AsyncProfilerTaskCommand.NAME.equals(commandName)) { return AsyncProfilerTaskCommand.DESERIALIZER.deserialize(command); + } else if (PprofTaskCommand.NAME.equals(commandName)) { + return PprofTaskCommand.DESERIALIZER.deserialize(command); } throw new UnsupportedCommandException(command); } diff --git a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java new file mode 100644 index 000000000000..30954b263217 --- /dev/null +++ b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.network.trace.component.command; + +import org.apache.skywalking.apm.network.common.v3.Command; +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import java.util.List; +import lombok.Getter; + +@Getter +public class PprofTaskCommand extends BaseCommand implements Serializable, Deserializable { + public static final Deserializable DESERIALIZER = new PprofTaskCommand("", "", "", 0, 0, 0); + public static final String NAME = "PprofTaskQuery"; + /** + * pprof taskId + */ + private String taskId; + /** + * event type of profiling (CPU/Heap/Block/Mutex/Goroutine/Threadcreate/Allocs) + */ + private String events; + /** + * run profiling for duration (minute) + */ + private long duration; + /** + * task create time + */ + private long createTime; + /** + * pprof dump period parameters. There are different dumpperiod configurations for different events. + * Here is a table of parameters. + * + *

for Block - sample an average of one blocking event per rate nanoseconds spent blocked. (default: 0)

+ *

for Mutex - sample an average of 1/rate events are reported. (default: 0)

+ * details @see pprof argument + */ + private int dumpPeriod; + + public PprofTaskCommand(String serialNumber, String taskId, String events, + long duration, long createTime, int dumpPeriod) { + super(NAME, serialNumber); + this.taskId = taskId; + this.duration = duration; + this.createTime = createTime; + this.dumpPeriod = dumpPeriod; + this.events = events; + } + + @Override + public PprofTaskCommand deserialize(Command command) { + final List argsList = command.getArgsList(); + String taskId = null; + String events = null; + long duration = 0; + long createTime = 0; + int dumpPeriod = 0; + String serialNumber = null; + for (final KeyStringValuePair pair : argsList) { + if ("SerialNumber".equals(pair.getKey())) { + serialNumber = pair.getValue(); + } else if ("TaskId".equals(pair.getKey())) { + taskId = pair.getValue(); + } else if ("Events".equals(pair.getKey())) { + events = pair.getValue(); + } else if ("Duration".equals(pair.getKey())) { + duration = Long.parseLong(pair.getValue()); + } else if ("CreateTime".equals(pair.getKey())) { + createTime = Long.parseLong(pair.getValue()); + } else if ("DumpPeriod".equals(pair.getKey())) { + dumpPeriod = Integer.parseInt(pair.getValue()); + } + } + return new PprofTaskCommand(serialNumber, taskId, events, duration, createTime, dumpPeriod); + } + + @Override + public Command.Builder serialize() { + final Command.Builder builder = commandBuilder(); + builder.addArgs(KeyStringValuePair.newBuilder().setKey("TaskId").setValue(taskId)) + .addArgs(KeyStringValuePair.newBuilder().setKey("Events").setValue(events)) + .addArgs(KeyStringValuePair.newBuilder().setKey("Duration").setValue(String.valueOf(duration))) + .addArgs(KeyStringValuePair.newBuilder().setKey("CreateTime").setValue(String.valueOf(createTime))) + .addArgs(KeyStringValuePair.newBuilder().setKey("DumpPeriod").setValue(String.valueOf(dumpPeriod))); + return builder; + } +} \ No newline at end of file diff --git a/docs/en/api/query-protocol.md b/docs/en/api/query-protocol.md index 1f97e908e376..90c9fb818844 100644 --- a/docs/en/api/query-protocol.md +++ b/docs/en/api/query-protocol.md @@ -215,7 +215,7 @@ extend type Query { Event query fetches the event list based on given sources and time range conditions. ### Profiling -SkyWalking offers two types of [profiling](../concepts-and-designs/profiling.md), in-process(tracing profiling and async-profiler) and out-process(ebpf profiling), allowing users to create tasks and check their execution status. +SkyWalking offers two types of [profiling](../concepts-and-designs/profiling.md), in-process(tracing profiling, async-profiler and pprof) and out-process(ebpf profiling), allowing users to create tasks and check their execution status. #### In-process profiling @@ -256,6 +256,25 @@ extend type Query { } ``` +##### pprof + +```graphql +extend type Mutation { + # Create a new pprof task + createPprofTask(pprofTaskCreationRequest: PprofTaskCreationRequest!): PprofTaskCreationResult! +} + +extend type Query { + # Query all task lists and sort them in descending order by create time + queryPprofTaskList(request: PprofTaskListRequest!): PprofTaskListResult! + # Query task progress, including task logs + queryPprofTaskProgress(taskId: String!): PprofTaskProgress! + # Query the flame graph produced by pprof + queryPprofAnalyze(request: PprofAnalyzationRequest!): PprofAnalyzation! +} +``` + + #### Out-process profiling ```graphql diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 24a3e5b4045b..e2bd5937b8bc 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -111,6 +111,8 @@ * Make MAL percentile align with OAL percentile calculation. * Update Grafana dashboards for OAP observability. * BanyanDB: fix query `getInstance` by instance ID. +* Support the go agent(0.7.0 release) bundled pprof profiling feature. + #### UI diff --git a/docs/en/concepts-and-designs/profiling.md b/docs/en/concepts-and-designs/profiling.md index 5c1ae91e9603..95992ef31b82 100644 --- a/docs/en/concepts-and-designs/profiling.md +++ b/docs/en/concepts-and-designs/profiling.md @@ -47,6 +47,20 @@ Async Profiler can trace the following kinds of events: Only Java agent support this. +### Go App Profiling + +Go App Profiling uses the [pprof](https://github.com/google/pprof) for sampling. + +pprof is a profiling tool by Google for visualizing and analyzing sampled performance data. +It reads samples in profile.proto format and generates text or graphical reports (via the dot visualization) to highlight performance hotspots. + +pprof supports profiling of: + +- CPU. +- Memory allocs / heap. +- Block / mutex. +- Gouroutine / threadcreate. + ## Out-of-process profiling Out-of-process profiling leverage [eBPF](https://ebpf.io/) technology with origins in the Linux kernel. diff --git a/docs/en/setup/backend/backend-go-app-profiling.md b/docs/en/setup/backend/backend-go-app-profiling.md new file mode 100644 index 000000000000..3cfbf0ba6d37 --- /dev/null +++ b/docs/en/setup/backend/backend-go-app-profiling.md @@ -0,0 +1,107 @@ +# Go App Profiling + +Go App Profiling uses the pprof for sampling + +pprof is bundled within the auto-instrument agent and corresponds to [In-Process Profiling](../../concepts-and-designs/profiling.md#in-process-profiling). + +It is delivered to the agent in the form of a task, allowing it to be enabled or disabled dynamically. +When service encounters performance issues (CPU usage, memory allocation, etc.), pprof task can be created. +When the agent receives a task, it enables pprof for sampling. +After sampling is completed, the sampling results are analyzed by requesting the server to render a flame graph for performance +analysis to determine the specific business code lines that cause performance problems. +Note, tracing profiling in the Go agent relies on the Go runtime’s global CPU sampling used by pprof. +Since only one CPU profiler can run at a time within the same instance, tracing and pprof CPU profiling cannot be enabled simultaneously. +If both are activated on the same instance, one task may fail to start. + +## Activate pprof in the OAP +OAP and the agent use a brand-new protocol to exchange pprof data, so it is necessary to start OAP with the following configuration: + +```yaml +receiver-pprof: + selector: ${SW_RECEIVER_PPROF:default} + default: + # Used to manage the maximum size of the pprof file that can be received, the unit is Byte, default is 30M + pprofMaxSize: ${SW_RECEIVER_PPROF_MAX_SIZE:31457280} + # Used to determine whether to receive pprof in memory file or physical file mode + # + # The memory file mode have fewer local file system limitations, so they are by default. But it costs more memory. + # + # The physical file mode will use less memory when parsing and is more friendly to parsing large files. + # However, if the storage of the tmp directory in the container is insufficient, the oap server instance may crash. + # It is recommended to use physical file mode when volume mounting is used or the tmp directory has sufficient storage. + memoryParserEnabled: ${SW_RECEIVER_PPROF_MEMORY_PARSER_ENABLED:true} +``` + +## pprof Task with Analysis + +To use the pprof feature, please follow these steps: + +1. **Create pprof task**: Use the UI or CLI tool to create a task. +2. **Wait agent collect data and upload**: Wait for pprof to collect pprof data and report. +3. **Query task progress**: Query the progress of tasks, including analyzing successful and failed instances and task logs. +4. **Analyze the data**: Analyze the pprof data to determine where performance bottlenecks exist in the service. + +### Create an pprof task + +Create an pprof task to notify some go-agent instances in the execution service to start pprof for data collection. + +When creating a task, the following configuration fields are required: + +1. **serviceId**: Define the service to execute the task. +2. **serviceInstanceIds**: Define which instances need to execute tasks. +3. **duration**: Define the duration of this task in minutes, required for CPU, BLOCK, MUTEX events. +4. **events**: Define which event types this task needs to collect. +5. **dumpPeriod**: Define the period of the pprof dump, required for BLOCK, MUTEX events. + +When the Agent receives a pprof task from OAP, it automatically generates a log to notify that the task has been acknowledged. The log contains the following field information: + +1. **Instance**: The name of the instance where the Agent is located. +2. **Type**: Supports "NOTIFIED" and "EXECUTION_FINISHED" and "PPROF_UPLOAD_FILE_TOO_LARGE_ERROR", "EXECUTION_TASK_ERROR", with the current log displaying "NOTIFIED". +3. **Time**: The time when the Agent received the task. + +### Wait the agent to collect data and upload + +At this point, pprof will trace the events you selected when you created the task: + +1. CPU: samples CPU usage over time to show which functions consume the most processing time. +2. ALLOC, HEAP: + - HEAP: a sampling of memory allocations of live objects. + - ALLOC: a sampling of all past memory allocations. +3. BLOCK, MUTEX: + - BLOCK: stack traces that led to blocking on synchronization primitives. + - MUTEX: stack traces of holders of contended mutexes. +4. GOROUTINE, THREADCREAT: + - GOROUTINE: stack traces of all current goroutines. + - THREADCREATE: stack traces that led to the creation of new OS threads. + +Finally, the agent will upload the pprof file produced by pprof to the oap server for online performance analysis. + +### Query the profiling task progresses + +Wait for pprof to complete data collection and upload successfully. +We can query the execution logs of the pprof task and the task status, which includes the following information: + +1. **successInstanceIds**: SuccessInstanceIds gives instances that have executed the task successfully. +2. **errorInstanceIds**: ErrorInstanceIds gives instances that failed to execute the task. +3. **logs**: All task execution logs of the current task. + 1. **id**: The task id. + 2. **instanceId**: InstanceId is the id of the instance which reported this task log. + 3. **instanceName**: InstanceName is the name of the instance which reported this task log. + 4. **operationType**: Contains "NOTIFIED" and "EXECUTION_FINISHED" and "PPROF_UPLOAD_FILE_TOO_LARGE_ERROR", "EXECUTION_TASK_ERROR". + 5. **operationTime**: operationTime is the time when the operation occurs. + +### Analyze the profiling data + +Once some agents completed the task, we can analyze the data through the following query: + +1. **taskId**: The task id. +2. **instanceIds**: InstanceIds defines the instances to be included for analysis + +After the query, the following data would be returned to render a flame graph: +1. **taskId**: The task id. +2. **elements**: Combined with "id" to determine the hierarchical relationship. + 1. **Id**: Id is the identity of the stack element. + 2. **parentId**: Parent element ID. The dependency relationship between elements can be determined using the element ID and parent element ID. + 3. **codeSignature**: Method signatures in tree nodes. + 4. **total**:The total number of samples of the current tree node, including child nodes. + 5. **self**: The sampling number of the current tree node, excluding samples of the children. \ No newline at end of file diff --git a/docs/menu.yml b/docs/menu.yml index 79e32693040a..702fd1923453 100644 --- a/docs/menu.yml +++ b/docs/menu.yml @@ -270,6 +270,8 @@ catalog: path: "/en/setup/backend/backend-continuous-profiling" - name: "Java App Profiling" path: "/en/setup/backend/backend-java-app-profiling" + - name: "Go App Profiling" + path: "en/setup/backend/backend-go-app-profiling.md" - name: "Event" path: "/en/concepts-and-designs/event/" - name: "Extension" diff --git a/oap-server/server-core/pom.xml b/oap-server/server-core/pom.xml index 784da5435c03..f6d233587ad2 100644 --- a/oap-server/server-core/pom.xml +++ b/oap-server/server-core/pom.xml @@ -44,6 +44,11 @@ library-async-profiler-jfr-parser ${project.version} + + org.apache.skywalking + library-pprof-parser + ${project.version} + org.apache.skywalking telemetry-api diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java index 91509759eab3..b6d1734d2f43 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java @@ -22,6 +22,7 @@ import java.util.List; import org.apache.skywalking.oap.server.core.analysis.meter.MeterSystem; import org.apache.skywalking.oap.server.core.cache.AsyncProfilerTaskCache; +import org.apache.skywalking.oap.server.core.cache.PprofTaskCache; import org.apache.skywalking.oap.server.core.cache.NetworkAddressAliasCache; import org.apache.skywalking.oap.server.core.cache.ProfileTaskCache; import org.apache.skywalking.oap.server.core.command.CommandService; @@ -41,6 +42,8 @@ import org.apache.skywalking.oap.server.core.profiling.continuous.ContinuousProfilingQueryService; import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingMutationService; import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingQueryService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofMutationService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofQueryService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskMutationService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskQueryService; import org.apache.skywalking.oap.server.core.query.AggregationQueryService; @@ -106,6 +109,7 @@ public Class[] services() { addManagementService(classes); addEBPFProfilingService(classes); addAsyncProfilerService(classes); + addPprofService(classes); classes.add(CommandService.class); classes.add(HierarchyService.class); @@ -137,6 +141,12 @@ private void addAsyncProfilerService(List classes) { classes.add(AsyncProfilerTaskCache.class); } + private void addPprofService(List classes) { + classes.add(PprofMutationService.class); + classes.add(PprofQueryService.class); + classes.add(PprofTaskCache.class); + } + private void addOALService(List classes) { classes.add(OALEngineLoaderService.class); } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java index cad853c86654..8052d8c44863 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java @@ -90,6 +90,10 @@ public class CoreModuleConfig extends ModuleConfig { * Following are cache setting for none stream(s) */ private long maxSizeOfProfileTask = 10_000L; + /** + * Following are cache setting for none stream(s) + */ + private long maxSizeOfPprofTask = 10_000L; /** * Analyze profile snapshots paging size. */ diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java index ad9f7c4b61ad..d55dc78853ee 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java @@ -35,6 +35,7 @@ import org.apache.skywalking.oap.server.core.cache.AsyncProfilerTaskCache; import org.apache.skywalking.oap.server.core.cache.CacheUpdateTimer; import org.apache.skywalking.oap.server.core.cache.NetworkAddressAliasCache; +import org.apache.skywalking.oap.server.core.cache.PprofTaskCache; import org.apache.skywalking.oap.server.core.cache.ProfileTaskCache; import org.apache.skywalking.oap.server.core.cluster.ClusterCoordinator; import org.apache.skywalking.oap.server.core.cluster.ClusterModule; @@ -65,6 +66,8 @@ import org.apache.skywalking.oap.server.core.profiling.continuous.ContinuousProfilingQueryService; import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingMutationService; import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingQueryService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofMutationService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofQueryService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskMutationService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskQueryService; import org.apache.skywalking.oap.server.core.query.AggregationQueryService; @@ -330,6 +333,12 @@ TTLStatusQuery.class, new TTLStatusQuery( AsyncProfilerQueryService.class, new AsyncProfilerQueryService(getManager())); this.registerServiceImplementation( AsyncProfilerTaskCache.class, new AsyncProfilerTaskCache(getManager(), moduleConfig)); + this.registerServiceImplementation( + PprofMutationService.class, new PprofMutationService(getManager())); + this.registerServiceImplementation( + PprofQueryService.class, new PprofQueryService(getManager())); + this.registerServiceImplementation( + PprofTaskCache.class, new PprofTaskCache(moduleConfig)); this.registerServiceImplementation( EBPFProfilingMutationService.class, new EBPFProfilingMutationService(getManager())); this.registerServiceImplementation( diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/CacheUpdateTimer.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/CacheUpdateTimer.java index c38f90daccd0..1344cf7f22fe 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/CacheUpdateTimer.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/CacheUpdateTimer.java @@ -27,12 +27,15 @@ import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.storage.AsyncProfilerTaskRecord; import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerTask; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; import org.apache.skywalking.oap.server.library.util.CollectionUtils; import org.apache.skywalking.oap.server.library.util.RunnableWithExceptionProtection; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.analysis.DisableRegister; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.manual.networkalias.NetworkAddressAlias; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskRecord; import org.apache.skywalking.oap.server.core.query.type.ProfileTask; import org.apache.skywalking.oap.server.core.storage.StorageModule; @@ -46,6 +49,8 @@ public enum CacheUpdateTimer { private AsyncProfilerTaskCache asyncProfilerTaskCache; private IAsyncProfilerTaskQueryDAO asyncProfilerTaskQueryDAO; + private PprofTaskCache pprofTaskCache; + private IPprofTaskQueryDAO pprofTaskQueryDAO; private int ttl = 10; @@ -72,6 +77,10 @@ private void update(ModuleDefineHolder moduleDefineHolder) { if (!DisableRegister.INSTANCE.include(AsyncProfilerTaskRecord.INDEX_NAME)) { updateAsyncProfilerTask(moduleDefineHolder); } + + if (!DisableRegister.INSTANCE.include(PprofTaskRecord.INDEX_NAME)) { + updatePprofTask(moduleDefineHolder); + } } /** @@ -163,4 +172,42 @@ private void updateAsyncProfilerTask(ModuleDefineHolder moduleDefineHolder) { return; } + + private PprofTaskCache getPprofTaskCache(ModuleDefineHolder moduleDefineHolder) { + if (pprofTaskCache == null) { + pprofTaskCache = moduleDefineHolder.find(CoreModule.NAME) + .provider() + .getService(PprofTaskCache.class); + } + return pprofTaskCache; + } + + private IPprofTaskQueryDAO getPprofTaskQueryDAO(ModuleDefineHolder moduleDefineHolder) { + if (pprofTaskQueryDAO == null) { + pprofTaskQueryDAO = moduleDefineHolder.find(StorageModule.NAME) + .provider() + .getService(IPprofTaskQueryDAO.class); + } + return pprofTaskQueryDAO; + } + + private void updatePprofTask(ModuleDefineHolder moduleDefineHolder) { + PprofTaskCache taskCache = getPprofTaskCache(moduleDefineHolder); + IPprofTaskQueryDAO taskQueryDAO = getPprofTaskQueryDAO(moduleDefineHolder); + + try { + List taskList = taskQueryDAO.getTaskList( + null, taskCache.getCacheStartTimeBucket(), taskCache.getCacheEndTimeBucket(), null + ); + taskList.stream().collect(Collectors.groupingBy(t -> t.getServiceId())).entrySet().stream().forEach(e -> { + final String serviceId = e.getKey(); + final List pprofTasks = e.getValue(); + + pprofTaskCache.saveTaskList(serviceId, pprofTasks); + }); + + } catch (IOException e) { + log.warn("Unable to update pprof task cache", e); + } + } } \ No newline at end of file diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java new file mode 100644 index 000000000000..ab4a52b66b63 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.cache; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.apache.skywalking.oap.server.core.CoreModuleConfig; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.library.module.Service; + +public class PprofTaskCache implements Service { + private final Cache> serviceId2taskCache; + + public PprofTaskCache(CoreModuleConfig moduleConfig) { + long initialSize = moduleConfig.getMaxSizeOfPprofTask() / 10L; + int initialCapacitySize = (int) (initialSize > Integer.MAX_VALUE ? Integer.MAX_VALUE : initialSize); + + serviceId2taskCache = CacheBuilder.newBuilder() + .initialCapacity(initialCapacitySize) + .maximumSize(moduleConfig.getMaxSizeOfProfileTask()) + // remove old pprof task data + .expireAfterWrite(Duration.ofMinutes(1)) + .build(); + } + + public List getPprofTaskList(String serviceId) { + // read pprof task list from cache only, use cache update timer mechanism + List pprofTaskList = serviceId2taskCache.getIfPresent(serviceId); + return pprofTaskList; + } + + /** + * save service task list + */ + public void saveTaskList(String serviceId, List taskList) { + if (taskList == null) { + taskList = Collections.emptyList(); + } + + serviceId2taskCache.put(serviceId, taskList); + } + + /** + * use for every db query, -5min start time + */ + public long getCacheStartTimeBucket() { + return TimeBucket.getRecordTimeBucket(System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(5)); + } + + /** + * use for every db query, +5min end time(because search through task's create time) + */ + public long getCacheEndTimeBucket() { + return TimeBucket.getRecordTimeBucket(System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(5)); + } + +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/command/CommandService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/command/CommandService.java index ac7ef401769a..40a8de67e06c 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/command/CommandService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/command/CommandService.java @@ -33,6 +33,7 @@ import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerEventType; import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerTask; import org.apache.skywalking.oap.server.core.query.type.EBPFProfilingTaskExtension; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; import org.apache.skywalking.oap.server.library.util.CollectionUtils; import org.apache.skywalking.oap.server.library.util.StringUtil; import org.apache.skywalking.oap.server.network.trace.component.command.AsyncProfilerTaskCommand; @@ -40,6 +41,7 @@ import org.apache.skywalking.oap.server.network.trace.component.command.ContinuousProfilingPolicyCommand; import org.apache.skywalking.oap.server.network.trace.component.command.EBPFProfilingTaskCommand; import org.apache.skywalking.oap.server.network.trace.component.command.EBPFProfilingTaskExtensionConfig; +import org.apache.skywalking.oap.server.network.trace.component.command.PprofTaskCommand; import org.apache.skywalking.oap.server.network.trace.component.command.ProfileTaskCommand; import org.apache.skywalking.oap.server.core.query.type.ProfileTask; import org.apache.skywalking.oap.server.library.module.ModuleManager; @@ -72,6 +74,19 @@ public AsyncProfilerTaskCommand newAsyncProfileTaskCommand(AsyncProfilerTask tas eventNames, task.getExecArgs(), task.getCreateTime()); } + /** + * Create a new pprof task command for Go agents + */ + public PprofTaskCommand newPprofTaskCommand(PprofTask task) { + final String serialNumber = UUID.randomUUID().toString(); + String events = ""; + if (task.getEvents() != null) { + events = task.getEvents().getName(); + } + return new PprofTaskCommand(serialNumber, task.getId(), events, + task.getDuration(), task.getCreateTime(), task.getDumpPeriod()); + } + /** * Used to notify the eBPF Profiling task to the eBPF agent side */ diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java new file mode 100644 index 000000000000..a220ba29304e --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.TimeUnit; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.worker.NoneStreamProcessor; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskCreationResult; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskCreationType; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.Service; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; + +@Slf4j +@RequiredArgsConstructor +public class PprofMutationService implements Service { + private final ModuleManager moduleManager; + + private IPprofTaskQueryDAO taskQueryDAO; + + private IPprofTaskQueryDAO getPprofTaskDAO() { + if (taskQueryDAO == null) { + this.taskQueryDAO = moduleManager.find(StorageModule.NAME) + .provider() + .getService(IPprofTaskQueryDAO.class); + } + return taskQueryDAO; + } + + public PprofTaskCreationResult createTask(String serviceId, + List serviceInstanceIds, + int duration, + PprofEventType events, + int dumpPeriod) throws IOException { + long createTime = System.currentTimeMillis(); + // check data + PprofTaskCreationResult checkResult = checkDataSuccess( + serviceId, serviceInstanceIds, duration, createTime, events, dumpPeriod + ); + if (checkResult != null) { + return checkResult; + } + // create task + PprofTaskRecord task = new PprofTaskRecord(); + String taskId = createTime + Const.ID_CONNECTOR + serviceId; + task.setTaskId(taskId); + task.setServiceId(serviceId); + task.setServiceInstanceIdsFromList(serviceInstanceIds); + task.setDuration(duration); + task.setEvents(events.toString()); + task.setDumpPeriod(dumpPeriod); + task.setCreateTime(createTime); + task.setTimeBucket(TimeBucket.getRecordTimeBucket(createTime)); + NoneStreamProcessor.getInstance().in(task); + return PprofTaskCreationResult.builder() + .id(task.id().build()) + .code(PprofTaskCreationType.SUCCESS) + .build(); + } + + private PprofTaskCreationResult checkDataSuccess(String serviceId, + List serviceInstanceIds, + int duration, + long createTime, + PprofEventType events, + int dumpPeriod) throws IOException { + String checkArgumentMessage = checkArgumentError(serviceId, serviceInstanceIds, duration, events, dumpPeriod); + if (checkArgumentMessage != null) { + return PprofTaskCreationResult.builder() + .code(PprofTaskCreationType.ARGUMENT_ERROR) + .errorReason(checkArgumentMessage) + .build(); + } + String checkTaskProfilingMessage = checkTaskProfiling(serviceId, events, createTime); + if (checkTaskProfilingMessage != null) { + return PprofTaskCreationResult.builder() + .code(PprofTaskCreationType.ALREADY_PROFILING_ERROR) + .errorReason(checkTaskProfilingMessage) + .build(); + } + return null; + } + + private String checkArgumentError(String serviceId, + List serviceInstanceIds, + int duration, + PprofEventType events, + int dumpPeriod) { + if (serviceId == null) { + return "service cannot be null"; + } + if (events == null) { + return "events cannot be empty"; + } + if (events == PprofEventType.CPU || events == PprofEventType.BLOCK || events == PprofEventType.MUTEX) { + if (duration <= 0) { + return "duration cannot be negative"; + } + if (duration > 15) { + return "duration cannot be greater than 15 minutes"; + } + } + if (events == PprofEventType.BLOCK || events == PprofEventType.MUTEX) { + if (dumpPeriod <= 0) { + return "dumpPeriod cannot be negative"; + } + } + if (CollectionUtils.isEmpty(serviceInstanceIds)) { + return "serviceInstanceIds cannot be empty"; + } + return null; + } + + private String checkTaskProfiling(String serviceId, + PprofEventType events, + long createTime) throws IOException { + // Each service can only enable one task at a time + long endTimeBucket = TimeBucket.getRecordTimeBucket(createTime); + final List alreadyHaveTaskList = getPprofTaskDAO().getTaskList( + serviceId, null, endTimeBucket, null + ); + if (CollectionUtils.isNotEmpty(alreadyHaveTaskList)) { + for (PprofTask task : alreadyHaveTaskList) { + if (task.getEvents().equals(events) && task.getCreateTime() + TimeUnit.MINUTES.toMillis( + task.getDuration()) >= createTime) { + // if the endTime is greater or equal than the createTime of the newly created task, i.e. there is overlap between two tasks, it is an invalid case, it will return an error + return "current service already has monitor pprof task execute at this time"; + } + } + } + return null; + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofQueryService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofQueryService.java new file mode 100644 index 000000000000..101b5a685fbd --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofQueryService.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof; + +import com.google.gson.Gson; +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.type.PprofStackTree; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.Service; +import org.apache.skywalking.oap.server.library.pprof.parser.PprofMergeBuilder; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; + +@Slf4j +@RequiredArgsConstructor +public class PprofQueryService implements Service { + private static final Gson GSON = new Gson(); + + private final ModuleManager moduleManager; + + private IPprofTaskQueryDAO taskQueryDAO; + private IPprofDataQueryDAO dataQueryDAO; + private IPprofTaskLogQueryDAO logQueryDAO; + + private IPprofTaskQueryDAO getTaskQueryDAO() { + if (taskQueryDAO == null) { + this.taskQueryDAO = moduleManager.find(StorageModule.NAME) + .provider() + .getService(IPprofTaskQueryDAO.class); + } + return taskQueryDAO; + } + + private IPprofDataQueryDAO getPprofDataQueryDAO() { + if (dataQueryDAO == null) { + this.dataQueryDAO = moduleManager.find(StorageModule.NAME) + .provider() + .getService(IPprofDataQueryDAO.class); + } + return dataQueryDAO; + } + + private IPprofTaskLogQueryDAO getTaskLogQueryDAO() { + if (logQueryDAO == null) { + this.logQueryDAO = moduleManager.find(StorageModule.NAME) + .provider() + .getService(IPprofTaskLogQueryDAO.class); + } + return logQueryDAO; + } + + public List queryTask(String serviceId, Duration duration, Integer limit) throws IOException { + Long startTimeBucket = null; + Long endTimeBucket = null; + if (Objects.nonNull(duration)) { + startTimeBucket = duration.getStartTimeBucketInSec(); + endTimeBucket = duration.getEndTimeBucketInSec(); + } + List tasks = getTaskQueryDAO().getTaskList(serviceId, startTimeBucket, endTimeBucket, limit); + return tasks; + } + + public PprofStackTree queryPprofData(String taskId, List instanceIds) throws IOException { + List pprofDataList = getPprofDataQueryDAO().getByTaskIdAndInstances( + taskId, instanceIds); + List trees = pprofDataList.stream() + .map(data -> GSON.fromJson( + new String(data.getDataBinary()), + FrameTree.class + )) + .collect(Collectors.toList()); + FrameTree resultTree = new PprofMergeBuilder() + .merge(trees) + .build(); + return new PprofStackTree(resultTree); + } + + public List queryPprofTaskLogs(String taskId) throws IOException { + List taskLogList = getTaskLogQueryDAO().getTaskLogList(); + return findMatchedLogs(taskId, taskLogList); + } + + private List findMatchedLogs(final String taskID, final List allLogs) { + return allLogs.stream() + .filter(l -> Objects.equals(l.getId(), taskID)) + .map(this::extendTaskLog) + .collect(Collectors.toList()); + } + + private PprofTaskLog extendTaskLog(PprofTaskLog log) { + final IDManager.ServiceInstanceID.InstanceIDDefinition instanceIDDefinition = IDManager.ServiceInstanceID + .analysisId(log.getInstanceId()); + log.setInstanceName(instanceIDDefinition.getName()); + return log; + } + +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java new file mode 100644 index 000000000000..631d11bf6f07 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof.storage; + +import com.google.gson.Gson; +import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.source.PprofProfilingData; + +public class PprofProfilingDataDispatcher implements SourceDispatcher { + private static final Gson GSON = new Gson(); + + @Override + public void dispatch(PprofProfilingData source) { + PprofProfilingDataRecord record = new PprofProfilingDataRecord(); + record.setTaskId(source.getTaskId()); + record.setInstanceId(source.getInstanceId()); + record.setDataBinary(GSON.toJson(source.getFrameTree()).getBytes()); + record.setUploadTime(source.getUploadTime()); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(source.getUploadTime())); + RecordStreamProcessor.getInstance().in(record); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java new file mode 100644 index 000000000000..10d724934e1b --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof.storage; + +import com.google.common.hash.Hashing; +import java.nio.charset.StandardCharsets; +import lombok.Data; +import org.apache.skywalking.oap.server.core.analysis.Stream; +import org.apache.skywalking.oap.server.core.analysis.record.Record; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.storage.StorageID; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB; +import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; +import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.PPROF_PROFILING_DATA; + +@Data +@Stream(name = PprofProfilingDataRecord.INDEX_NAME, scopeId = PPROF_PROFILING_DATA, + builder = PprofProfilingDataRecord.Builder.class, processor = RecordStreamProcessor.class) +@BanyanDB.TimestampColumn(PprofProfilingDataRecord.UPLOAD_TIME) +@BanyanDB.Group(streamGroup = BanyanDB.StreamGroup.RECORDS) +public class PprofProfilingDataRecord extends Record { + public static final String INDEX_NAME = "pprof_profiling_data"; + public static final String TASK_ID = "task_id"; + public static final String INSTANCE_ID = "instance_id"; + public static final String DATA_BINARY = "data_binary"; + public static final String UPLOAD_TIME = "upload_time"; + + @Column(name = TASK_ID) + private String taskId; + + @Column(name = INSTANCE_ID) + @BanyanDB.SeriesID(index = 0) + private String instanceId; + + @Column(name = UPLOAD_TIME) + private long uploadTime; + + @Column(name = DATA_BINARY, storageOnly = true) + private byte[] dataBinary; + + @Override + public StorageID id() { + return new StorageID().append( + Hashing.sha256().newHasher() + .putString(taskId, StandardCharsets.UTF_8) + .putString(instanceId, StandardCharsets.UTF_8) + .putLong(uploadTime) + .hash().toString() + ); + } + + public static class Builder implements StorageBuilder { + @Override + public PprofProfilingDataRecord storage2Entity(final Convert2Entity converter) { + final PprofProfilingDataRecord dataTraffic = new PprofProfilingDataRecord(); + dataTraffic.setTimeBucket(((Number) converter.get(TIME_BUCKET)).longValue()); + dataTraffic.setTaskId((String) converter.get(TASK_ID)); + dataTraffic.setInstanceId((String) converter.get(INSTANCE_ID)); + dataTraffic.setUploadTime(((Number) converter.get(UPLOAD_TIME)).longValue()); + dataTraffic.setDataBinary(converter.getBytes(DATA_BINARY)); + return dataTraffic; + } + + @Override + public void entity2Storage(final PprofProfilingDataRecord storageData, final Convert2Storage converter) { + converter.accept(TIME_BUCKET, storageData.getTimeBucket()); + converter.accept(TASK_ID, storageData.getTaskId()); + converter.accept(INSTANCE_ID, storageData.getInstanceId()); + converter.accept(UPLOAD_TIME, storageData.getUploadTime()); + converter.accept(DATA_BINARY, storageData.getDataBinary()); + } + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java new file mode 100644 index 000000000000..c1cf1b09e843 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof.storage; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.analysis.Stream; +import org.apache.skywalking.oap.server.core.analysis.record.Record; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.source.ScopeDeclaration; +import org.apache.skywalking.oap.server.core.storage.StorageID; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB; +import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearch; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; +import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.PPROF_TASK_LOG; + +@Getter +@Setter +@ScopeDeclaration(id = PPROF_TASK_LOG, name = "PprofTaskLog") +@Stream(name = PprofTaskLogRecord.INDEX_NAME, scopeId = PPROF_TASK_LOG, builder = PprofTaskLogRecord.Builder.class, processor = RecordStreamProcessor.class) +@BanyanDB.TimestampColumn(PprofTaskLogRecord.TIMESTAMP) +@BanyanDB.Group(streamGroup = BanyanDB.StreamGroup.RECORDS) +public class PprofTaskLogRecord extends Record { + public static final String INDEX_NAME = "pprof_task_log"; + public static final String TASK_ID = "task_id"; + public static final String INSTANCE_ID = "instance_id"; + public static final String OPERATION_TYPE = "operation_type"; + public static final String OPERATION_TIME = "operation_time"; + public static final String TIMESTAMP = "timestamp"; + + @Column(name = TASK_ID) + private String taskId; + @Column(name = INSTANCE_ID) + @BanyanDB.SeriesID(index = 0) + private String instanceId; + @Column(name = OPERATION_TYPE, storageOnly = true) + private int operationType; + @ElasticSearch.EnableDocValues + @Column(name = OPERATION_TIME) + private long operationTime; + @Getter + @Setter + @ElasticSearch.EnableDocValues + @Column(name = TIMESTAMP) + private long timestamp; + + @Override + public StorageID id() { + return new StorageID() + .append(TASK_ID, getTaskId()) + .append(INSTANCE_ID, getInstanceId()) + .append(OPERATION_TYPE, getOperationType()) + .append(OPERATION_TIME, getOperationTime()); + } + + public static class Builder implements StorageBuilder { + @Override + public PprofTaskLogRecord storage2Entity(final Convert2Entity converter) { + final PprofTaskLogRecord log = new PprofTaskLogRecord(); + log.setTaskId((String) converter.get(TASK_ID)); + log.setInstanceId((String) converter.get(INSTANCE_ID)); + log.setOperationType(((Number) converter.get(OPERATION_TYPE)).intValue()); + log.setOperationTime(((Number) converter.get(OPERATION_TIME)).longValue()); + log.setTimestamp(((Number) converter.get(TIMESTAMP)).longValue()); + log.setTimeBucket(((Number) converter.get(TIME_BUCKET)).longValue()); + return log; + } + + @Override + public void entity2Storage(final PprofTaskLogRecord storageData, final Convert2Storage converter) { + converter.accept(TASK_ID, storageData.getTaskId()); + converter.accept(INSTANCE_ID, storageData.getInstanceId()); + converter.accept(OPERATION_TYPE, storageData.getOperationType()); + converter.accept(OPERATION_TIME, storageData.getOperationTime()); + converter.accept(TIME_BUCKET, storageData.getTimeBucket()); + converter.accept(TIMESTAMP, storageData.getTimestamp()); + } + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java new file mode 100644 index 000000000000..ab207b5a6a2c --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof.storage; + +import com.google.gson.Gson; +import java.util.List; +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.analysis.Stream; +import org.apache.skywalking.oap.server.core.analysis.config.NoneStream; +import org.apache.skywalking.oap.server.core.analysis.worker.NoneStreamProcessor; +import org.apache.skywalking.oap.server.core.source.ScopeDeclaration; +import org.apache.skywalking.oap.server.core.storage.StorageID; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB; +import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearch; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; +import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.PPROF_TASK; + +/** + * Pprof task database bean, use none stream + */ +@Getter +@Setter +@ScopeDeclaration(id = PPROF_TASK, name = "PprofTask") +@Stream(name = PprofTaskRecord.INDEX_NAME, scopeId = PPROF_TASK, builder = PprofTaskRecord.Builder.class, processor = NoneStreamProcessor.class) +@BanyanDB.TimestampColumn(PprofTaskRecord.CREATE_TIME) +@BanyanDB.Group(streamGroup = BanyanDB.StreamGroup.RECORDS) +public class PprofTaskRecord extends NoneStream { + private static final Gson GSON = new Gson(); + + public static final String INDEX_NAME = "pprof_task"; + public static final String TASK_ID = "task_id"; + public static final String SERVICE_ID = "service_id"; + public static final String SERVICE_INSTANCE_IDS = "service_instance_ids"; + public static final String CREATE_TIME = "create_time"; + public static final String EVENT_TYPES = "events"; + public static final String DURATION = "duration"; + public static final String DUMP_PERIOD = "dump_period"; + + @Override + public StorageID id() { + return new StorageID().append(TASK_ID, taskId); + } + + @Column(name = SERVICE_ID) + @BanyanDB.SeriesID(index = 0) + private String serviceId; + @Column(name = SERVICE_INSTANCE_IDS) + private String serviceInstanceIds; + @Column(name = TASK_ID) + private String taskId; + @ElasticSearch.EnableDocValues + @Column(name = CREATE_TIME) + private long createTime; + @ElasticSearch.EnableDocValues + @Column(name = EVENT_TYPES) + private String events; + @Column(name = DURATION) + private int duration; + @Column(name = DUMP_PERIOD) + private int dumpPeriod; + + public static class Builder implements StorageBuilder { + + @Override + public PprofTaskRecord storage2Entity(final Convert2Entity converter) { + PprofTaskRecord record = new PprofTaskRecord(); + record.setServiceId((String) converter.get(SERVICE_ID)); + record.setServiceInstanceIds((String) converter.get(SERVICE_INSTANCE_IDS)); + record.setTaskId((String) converter.get(TASK_ID)); + record.setCreateTime(((Number) converter.get(CREATE_TIME)).longValue()); + record.setEvents((String) converter.get(EVENT_TYPES)); + record.setDuration(((Number) converter.get(DURATION)).intValue()); + record.setDumpPeriod(((Number) converter.get(DUMP_PERIOD)).intValue()); + record.setTimeBucket(((Number) converter.get(TIME_BUCKET)).longValue()); + return record; + } + + @Override + public void entity2Storage(final PprofTaskRecord storageData, final Convert2Storage converter) { + converter.accept(SERVICE_ID, storageData.getServiceId()); + converter.accept(SERVICE_INSTANCE_IDS, storageData.getServiceInstanceIds()); + converter.accept(TASK_ID, storageData.getTaskId()); + converter.accept(CREATE_TIME, storageData.getCreateTime()); + converter.accept(EVENT_TYPES, storageData.getEvents()); + converter.accept(DURATION, storageData.getDuration()); + converter.accept(DUMP_PERIOD, storageData.getDumpPeriod()); + converter.accept(TIME_BUCKET, storageData.getTimeBucket()); + } + } + + public void setServiceInstanceIdsFromList(List serviceInstanceIds) { + this.serviceInstanceIds = GSON.toJson(serviceInstanceIds); + } + +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PprofTaskLog.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PprofTaskLog.java new file mode 100644 index 000000000000..471bde9cdfaa --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PprofTaskLog.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; + +@Setter +@Getter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class PprofTaskLog { + // task id + private String id; + + // instance + private String instanceId; + private String instanceName; + + // operation + private PprofTaskLogOperationType operationType; + private long operationTime; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofAnalyzationRequest.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofAnalyzationRequest.java new file mode 100644 index 000000000000..b25a9eea93fd --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofAnalyzationRequest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.input; + +import java.util.List; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class PprofAnalyzationRequest { + private String taskId; + private List instanceIds; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskCreationRequest.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskCreationRequest.java new file mode 100644 index 000000000000..a487bdf0a42f --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskCreationRequest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.input; + +import java.util.List; +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; + +@Getter +@Setter +public class PprofTaskCreationRequest { + private String serviceId; + private List serviceInstanceIds; + private int duration; + private PprofEventType events; + private int dumpPeriod; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java new file mode 100644 index 000000000000..a2304a04ba95 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.input; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class PprofTaskListRequest { + private String serviceId; + private Duration queryDuration; + private Integer limit; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofAnalyzation.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofAnalyzation.java new file mode 100644 index 000000000000..ef56f268acad --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofAnalyzation.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class PprofAnalyzation { + private PprofStackTree tree; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofEventType.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofEventType.java new file mode 100644 index 000000000000..2429a102380f --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofEventType.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public enum PprofEventType { + CPU(0, "cpu"), + HEAP(1, "heap"), + BLOCK(2, "block"), + MUTEX(3, "mutex"), + GOROUTINE(4, "goroutine"), + THREADCREATE(5, "threadcreate"), + ALLOCS(6, "allocs"); + + private final int code; + private final String name; + + public static PprofEventType valueOfString(String event) { + return PprofEventType.valueOf(event); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackElement.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackElement.java new file mode 100644 index 000000000000..404b18cc4197 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackElement.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter + +public class PprofStackElement { + // work for tree building, id matches multiple parentId + private int id; + private int parentId; + + // stack code signature + private String codeSignature; + + private long total; + private long self; + +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackTree.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackTree.java new file mode 100644 index 000000000000..53e8f88ba7a2 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackTree.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import com.google.common.collect.Lists; +import java.util.List; +import java.util.Objects; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; + +@Setter +@Getter +@NoArgsConstructor +public class PprofStackTree { + private List elements; + + private int idGen = 0; + + public PprofStackTree(FrameTree tree) { + this.elements = convertTree(-1, tree); + } + + private List convertTree(int parentId, FrameTree tree) { + PprofStackElement pprofStackElement = new PprofStackElement(); + pprofStackElement.setId(idGen++); + pprofStackElement.setParentId(parentId); + pprofStackElement.setCodeSignature(tree.getSignature()); + pprofStackElement.setTotal(tree.getTotal()); + pprofStackElement.setSelf(tree.getSelf()); + + List children = tree.getChildren(); + List result = Lists.newArrayList(pprofStackElement); + if (Objects.isNull(children) || children.isEmpty()) { + return result; + } + + for (FrameTree child : children) { + List childElements = convertTree(pprofStackElement.getId(), child); + result.addAll(childElements); + } + + return result; + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTask.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTask.java new file mode 100644 index 000000000000..e54b3fab55e9 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTask.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Setter +@Getter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class PprofTask { + + private String id; + private String serviceId; + private List serviceInstanceIds; + private PprofEventType events; + private long createTime; + private int duration; + private int dumpPeriod; + +} \ No newline at end of file diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java new file mode 100644 index 000000000000..10f7653e9aa6 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +/** + * create pprof task result + */ +@Setter +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class PprofTaskCreationResult { + // ErrorReason gives detailed reason for the exception, if the code returned represents a kind of failure. + private String errorReason; + // Code defines the status of the response, i.e. success or failure. + private PprofTaskCreationType code; + // Task id, if code is SUCCESS. + private String id; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java new file mode 100644 index 000000000000..14519f62d3c0 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +public enum PprofTaskCreationType { + SUCCESS, + ARGUMENT_ERROR, + ALREADY_PROFILING_ERROR, + ; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskListResult.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskListResult.java new file mode 100644 index 000000000000..f136b35f504a --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskListResult.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class PprofTaskListResult { + private String errorReason; + private List tasks; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskLogOperationType.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskLogOperationType.java new file mode 100644 index 000000000000..3c1e544759f1 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskLogOperationType.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import java.util.HashMap; +import java.util.Map; + +public enum PprofTaskLogOperationType { + NOTIFIED(1), // when sniffer has execution finished to report + EXECUTION_FINISHED(2), // when sniffer has execution finished to report + PPROF_UPLOAD_FILE_TOO_LARGE_ERROR( + 3), // when sniffer finished task but jfr file is to large that oap server can not receive + EXECUTION_TASK_ERROR(4) // when sniffer fails to execute its task + ; + + private final int code; + private static final Map CACHE = new HashMap(); + + static { + for (PprofTaskLogOperationType val : PprofTaskLogOperationType.values()) { + CACHE.put(val.getCode(), val); + } + } + + /** + * Parse operation type by code + */ + public static PprofTaskLogOperationType parse(int code) { + return CACHE.get(code); + } + + PprofTaskLogOperationType(int code) { + this.code = code; + } + + public int getCode() { + return this.code; + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskProgress.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskProgress.java new file mode 100644 index 000000000000..95de35c035de --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskProgress.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import java.util.List; +import lombok.Data; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; + +@Data +public class PprofTaskProgress { + private List logs; + private List errorInstanceIds; + private List successInstanceIds; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java index 71fe59b4e0cc..359910b05a89 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java @@ -152,6 +152,9 @@ public class DefaultScopeDefine { public static final int BROWSER_APP_WEB_INTERACTION_PAGE_PERF = 89; public static final int SW_SPAN_ATTACHED_EVENT = 90; public static final int SERVICE_DATABASE_SLOW_STATEMENT = 91; + public static final int PPROF_TASK = 92; + public static final int PPROF_PROFILING_DATA = 93; + public static final int PPROF_TASK_LOG = 94; /** * Catalog of scope, the metrics processor could use this to group all generated metrics by oal rt. diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/PprofProfilingData.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/PprofProfilingData.java new file mode 100644 index 000000000000..d9c23835f86c --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/PprofProfilingData.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.source; + +import lombok.Data; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.PPROF_PROFILING_DATA; + +@Data +@ScopeDeclaration(id = PPROF_PROFILING_DATA, name = "PprofProfilingData") +@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class) +public class PprofProfilingData extends Source { + private volatile String entityId; + + @Override + public int scope() { + return PPROF_PROFILING_DATA; + } + + @Override + public String getEntityId() { + if (entityId == null) { + return taskId + instanceId + eventType.name() + uploadTime; + } + return entityId; + } + + private String taskId; + private String instanceId; + private long uploadTime; + private PprofEventType eventType; + private Object frameTree; +} \ No newline at end of file diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java index 89ac015cd0b4..cd711dedeaa7 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java @@ -24,6 +24,9 @@ import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IJFRDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.continuous.IContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IServiceLabelDAO; import org.apache.skywalking.oap.server.core.storage.profiling.trace.IProfileTaskLogQueryDAO; @@ -96,6 +99,9 @@ public Class[] services() { IAsyncProfilerTaskQueryDAO.class, IAsyncProfilerTaskLogQueryDAO.class, IJFRDataQueryDAO.class, + IPprofTaskQueryDAO.class, + IPprofTaskLogQueryDAO.class, + IPprofDataQueryDAO.class, StorageTTLStatusQuery.class }; } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofDataQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofDataQueryDAO.java new file mode 100644 index 000000000000..f593d1f07e89 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofDataQueryDAO.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.oap.server.core.storage.profiling.pprof; + +import java.io.IOException; +import java.util.List; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import org.apache.skywalking.oap.server.library.module.Service; + +public interface IPprofDataQueryDAO extends Service { + /** + * get pprof data record + * + * @param taskId taskId + * @param instanceIds instances of successfully uploaded file and parsed + * @return record list + */ + List getByTaskIdAndInstances(final String taskId, + List instanceIds) throws IOException; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java new file mode 100644 index 000000000000..37b0205d8784 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.storage.profiling.pprof; + +import java.io.IOException; +import java.util.List; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.storage.DAO; + +public interface IPprofTaskLogQueryDAO extends DAO { + /** + * search all task log list in appoint task id + */ + List getTaskLogList() throws IOException; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskQueryDAO.java new file mode 100644 index 000000000000..f97a3249f3d6 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskQueryDAO.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.storage.profiling.pprof; + +import java.io.IOException; +import java.util.List; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.storage.DAO; + +public interface IPprofTaskQueryDAO extends DAO { + + /** + * search task list in appoint time bucket + * + * @param serviceId monitor service id, maybe null + * @param startTimeBucket time bucket bigger than or equals, nullable + * @param endTimeBucket time bucket smaller than or equals, nullable + * @param limit limit count, if null means query all + */ + List getTaskList(final String serviceId, final Long startTimeBucket, + final Long endTimeBucket, final Integer limit) throws IOException; + + /** + * query profile task by id + * + * @param id taskId + * @return task data + */ + PprofTask getById(final String id) throws IOException; + +} diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/CoreModuleTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/CoreModuleTest.java index a8e0098a72e2..cd1edcc1cac9 100644 --- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/CoreModuleTest.java +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/CoreModuleTest.java @@ -26,6 +26,6 @@ public class CoreModuleTest { public void testOpenServiceList() { CoreModule coreModule = new CoreModule(); - Assertions.assertEquals(49, coreModule.services().length); + Assertions.assertEquals(52, coreModule.services().length); } } diff --git a/oap-server/server-library/library-pprof-parser/pom.xml b/oap-server/server-library/library-pprof-parser/pom.xml new file mode 100755 index 000000000000..25a549747ea2 --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/pom.xml @@ -0,0 +1,104 @@ + + + + + 4.0.0 + + + org.apache.skywalking + server-library + ${revision} + + + library-pprof-parser + + + 11 + 11 + UTF-8 + true + + + + + com.google.protobuf + protobuf-java + 3.25.5 + + + com.google.code.gson + gson + 2.10.1 + + + org.projectlombok + lombok + 1.18.30 + provided + + + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.6.1 + + com.google.protobuf:protoc:3.25.5:exe:${os.detected.classifier} + + + + + compile + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.3.0 + + + add-generated-sources + generate-sources + + add-source + + + + target/generated-sources/protobuf/java + + + + + + + + + kr.motd.maven + os-maven-plugin + 1.7.0 + + + + \ No newline at end of file diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java new file mode 100644 index 000000000000..dd9f3130d47f --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.pprof.parser; + +import java.util.List; +import org.apache.skywalking.oap.server.library.pprof.type.Frame; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; + +public class PprofMergeBuilder { + private final Frame root = new Frame("root"); + + public PprofMergeBuilder merge(List trees) { + if (trees == null || trees.isEmpty()) { + return this; + } + for (FrameTree tree : trees) { + merge0(root, tree); + } + return this; + } + + public PprofMergeBuilder merge(FrameTree tree) { + merge0(root, tree); + return this; + } + + private void merge0(Frame frame, FrameTree tree) { + if (tree == null) { + return; + } + if (tree.getChildren() != null) { + for (FrameTree childTree : tree.getChildren()) { + Frame child = addChild(frame, childTree.getSignature()); + merge0(child, childTree); + } + } + frame.setTotal(frame.getTotal() + tree.getTotal()); + frame.setSelf(frame.getSelf() + tree.getSelf()); + } + + private Frame addChild(Frame parent, String signature) { + return parent.getChild(signature); + } + + public FrameTree build() { + return FrameTree.buildTree(root); + } + +} diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java new file mode 100644 index 000000000000..2475a1969477 --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.pprof.parser; + +import com.google.perftools.profiles.ProfileProto; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.zip.GZIPInputStream; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTreeBuilder; + +/** + * Parses pprof protobuf format files and converts them to frame trees. + */ +public class PprofParser { + + public static FrameTree dumpTree(ByteBuffer buf) throws IOException { + byte[] bytes = new byte[buf.remaining()]; + buf.get(bytes); + InputStream stream = new java.io.ByteArrayInputStream(bytes); + InputStream inputStream = new GZIPInputStream(stream); + ProfileProto.Profile profile = ProfileProto.Profile.parseFrom(inputStream); + FrameTree tree = new FrameTreeBuilder(profile).build(); + return tree; + } + + public static FrameTree dumpTree(String filePath) throws IOException { + File file = new File(filePath); + if (!file.exists()) { + throw new IOException("Pprof file not found: " + filePath); + } + InputStream fileStream = new FileInputStream(file); + InputStream stream = new GZIPInputStream(fileStream); + ProfileProto.Profile profile = ProfileProto.Profile.parseFrom(stream); + FrameTree tree = new FrameTreeBuilder(profile).build(); + return tree; + } +} diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java new file mode 100644 index 000000000000..34a050cee0b2 --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.pprof.type; + +import java.util.HashMap; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class Frame extends HashMap { + final String signature; + long total; + long self; + + public Frame(String signature) { + this.signature = signature; + } + + public Frame getChild(String signature) { + return super.computeIfAbsent(signature.hashCode(), k -> new Frame(signature)); + } +} diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java new file mode 100755 index 000000000000..3b038a7e7559 --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.pprof.type; + +import java.util.ArrayList; +import java.util.List; +import lombok.Getter; + +@Getter +public class FrameTree { + private String signature; + private long total; + private long self; + private List children; + + public FrameTree(Frame frame) { + this.signature = frame.getSignature(); + this.total = frame.getTotal(); + this.self = frame.getSelf(); + this.children = new ArrayList<>(frame.size()); + } + + public FrameTree(String signature, long total, long self) { + this.signature = signature; + this.total = total; + this.self = self; + this.children = new ArrayList<>(); + } + + public static FrameTree buildTree(Frame frame) { + if (frame == null) + return null; + + FrameTree frameTree = new FrameTree(frame); + // has children? + if (!frame.isEmpty()) { + frameTree.children = new ArrayList<>(frame.size()); + // build tree + for (Frame childFrame : frame.values()) { + FrameTree childFrameTree = buildTree(childFrame); + frameTree.children.add(childFrameTree); + } + } + return frameTree; + } +} diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java new file mode 100644 index 000000000000..43c7d47f64de --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.pprof.type; + +import com.google.perftools.profiles.ProfileProto; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +class RawFrameTree { + private long locationId; + private long total; + private long self; + private final Map children = new HashMap<>(); +} + +public class FrameTreeBuilder { + private final ProfileProto.Profile profile; + private final RawFrameTree root; + + public FrameTreeBuilder(ProfileProto.Profile profile) { + this.profile = profile; + this.root = new RawFrameTree(0, 0, 0); + } + + private FrameTree parseTree(RawFrameTree rawTree) { + FrameTree tree = new FrameTree(getSignature(rawTree.getLocationId()), rawTree.getTotal(), rawTree.getSelf()); + for (RawFrameTree rawChild : rawTree.getChildren().values()) { + FrameTree child = parseTree(rawChild); + tree.getChildren().add(child); + } + return tree; + } + + private String getSignature(long locationId) { + if (locationId == 0) { + return "root"; + } + ProfileProto.Location location = profile.getLocation((int) locationId - 1); + return location.getLineList().stream().map((line) -> { + ProfileProto.Function function = profile.getFunction((int) line.getFunctionId() - 1); + String functionName = profile.getStringTable((int) function.getName()); + return functionName + ":" + line.getLine(); + }).collect(Collectors.joining(";")); + } + + public FrameTree build() { + for (ProfileProto.Sample sample : profile.getSampleList()) { + mergeSample(sample); + } + return parseTree(this.root); + } + + private void mergeSample(ProfileProto.Sample sample) { + // merge sample data + Map children = root.getChildren(); + List locationIdList = new ArrayList<>(sample.getLocationIdList()); + // from root to leaf + Collections.reverse(locationIdList); + int size = locationIdList.size(); + for (int i = 0; i < size; i++) { + boolean isEnd = i == size - 1; + long locationId = locationIdList.get(i); + if (children.containsKey(locationId)) { + // if the child exists, merge the sample data + RawFrameTree child = children.get(locationId); + child.setTotal(child.getTotal() + 1); + child.setSelf(child.getSelf() + (isEnd ? 1 : 0)); + children = child.getChildren(); + } else { + // if the child does not exist, create a new child + RawFrameTree child = new RawFrameTree(locationId, 1, (isEnd ? 1 : 0)); + children.put(locationId, child); + children = child.getChildren(); + } + } + root.setTotal(root.getTotal() + 1); + } +} \ No newline at end of file diff --git a/oap-server/server-library/library-pprof-parser/src/main/proto/profile.proto b/oap-server/server-library/library-pprof-parser/src/main/proto/profile.proto new file mode 100755 index 000000000000..d1cb0a5a4040 --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/proto/profile.proto @@ -0,0 +1,250 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Profile is a common stacktrace profile format. +// +// Measurements represented with this format should follow the +// following conventions: +// +// - Consumers should treat unset optional fields as if they had been +// set with their default value. +// +// - When possible, measurements should be stored in "unsampled" form +// that is most useful to humans. There should be enough +// information present to determine the original sampled values. +// +// - On-disk, the serialized proto must be gzip-compressed. +// +// - The profile is represented as a set of samples, where each sample +// references a sequence of locations, and where each location belongs +// to a mapping. +// - There is a N->1 relationship from sample.location_id entries to +// locations. For every sample.location_id entry there must be a +// unique Location with that id. +// - There is an optional N->1 relationship from locations to +// mappings. For every nonzero Location.mapping_id there must be a +// unique Mapping with that id. + +syntax = "proto3"; + +package perftools.profiles; + +option java_package = "com.google.perftools.profiles"; +option java_outer_classname = "ProfileProto"; + +message Profile { + // A description of the samples associated with each Sample.value. + // For a cpu profile this might be: + // [["cpu","nanoseconds"]] or [["wall","seconds"]] or [["syscall","count"]] + // For a heap profile, this might be: + // [["allocations","count"], ["space","bytes"]], + // If one of the values represents the number of events represented + // by the sample, by convention it should be at index 0 and use + // sample_type.unit == "count". + repeated ValueType sample_type = 1; + // The set of samples recorded in this profile. + repeated Sample sample = 2; + // Mapping from address ranges to the image/binary/library mapped + // into that address range. mapping[0] will be the main binary. + repeated Mapping mapping = 3; + // Locations referenced by samples. + repeated Location location = 4; + // Functions referenced by locations. + repeated Function function = 5; + // A common table for strings referenced by various messages. + // string_table[0] must always be "". + repeated string string_table = 6; + // frames with Function.function_name fully matching the following + // regexp will be dropped from the samples, along with their successors. + int64 drop_frames = 7; // Index into string table. + // frames with Function.function_name fully matching the following + // regexp will be kept, even if it matches drop_frames. + int64 keep_frames = 8; // Index into string table. + + // The following fields are informational, do not affect + // interpretation of results. + + // Time of collection (UTC) represented as nanoseconds past the epoch. + int64 time_nanos = 9; + // Duration of the profile, if a duration makes sense. + int64 duration_nanos = 10; + // The kind of events between sampled occurrences. + // e.g [ "cpu","cycles" ] or [ "heap","bytes" ] + ValueType period_type = 11; + // The number of events between sampled occurrences. + int64 period = 12; + // Free-form text associated with the profile. The text is displayed as is + // to the user by the tools that read profiles (e.g. by pprof). This field + // should not be used to store any machine-readable information, it is only + // for human-friendly content. The profile must stay functional if this field + // is cleaned. + repeated int64 comment = 13; // Indices into string table. + // Index into the string table of the type of the preferred sample + // value. If unset, clients should default to the last sample value. + int64 default_sample_type = 14; + // Documentation link for this profile. The URL must be absolute, + // e.g., http://pprof.example.com/cpu-profile.html + // + // The URL may be missing if the profile was generated by older code or code + // that did not bother to supply a link. + int64 doc_url = 15; // Index into string table. +} + +// ValueType describes the semantics and measurement units of a value. +message ValueType { + int64 type = 1; // Index into string table. + int64 unit = 2; // Index into string table. +} + +// Each Sample records values encountered in some program +// context. The program context is typically a stack trace, perhaps +// augmented with auxiliary information like the thread-id, some +// indicator of a higher level request being handled etc. +message Sample { + // The ids recorded here correspond to a Profile.location.id. + // The leaf is at location_id[0]. + repeated uint64 location_id = 1; + // The type and unit of each value is defined by the corresponding + // entry in Profile.sample_type. All samples must have the same + // number of values, the same as the length of Profile.sample_type. + // When aggregating multiple samples into a single sample, the + // result has a list of values that is the element-wise sum of the + // lists of the originals. + repeated int64 value = 2; + // label includes additional context for this sample. It can include + // things like a thread id, allocation size, etc. + // + // NOTE: While possible, having multiple values for the same label key is + // strongly discouraged and should never be used. Most tools (e.g. pprof) do + // not have good (or any) support for multi-value labels. And an even more + // discouraged case is having a string label and a numeric label of the same + // name on a sample. Again, possible to express, but should not be used. + repeated Label label = 3; +} + +message Label { + // Index into string table. An annotation for a sample (e.g. + // "allocation_size") with an associated value. + // Keys with "pprof::" prefix are reserved for internal use by pprof. + int64 key = 1; + + // At most one of the following must be present + int64 str = 2; // Index into string table + int64 num = 3; + + // Should only be present when num is present. + // Specifies the units of num. + // Use arbitrary string (for example, "requests") as a custom count unit. + // If no unit is specified, consumer may apply heuristic to deduce the unit. + // Consumers may also interpret units like "bytes" and "kilobytes" as memory + // units and units like "seconds" and "nanoseconds" as time units, + // and apply appropriate unit conversions to these. + int64 num_unit = 4; // Index into string table +} + +message Mapping { + // Unique nonzero id for the mapping. + uint64 id = 1; + // Address at which the binary (or DLL) is loaded into memory. + uint64 memory_start = 2; + // The limit of the address range occupied by this mapping. + uint64 memory_limit = 3; + // Offset in the binary that corresponds to the first mapped address. + uint64 file_offset = 4; + // The object this entry is loaded from. This can be a filename on + // disk for the main binary and shared libraries, or virtual + // abstractions like "[vdso]". + int64 filename = 5; // Index into string table + // A string that uniquely identifies a particular program version + // with high probability. E.g., for binaries generated by GNU tools, + // it could be the contents of the .note.gnu.build-id field. + int64 build_id = 6; // Index into string table + + // The following fields indicate the resolution of symbolic info. + bool has_functions = 7; + bool has_filenames = 8; + bool has_line_numbers = 9; + bool has_inline_frames = 10; +} + +// Describes function and line table debug information. +message Location { + // Unique nonzero id for the location. A profile could use + // instruction addresses or any integer sequence as ids. + uint64 id = 1; + // The id of the corresponding profile.Mapping for this location. + // It can be unset if the mapping is unknown or not applicable for + // this profile type. + uint64 mapping_id = 2; + // The instruction address for this location, if available. It + // should be within [Mapping.memory_start...Mapping.memory_limit] + // for the corresponding mapping. A non-leaf address may be in the + // middle of a call instruction. It is up to display tools to find + // the beginning of the instruction if necessary. + uint64 address = 3; + // Multiple line indicates this location has inlined functions, + // where the last entry represents the caller into which the + // preceding entries were inlined. + // + // E.g., if memcpy() is inlined into printf: + // line[0].function_name == "memcpy" + // line[1].function_name == "printf" + repeated Line line = 4; + // Provides an indication that multiple symbols map to this location's + // address, for example due to identical code folding by the linker. In that + // case the line information above represents one of the multiple + // symbols. This field must be recomputed when the symbolization state of the + // profile changes. + bool is_folded = 5; +} + +message Line { + // The id of the corresponding profile.Function for this line. + uint64 function_id = 1; + // Line number in source code. + int64 line = 2; + // Column number in source code. + int64 column = 3; +} + +message Function { + // Unique nonzero id for the function. + uint64 id = 1; + // Name of the function, in human-readable form if available. + int64 name = 2; // Index into string table + // Name of the function, as identified by the system. + // For instance, it can be a C++ mangled name. + int64 system_name = 3; // Index into string table + // Source file containing the function. + int64 filename = 4; // Index into string table + // Line number in source file. + int64 start_line = 5; +} \ No newline at end of file diff --git a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/CollectionUtils.java b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/CollectionUtils.java index d41bce7f2b49..110a4f341e8b 100644 --- a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/CollectionUtils.java +++ b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/CollectionUtils.java @@ -32,6 +32,10 @@ public static boolean isEmpty(List list) { return list == null || list.size() == 0; } + public static boolean isEmpty(String str) { + return str == null || str.length() == 0; + } + public static boolean isEmpty(Set set) { return set == null || set.size() == 0; } diff --git a/oap-server/server-library/pom.xml b/oap-server/server-library/pom.xml index 3e92ba599bae..93481db0742a 100644 --- a/oap-server/server-library/pom.xml +++ b/oap-server/server-library/pom.xml @@ -36,6 +36,7 @@ library-datacarrier-queue library-kubernetes-support library-async-profiler-jfr-parser + library-pprof-parser library-integration-test diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java index c2f2cfcfde82..a07954aa79d6 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java @@ -47,6 +47,8 @@ import org.apache.skywalking.oap.query.graphql.resolver.OndemandLogQuery; import org.apache.skywalking.oap.query.graphql.resolver.ProfileMutation; import org.apache.skywalking.oap.query.graphql.resolver.ProfileQuery; +import org.apache.skywalking.oap.query.graphql.resolver.PprofMutation; +import org.apache.skywalking.oap.query.graphql.resolver.PprofQuery; import org.apache.skywalking.oap.query.graphql.resolver.Query; import org.apache.skywalking.oap.query.graphql.resolver.RecordsQuery; import org.apache.skywalking.oap.query.graphql.resolver.TopNRecordsQuery; @@ -153,7 +155,9 @@ public void prepare() throws ServiceNotProvidedException { .resolvers(new RecordsQuery(getManager())) .file("query-protocol/hierarchy.graphqls").resolvers(new HierarchyQuery(getManager())) .file("query-protocol/async-profiler.graphqls") - .resolvers(new AsyncProfilerQuery(getManager()), new AsyncProfilerMutation(getManager())); + .resolvers(new AsyncProfilerQuery(getManager()), new AsyncProfilerMutation(getManager())) + .file("query-protocol/pprof.graphqls") + .resolvers(new PprofQuery(getManager()), new PprofMutation(getManager())); if (config.isEnableOnDemandPodLog()) { schemaBuilder diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofMutation.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofMutation.java new file mode 100644 index 000000000000..eb98f8d7825a --- /dev/null +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofMutation.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.query.graphql.resolver; + +import graphql.kickstart.tools.GraphQLMutationResolver; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofMutationService; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskCreationResult; +import org.apache.skywalking.oap.server.core.query.input.PprofTaskCreationRequest; + +import java.io.IOException; + +@Slf4j +public class PprofMutation implements GraphQLMutationResolver { + private final ModuleManager moduleManager; + + private PprofMutationService mutationService; + + public PprofMutation(ModuleManager moduleManager) { + this.moduleManager = moduleManager; + } + + private PprofMutationService getPprofMutationService() { + if (mutationService == null) { + this.mutationService = moduleManager.find(CoreModule.NAME) + .provider() + .getService(PprofMutationService.class); + } + return mutationService; + } + + public PprofTaskCreationResult createPprofTask(PprofTaskCreationRequest request) throws IOException { + PprofMutationService pprofMutationService = getPprofMutationService(); + return pprofMutationService.createTask(request.getServiceId(), request.getServiceInstanceIds(), + request.getDuration(), request.getEvents(), request.getDumpPeriod()); + } +} \ No newline at end of file diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofQuery.java new file mode 100644 index 000000000000..9b1096cfbdec --- /dev/null +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofQuery.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.query.graphql.resolver; + +import org.apache.skywalking.oap.server.core.CoreModule; +import groovy.util.logging.Slf4j; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import graphql.kickstart.tools.GraphQLQueryResolver; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofQueryService; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskListResult; +import org.apache.skywalking.oap.server.core.query.input.PprofTaskListRequest; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskProgress; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.oap.server.core.query.type.PprofAnalyzation; +import org.apache.skywalking.oap.server.core.query.type.PprofStackTree; +import org.apache.skywalking.oap.server.core.query.input.PprofAnalyzationRequest; +import java.util.ArrayList; +import java.io.IOException; +import java.util.List; + +@Slf4j +public class PprofQuery implements GraphQLQueryResolver { + private final ModuleManager moduleManager; + + private PprofQueryService queryService; + + public PprofQuery(ModuleManager moduleManager) { + this.moduleManager = moduleManager; + } + + private PprofQueryService getPprofQueryService() { + if (queryService == null) { + this.queryService = moduleManager.find(CoreModule.NAME).provider().getService(PprofQueryService.class); + } + return queryService; + } + + public PprofTaskListResult queryPprofTaskList(PprofTaskListRequest request) throws IOException { + List tasks = getPprofQueryService().queryTask( + request.getServiceId(), request.getQueryDuration(), request.getLimit() + ); + return new PprofTaskListResult(null, tasks); + } + + public PprofAnalyzation queryPprofAnalyze(PprofAnalyzationRequest request) throws IOException { + PprofStackTree eventFrameTrees = getPprofQueryService().queryPprofData( + request.getTaskId(), request.getInstanceIds() + ); + return new PprofAnalyzation(eventFrameTrees); + } + + public PprofTaskProgress queryPprofTaskProgress(String taskId) throws IOException { + PprofTaskProgress pprofTaskProgress = new PprofTaskProgress(); + List logs = getPprofQueryService().queryPprofTaskLogs(taskId); + pprofTaskProgress.setLogs(logs); + List errorInstances = new ArrayList<>(); + List successInstances = new ArrayList<>(); + logs.forEach(log -> { + if (PprofTaskLogOperationType.EXECUTION_FINISHED.equals(log.getOperationType())) { + successInstances.add(log.getInstanceId()); + } else if (PprofTaskLogOperationType.EXECUTION_TASK_ERROR.equals(log.getOperationType()) + || PprofTaskLogOperationType.PPROF_UPLOAD_FILE_TOO_LARGE_ERROR.equals(log.getOperationType())) { + errorInstances.add(log.getInstanceId()); + } + }); + pprofTaskProgress.setErrorInstanceIds(errorInstances); + pprofTaskProgress.setSuccessInstanceIds(successInstances); + return pprofTaskProgress; + } +} diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol index 97f9bfc9dbf9..003664676984 160000 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol @@ -1 +1 @@ -Subproject commit 97f9bfc9dbf9313a951a5fa3791e92186cfede05 +Subproject commit 0036646769842e915e2828fde0b6c1da0179a1e5 diff --git a/oap-server/server-receiver-plugin/pom.xml b/oap-server/server-receiver-plugin/pom.xml index 17892a71a48e..068a86167c62 100644 --- a/oap-server/server-receiver-plugin/pom.xml +++ b/oap-server/server-receiver-plugin/pom.xml @@ -49,6 +49,7 @@ skywalking-telegraf-receiver-plugin aws-firehose-receiver skywalking-async-profiler-receiver-plugin + skywalking-pprof-receiver-plugin diff --git a/oap-server/server-receiver-plugin/receiver-proto/pom.xml b/oap-server/server-receiver-plugin/receiver-proto/pom.xml index 0c057f133cef..aaff44d8c39a 100644 --- a/oap-server/server-receiver-plugin/receiver-proto/pom.xml +++ b/oap-server/server-receiver-plugin/receiver-proto/pom.xml @@ -41,6 +41,14 @@ flatbuffers-java provided + + com.google.protobuf + protobuf-java + + + com.google.protobuf + protobuf-java-util + diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/pom.xml b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/pom.xml new file mode 100644 index 000000000000..985a5b30702f --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/pom.xml @@ -0,0 +1,53 @@ + + + + + + server-receiver-plugin + org.apache.skywalking + ${revision} + + 4.0.0 + + skywalking-pprof-receiver-plugin + + + + org.apache.skywalking + library-module + ${project.version} + + + org.apache.skywalking + skywalking-sharing-server-plugin + ${project.version} + + + org.apache.skywalking + apm-network + ${project.version} + + + org.apache.skywalking + library-pprof-parser + ${project.version} + + + diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModule.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModule.java new file mode 100644 index 000000000000..f768f1a7ec91 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModule.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.module; + +import org.apache.skywalking.oap.server.library.module.ModuleDefine; + +public class PprofModule extends ModuleDefine { + public static final String NAME = "receiver-pprof"; + + public PprofModule() { + super(NAME); + } + + @Override + public Class[] services() { + return new Class[] {}; + } +} \ No newline at end of file diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java new file mode 100644 index 000000000000..acb92227d020 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.module; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; + +@Getter +@Setter +public class PprofModuleConfig extends ModuleConfig { + /** + * Used to manage the maximum size of the pprof file that can be received, the unit is Byte + * default is 30M + */ + private int pprofMaxSize = 30 * 1024 * 1024; + /** + * default is true + *

+ * If memoryParserEnabled is true, then PprofByteBufCollectionObserver will be enabled + * will use memory to receive pprof files without writing files (this is currently used). + * This can prevent the oap server from crashing due to no volume mounting. + *

+ * If memoryParserEnabled is false, then PprofFileCollectionObserver will be enabled + * which uses createTemp to write files and then reads the files for parsing. + * The advantage of this is that it reduces memory and prevents the oap server from crashing due to + * insufficient memory, but it may report an error due to no volume mounting. + */ + private boolean memoryParserEnabled = true; +} diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/PprofModuleProvider.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/PprofModuleProvider.java new file mode 100644 index 000000000000..9c8d4adb8b20 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/PprofModuleProvider.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.provider; + +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.server.GRPCHandlerRegister; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; +import org.apache.skywalking.oap.server.library.module.ModuleDefine; +import org.apache.skywalking.oap.server.library.module.ModuleProvider; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; +import org.apache.skywalking.oap.server.receiver.pprof.module.PprofModule; +import org.apache.skywalking.oap.server.receiver.pprof.module.PprofModuleConfig; +import org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler; +import org.apache.skywalking.oap.server.receiver.sharing.server.SharingServerModule; + +public class PprofModuleProvider extends ModuleProvider { + private PprofModuleConfig config; + + @Override + public String name() { + return "default"; + } + + @Override + public Class module() { + return PprofModule.class; + } + + @Override + public ConfigCreator newConfigCreator() { + return new ConfigCreator() { + @Override + public Class type() { + return PprofModuleConfig.class; + } + + @Override + public void onInitialized(final PprofModuleConfig initialized) { + config = initialized; + } + }; + } + + @Override + public void prepare() throws ServiceNotProvidedException, ModuleStartException { + + } + + @Override + public void start() throws ServiceNotProvidedException, ModuleStartException { + GRPCHandlerRegister grpcHandlerRegister = getManager().find(SharingServerModule.NAME) + .provider() + .getService(GRPCHandlerRegister.class); + PprofServiceHandler pprofServiceHandler = new PprofServiceHandler( + getManager(), config.getPprofMaxSize(), config.isMemoryParserEnabled()); + grpcHandlerRegister.addHandler(pprofServiceHandler); + } + + @Override + public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException { + } + + @Override + public String[] requiredModules() { + return new String[] { + CoreModule.NAME, + SharingServerModule.NAME + }; + } + +} diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java new file mode 100644 index 000000000000..30773ec2b8d2 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.provider.handler; + +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.apm.network.common.v3.Commands; +import org.apache.skywalking.apm.network.pprof.v10.PprofCollectionResponse; +import org.apache.skywalking.apm.network.pprof.v10.PprofData; +import org.apache.skywalking.apm.network.pprof.v10.PprofMetaData; +import org.apache.skywalking.apm.network.pprof.v10.PprofTaskCommandQuery; +import org.apache.skywalking.apm.network.pprof.v10.PprofTaskGrpc; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.cache.PprofTaskCache; +import org.apache.skywalking.oap.server.core.command.CommandService; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskLogRecord; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.server.grpc.GRPCHandler; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.oap.server.network.trace.component.command.PprofTaskCommand; +import org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream.PprofByteBufCollectionObserver; +import org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream.PprofCollectionMetaData; +import org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream.PprofFileCollectionObserver; + +@Slf4j +public class PprofServiceHandler extends PprofTaskGrpc.PprofTaskImplBase implements GRPCHandler { + + private final IPprofTaskQueryDAO taskDAO; + private final SourceReceiver sourceReceiver; + private final CommandService commandService; + private final PprofTaskCache taskCache; + private final int pprofMaxSize; + private final boolean memoryParserEnabled; + + public PprofServiceHandler(ModuleManager moduleManager, int pprofMaxSize, boolean memoryParserEnabled) { + this.taskDAO = moduleManager.find(StorageModule.NAME).provider().getService(IPprofTaskQueryDAO.class); + this.commandService = moduleManager.find(CoreModule.NAME).provider().getService(CommandService.class); + this.sourceReceiver = moduleManager.find(CoreModule.NAME).provider().getService(SourceReceiver.class); + this.taskCache = moduleManager.find(CoreModule.NAME).provider().getService(PprofTaskCache.class); + this.pprofMaxSize = pprofMaxSize; + this.memoryParserEnabled = memoryParserEnabled; + } + + @Override + public StreamObserver collect(StreamObserver responseObserver) { + return memoryParserEnabled ? new PprofByteBufCollectionObserver( + taskDAO, responseObserver, sourceReceiver, pprofMaxSize) : new PprofFileCollectionObserver( + taskDAO, responseObserver, sourceReceiver, pprofMaxSize); + } + + @Override + public void getPprofTaskCommands(PprofTaskCommandQuery request, StreamObserver responseObserver) { + String serviceId = IDManager.ServiceID.buildId(request.getService(), true); + String serviceInstanceId = IDManager.ServiceInstanceID.buildId(serviceId, request.getServiceInstance()); + List taskList = taskCache.getPprofTaskList(serviceId); + if (CollectionUtils.isEmpty(taskList)) { + responseObserver.onNext(Commands.newBuilder().build()); + responseObserver.onCompleted(); + return; + } + + final long lastCommandTime = request.getLastCommandTime(); + long minCreateTime = Long.MAX_VALUE; + PprofTask taskResult = null; + for (PprofTask task : taskList) { + if (task.getCreateTime() <= lastCommandTime) { + continue; + } + + if (!CollectionUtils.isEmpty(task.getServiceInstanceIds()) + && !task.getServiceInstanceIds().contains(serviceInstanceId)) { + continue; + } + + if (task.getCreateTime() < minCreateTime) { + minCreateTime = task.getCreateTime(); + taskResult = task; + } + } + + if (taskResult == null) { + responseObserver.onNext(Commands.newBuilder().build()); + responseObserver.onCompleted(); + return; + } + + PprofTaskCommand pprofTaskCommand = commandService.newPprofTaskCommand(taskResult); + Commands commands = Commands.newBuilder().addCommands(pprofTaskCommand.serialize()).build(); + responseObserver.onNext(commands); + responseObserver.onCompleted(); + recordPprofTaskLog(taskResult, serviceInstanceId, PprofTaskLogOperationType.NOTIFIED); + } + + public static void recordPprofTaskLog(PprofTask task, String instanceId, PprofTaskLogOperationType operationType) { + PprofTaskLogRecord logRecord = new PprofTaskLogRecord(); + logRecord.setTaskId(task.getId()); + logRecord.setInstanceId(instanceId); + logRecord.setOperationType(operationType.getCode()); + logRecord.setOperationTime(System.currentTimeMillis()); + long timestamp = task.getCreateTime() + TimeUnit.SECONDS.toMillis(task.getDuration()); + logRecord.setTimestamp(timestamp); + logRecord.setTimeBucket(TimeBucket.getRecordTimeBucket(timestamp)); + RecordStreamProcessor.getInstance().in(logRecord); + } + + public static PprofCollectionMetaData parseMetaData(PprofMetaData metaData, + IPprofTaskQueryDAO taskDAO) throws IOException { + String taskId = metaData.getTaskId(); + PprofTask task = taskDAO.getById(taskId); + String serviceId = IDManager.ServiceID.buildId(metaData.getService(), true); + String serviceInstanceId = IDManager.ServiceInstanceID.buildId(serviceId, metaData.getServiceInstance()); + + return PprofCollectionMetaData.builder() + .task(task) + .serviceId(serviceId) + .instanceId(serviceInstanceId) + .type(metaData.getType()) + .contentSize(metaData.getContentSize()) + .uploadTime(System.currentTimeMillis()) + .build(); + } +} \ No newline at end of file diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java new file mode 100644 index 000000000000..245c76811822 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream; + +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.apm.network.pprof.v10.PprofCollectionResponse; +import org.apache.skywalking.apm.network.pprof.v10.PprofData; +import org.apache.skywalking.apm.network.pprof.v10.PprofProfilingStatus; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.oap.server.core.source.PprofProfilingData; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.pprof.parser.PprofParser; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; + +import static org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler.parseMetaData; +import static org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler.recordPprofTaskLog; + +@Slf4j +public class PprofByteBufCollectionObserver implements StreamObserver { + private final IPprofTaskQueryDAO taskDAO; + private final StreamObserver responseObserver; + private final SourceReceiver sourceReceiver; + private final int pprofMaxSize; + private PprofCollectionMetaData taskMetaData; + private ByteBuffer buf; + + public PprofByteBufCollectionObserver(IPprofTaskQueryDAO taskDAO, + StreamObserver responseObserver, + SourceReceiver sourceReceiver, int pprofMaxSize) { + this.taskDAO = taskDAO; + this.responseObserver = responseObserver; + this.sourceReceiver = sourceReceiver; + this.pprofMaxSize = pprofMaxSize; + } + + @Override + public void onNext(PprofData pprofData) { + try { + if (Objects.isNull(taskMetaData) && pprofData.hasMetadata()) { + taskMetaData = parseMetaData(pprofData.getMetadata(), taskDAO); + if (PprofProfilingStatus.PPROF_PROFILING_SUCCESS.equals(taskMetaData.getType())) { + int size = taskMetaData.getContentSize(); + if (pprofMaxSize >= size) { + buf = ByteBuffer.allocate(size); + // Send success response to allow client to continue uploading + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus( + PprofProfilingStatus.PPROF_PROFILING_SUCCESS) + .build()); + } else { + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus( + PprofProfilingStatus.PPROF_TERMINATED_BY_OVERSIZE) + .build()); + recordPprofTaskLog( + taskMetaData.getTask(), taskMetaData.getInstanceId(), + PprofTaskLogOperationType.PPROF_UPLOAD_FILE_TOO_LARGE_ERROR + ); + } + } else { + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus( + PprofProfilingStatus.PPROF_EXECUTION_TASK_ERROR) + .build()); + recordPprofTaskLog( + taskMetaData.getTask(), taskMetaData.getInstanceId(), + PprofTaskLogOperationType.EXECUTION_TASK_ERROR + ); + } + } else if (pprofData.hasContent()) { + if (buf != null) { + pprofData.getContent().copyTo(buf); + } + } + } catch (IOException e) { + log.error("Error processing pprof data", e); + responseObserver.onError( + Status.INTERNAL.withDescription("Error processing pprof data: " + e.getMessage()).asRuntimeException()); + } + } + + @Override + public void onError(Throwable throwable) { + Status status = Status.fromThrowable(throwable); + if (Status.CANCELLED.getCode() == status.getCode()) { + if (log.isDebugEnabled()) { + log.debug(throwable.getMessage(), throwable); + } + return; + } + log.error("Error in receiving pprof profiling data", throwable); + + } + + @Override + public void onCompleted() { + responseObserver.onCompleted(); + if (Objects.nonNull(buf)) { + buf.flip(); + try { + parseAndStorageData(taskMetaData, buf); + } catch (IOException e) { + log.error("Failed to parse and store pprof data", e); + } + } + } + + private void parseAndStorageData(PprofCollectionMetaData taskMetaData, ByteBuffer buf) throws IOException { + PprofTask task = taskMetaData.getTask(); + if (task == null) { + log.error( + "Pprof instanceId:{} has not been assigned a task but still uploaded data", + taskMetaData.getInstanceId() + ); + return; + } + recordPprofTaskLog(task, taskMetaData.getInstanceId(), PprofTaskLogOperationType.EXECUTION_FINISHED); + parsePprofAndStorage(taskMetaData, buf); + } + + public void parsePprofAndStorage(PprofCollectionMetaData taskMetaData, ByteBuffer buf) throws IOException { + PprofTask task = taskMetaData.getTask(); + FrameTree tree = PprofParser.dumpTree(buf); + PprofProfilingData data = new PprofProfilingData(); + data.setEventType(PprofEventType.valueOfString(task.getEvents().name())); + data.setFrameTree(tree); + data.setTaskId(task.getId()); + data.setInstanceId(taskMetaData.getInstanceId()); + data.setUploadTime(taskMetaData.getUploadTime()); + sourceReceiver.receive(data); + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofCollectionMetaData.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofCollectionMetaData.java new file mode 100644 index 000000000000..8a5c0680ffda --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofCollectionMetaData.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream; + +import lombok.Builder; +import lombok.Data; +import org.apache.skywalking.apm.network.pprof.v10.PprofProfilingStatus; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; + +@Data +@Builder +public class PprofCollectionMetaData { + private PprofTask task; + private String serviceId; + private String instanceId; + private int contentSize; + private PprofProfilingStatus type; + private long uploadTime; +} diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java new file mode 100644 index 000000000000..f197d1af6f08 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream; + +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.apm.network.pprof.v10.PprofCollectionResponse; +import org.apache.skywalking.apm.network.pprof.v10.PprofData; +import org.apache.skywalking.apm.network.pprof.v10.PprofProfilingStatus; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.oap.server.core.source.PprofProfilingData; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.pprof.parser.PprofParser; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; + +import static org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler.parseMetaData; +import static org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler.recordPprofTaskLog; + +@Slf4j +public class PprofFileCollectionObserver implements StreamObserver { + private final IPprofTaskQueryDAO taskDAO; + private final StreamObserver responseObserver; + private final SourceReceiver sourceReceiver; + private final int pprofMaxSize; + private PprofCollectionMetaData taskMetaData; + private Path tempFile; + private FileOutputStream fileOutputStream; + + public PprofFileCollectionObserver(IPprofTaskQueryDAO taskDAO, + StreamObserver responseObserver, + SourceReceiver sourceReceiver, int pprofMaxSize) { + this.taskDAO = taskDAO; + this.responseObserver = responseObserver; + this.sourceReceiver = sourceReceiver; + this.pprofMaxSize = pprofMaxSize; + } + + @SneakyThrows + @Override + public void onNext(PprofData pprofData) { + if (Objects.isNull(taskMetaData) && pprofData.hasMetadata()) { + taskMetaData = parseMetaData(pprofData.getMetadata(), taskDAO); + + if (PprofProfilingStatus.PPROF_PROFILING_SUCCESS.equals(taskMetaData.getType())) { + int size = taskMetaData.getContentSize(); + if (pprofMaxSize >= size) { + // Create temporary file for pprof data + tempFile = Files.createTempFile( + taskMetaData.getTask().getId() + taskMetaData.getInstanceId() + System.currentTimeMillis(), + ".pprof" + ); + fileOutputStream = new FileOutputStream(tempFile.toFile()); + + // Send success response to allow client to continue uploading + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus( + PprofProfilingStatus.PPROF_PROFILING_SUCCESS) + .build()); + } else { + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus( + PprofProfilingStatus.PPROF_TERMINATED_BY_OVERSIZE) + .build()); + recordPprofTaskLog( + taskMetaData.getTask(), taskMetaData.getInstanceId(), + PprofTaskLogOperationType.PPROF_UPLOAD_FILE_TOO_LARGE_ERROR + ); + } + } else { + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus( + PprofProfilingStatus.PPROF_EXECUTION_TASK_ERROR) + .build()); + recordPprofTaskLog( + taskMetaData.getTask(), taskMetaData.getInstanceId(), + PprofTaskLogOperationType.EXECUTION_TASK_ERROR + ); + } + } else if (pprofData.hasContent()) { + if (fileOutputStream != null) { + fileOutputStream.write(pprofData.getContent().toByteArray()); + + if (log.isDebugEnabled()) { + log.debug("Received {} bytes of pprof data", pprofData.getContent().size()); + } + } + } + } + + @Override + public void onError(Throwable throwable) { + Status status = Status.fromThrowable(throwable); + if (Status.CANCELLED.getCode() == status.getCode()) { + if (log.isDebugEnabled()) { + log.debug("Pprof data collection cancelled: {}", throwable.getMessage()); + } + } else { + log.error("Error in receiving pprof profiling data", throwable); + } + + // Clean up resources + closeFileStream(); + } + + @Override + @SneakyThrows + public void onCompleted() { + responseObserver.onCompleted(); + + if (Objects.nonNull(tempFile)) { + closeFileStream(); + parseAndStorageData(taskMetaData, tempFile.toAbsolutePath().toString()); + } + } + + private void closeFileStream() { + if (fileOutputStream != null) { + try { + fileOutputStream.close(); + fileOutputStream = null; + } catch (IOException e) { + log.error("Failed to close file output stream", e); + } + } + } + + @SneakyThrows + private void parseAndStorageData(PprofCollectionMetaData taskMetaData, String fileName) { + PprofTask task = taskMetaData.getTask(); + if (task == null) { + log.error( + "Pprof instanceId:{} has not been assigned a task but still uploaded data", + taskMetaData.getInstanceId() + ); + return; + } + recordPprofTaskLog(task, taskMetaData.getInstanceId(), PprofTaskLogOperationType.EXECUTION_FINISHED); + parsePprofAndStorage(taskMetaData, fileName); + } + + public void parsePprofAndStorage(PprofCollectionMetaData taskMetaData, + String fileName) throws IOException { + PprofTask task = taskMetaData.getTask(); + FrameTree tree = PprofParser.dumpTree(fileName); + PprofProfilingData data = new PprofProfilingData(); + data.setEventType(PprofEventType.valueOfString(task.getEvents().name())); + data.setFrameTree(tree); + data.setTaskId(task.getId()); + data.setInstanceId(taskMetaData.getInstanceId()); + data.setUploadTime(taskMetaData.getUploadTime()); + sourceReceiver.receive(data); + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine new file mode 100644 index 000000000000..3ef9c8a560ef --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.oap.server.receiver.pprof.module.PprofModule diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider new file mode 100644 index 000000000000..e4e90929ba62 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.oap.server.receiver.pprof.provider.PprofModuleProvider diff --git a/oap-server/server-receiver-plugin/skywalking-sharing-server-plugin/pom.xml b/oap-server/server-receiver-plugin/skywalking-sharing-server-plugin/pom.xml index 5cf914ba25c2..19b45143d0ea 100644 --- a/oap-server/server-receiver-plugin/skywalking-sharing-server-plugin/pom.xml +++ b/oap-server/server-receiver-plugin/skywalking-sharing-server-plugin/pom.xml @@ -27,4 +27,12 @@ skywalking-sharing-server-plugin jar + + + + org.apache.skywalking + library-server + ${project.version} + + \ No newline at end of file diff --git a/oap-server/server-starter/pom.xml b/oap-server/server-starter/pom.xml index d1971572aa90..7d44453471ae 100644 --- a/oap-server/server-starter/pom.xml +++ b/oap-server/server-starter/pom.xml @@ -176,6 +176,11 @@ skywalking-async-profiler-receiver-plugin ${project.version} + + org.apache.skywalking + skywalking-pprof-receiver-plugin + ${project.version} + org.apache.skywalking skywalking-telegraf-receiver-plugin diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index 281321788e3f..7f082649425d 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -183,6 +183,7 @@ storage: segmentQueryMaxSize: ${SW_STORAGE_ES_QUERY_SEGMENT_SIZE:200} profileTaskQueryMaxSize: ${SW_STORAGE_ES_QUERY_PROFILE_TASK_SIZE:200} asyncProfilerTaskQueryMaxSize: ${SW_STORAGE_ES_QUERY_ASYNC_PROFILER_TASK_SIZE:200} + pprofTaskQueryMaxSize: ${SW_STORAGE_ES_QUERY_PPROF_TASK_SIZE:200} profileDataQueryBatchSize: ${SW_STORAGE_ES_QUERY_PROFILE_DATA_BATCH_SIZE:100} oapAnalyzer: ${SW_STORAGE_ES_OAP_ANALYZER:"{\"analyzer\":{\"oap_analyzer\":{\"type\":\"stop\"}}}"} # the oap analyzer. oapLogAnalyzer: ${SW_STORAGE_ES_OAP_LOG_ANALYZER:"{\"analyzer\":{\"oap_log_analyzer\":{\"type\":\"standard\"}}}"} # the oap log analyzer. It could be customized by the ES analyzer configuration to support more language log formats, such as Chinese log, Japanese log and etc. @@ -296,6 +297,20 @@ receiver-async-profiler: # It is recommended to use physical file mode when volume mounting is used or the tmp directory has sufficient storage. memoryParserEnabled: ${SW_RECEIVER_ASYNC_PROFILER_MEMORY_PARSER_ENABLED:true} +receiver-pprof: + selector: ${SW_RECEIVER_PPROF:default} + default: + # Used to manage the maximum size of the pprof file that can be received, the unit is Byte, default is 30M + pprofMaxSize: ${SW_RECEIVER_PPROF_MAX_SIZE:31457280} + # Used to determine whether to receive pprof in memory file or physical file mode + # + # The memory file mode have fewer local file system limitations, so they are by default. But it costs more memory. + # + # The physical file mode will use less memory when parsing and is more friendly to parsing large files. + # However, if the storage of the tmp directory in the container is insufficient, the oap server instance may crash. + # It is recommended to use physical file mode when volume mounting is used or the tmp directory has sufficient storage. + memoryParserEnabled: ${SW_RECEIVER_PPROF_MEMORY_PARSER_ENABLED:true} + receiver-zabbix: selector: ${SW_RECEIVER_ZABBIX:-} default: diff --git a/oap-server/server-starter/src/main/resources/bydb.yml b/oap-server/server-starter/src/main/resources/bydb.yml index 1260ffa34d0f..496d6314c543 100644 --- a/oap-server/server-starter/src/main/resources/bydb.yml +++ b/oap-server/server-starter/src/main/resources/bydb.yml @@ -43,6 +43,7 @@ global: # The batch size for querying profile data. profileDataQueryBatchSize: ${SW_STORAGE_BANYANDB_QUERY_PROFILE_DATA_BATCH_SIZE:100} asyncProfilerTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_ASYNC_PROFILER_TASK_QUERY_MAX_SIZE:200} + pprofTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_PPROF_TASK_QUERY_MAX_SIZE:200} user: ${SW_STORAGE_BANYANDB_USER:""} password: ${SW_STORAGE_BANYANDB_PASSWORD:""} # If the BanyanDB server is configured with TLS, configure the TLS cert file path and enable TLS connection. diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java index 41d9bc2c412c..97da501178ce 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java @@ -100,7 +100,12 @@ public static class Global { * single request. */ private int asyncProfilerTaskQueryMaxSize; - + /** + * Max size of {@link org.apache.skywalking.oap.server.core.query.type.PprofTask} to be fetched in a + * single request. + */ + private int pprofTaskQueryMaxSize; + private int resultWindowMaxSize = 10000; private int metadataQueryMaxSize = 5000; private int segmentQueryMaxSize = 200; diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java index 5322d9305a07..47e81cfc1d37 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java @@ -40,6 +40,9 @@ import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IJFRDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.continuous.IContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingDataDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingScheduleDAO; @@ -81,6 +84,9 @@ import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBAlarmQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBAsyncProfilerTaskQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBPprofDataQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBPprofTaskQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBBrowserLogQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBEBPFProfilingDataDAO; @@ -203,6 +209,17 @@ IAsyncProfilerTaskLogQueryDAO.class, new BanyanDBAsyncProfilerTaskLogQueryDAO(cl this.config.getGlobal().getAsyncProfilerTaskQueryMaxSize() )); this.registerServiceImplementation(IJFRDataQueryDAO.class, new BanyanDBJFRDataQueryDAO(client)); + this.registerServiceImplementation( + IPprofTaskQueryDAO.class, new BanyanDBPprofTaskQueryDAO(client, + this.config.getGlobal().getPprofTaskQueryMaxSize() + )); + this.registerServiceImplementation( + IPprofTaskLogQueryDAO.class, new BanyanDBPprofTaskLogQueryDAO(client, + this.config.getGlobal().getPprofTaskQueryMaxSize() + )); + this.registerServiceImplementation( + IPprofDataQueryDAO.class, new BanyanDBPprofDataQueryDAO(client) + ); this.registerServiceImplementation( StorageTTLStatusQuery.class, new BanyanDBTTLStatusQuery(config) diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java new file mode 100644 index 000000000000..3cd01466165c --- /dev/null +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream; + +import com.google.common.collect.ImmutableSet; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import org.apache.skywalking.banyandb.v1.client.RowEntity; +import org.apache.skywalking.banyandb.v1.client.StreamQuery; +import org.apache.skywalking.banyandb.v1.client.StreamQueryResponse; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBConverter; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient; + +public class BanyanDBPprofDataQueryDAO extends AbstractBanyanDBDAO implements IPprofDataQueryDAO { + private static final Set TAGS = ImmutableSet.of( + PprofProfilingDataRecord.TASK_ID, + PprofProfilingDataRecord.INSTANCE_ID, + PprofProfilingDataRecord.UPLOAD_TIME, + PprofProfilingDataRecord.DATA_BINARY + ); + + public BanyanDBPprofDataQueryDAO(BanyanDBStorageClient client) { + super(client); + } + + @Override + public List getByTaskIdAndInstances(String taskId, + List instanceIds) throws IOException { + if (StringUtil.isBlank(taskId)) { + return new ArrayList<>(); + } + StreamQueryResponse resp = query( + false, PprofProfilingDataRecord.INDEX_NAME, TAGS, + new QueryBuilder() { + @Override + protected void apply(StreamQuery query) { + query.and(eq(PprofProfilingDataRecord.TASK_ID, taskId)); + if (CollectionUtils.isNotEmpty(instanceIds)) { + query.and(in(PprofProfilingDataRecord.INSTANCE_ID, instanceIds)); + } + } + } + ); + List records = new ArrayList<>(resp.size()); + for (final RowEntity entity : resp.getElements()) { + records.add(buildProfilingDataRecord(entity)); + } + + return records; + } + + private PprofProfilingDataRecord buildProfilingDataRecord(RowEntity entity) { + final PprofProfilingDataRecord.Builder builder = new PprofProfilingDataRecord.Builder(); + BanyanDBConverter.StorageToStream storageToStream = new BanyanDBConverter.StorageToStream( + PprofProfilingDataRecord.INDEX_NAME, entity); + return builder.storage2Entity(storageToStream); + } +} diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskLogQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskLogQueryDAO.java new file mode 100644 index 000000000000..0f82af2f267c --- /dev/null +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskLogQueryDAO.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream; + +import com.google.common.collect.ImmutableSet; +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import org.apache.skywalking.banyandb.v1.client.Element; +import org.apache.skywalking.banyandb.v1.client.StreamQuery; +import org.apache.skywalking.banyandb.v1.client.StreamQueryResponse; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskLogRecord; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient; + +/** + * {@link PprofTaskLogRecord} is a stream + */ +public class BanyanDBPprofTaskLogQueryDAO extends AbstractBanyanDBDAO implements IPprofTaskLogQueryDAO { + private static final Set TAGS = ImmutableSet.of( + PprofTaskLogRecord.OPERATION_TIME, + PprofTaskLogRecord.INSTANCE_ID, + PprofTaskLogRecord.TASK_ID, + PprofTaskLogRecord.OPERATION_TYPE + ); + + private final int queryMaxSize; + + public BanyanDBPprofTaskLogQueryDAO(BanyanDBStorageClient client, int taskQueryMaxSize) { + super(client); + // query log size use pprof task query max size * per log count + this.queryMaxSize = taskQueryMaxSize * 50; + } + + @Override + public List getTaskLogList() throws IOException { + StreamQueryResponse resp = query( + false, PprofTaskLogRecord.INDEX_NAME, TAGS, + new QueryBuilder() { + @Override + public void apply(StreamQuery query) { + query.setLimit(BanyanDBPprofTaskLogQueryDAO.this.queryMaxSize); + } + } + ); + + final LinkedList tasks = new LinkedList<>(); + for (final Element element : resp.getElements()) { + tasks.add(buildPprofTaskLog(element)); + } + return tasks; + } + + private PprofTaskLog buildPprofTaskLog(Element data) { + int operationTypeInt = ((Number) data.getTagValue(PprofTaskLogRecord.OPERATION_TYPE)).intValue(); + PprofTaskLogOperationType operationType = PprofTaskLogOperationType.parse(operationTypeInt); + return PprofTaskLog.builder() + .id(data.getTagValue(PprofTaskLogRecord.TASK_ID)) + .instanceId(data.getTagValue(PprofTaskLogRecord.INSTANCE_ID)) + .operationType(operationType) + .operationTime(((Number) data.getTagValue(PprofTaskLogRecord.OPERATION_TIME)).longValue()) + .build(); + } +} diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java new file mode 100644 index 000000000000..6a2332faa345 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream; + +import com.google.common.collect.ImmutableSet; +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.banyandb.v1.client.AbstractQuery; +import org.apache.skywalking.banyandb.v1.client.RowEntity; +import org.apache.skywalking.banyandb.v1.client.StreamQuery; +import org.apache.skywalking.banyandb.v1.client.StreamQueryResponse; +import org.apache.skywalking.banyandb.v1.client.TimestampRange; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient; + +@Slf4j +public class BanyanDBPprofTaskQueryDAO extends AbstractBanyanDBDAO implements IPprofTaskQueryDAO { + private static final Gson GSON = new Gson(); + private static final Set TAGS = ImmutableSet.of( + PprofTaskRecord.SERVICE_ID, + PprofTaskRecord.SERVICE_INSTANCE_IDS, + PprofTaskRecord.TASK_ID, + PprofTaskRecord.CREATE_TIME, + PprofTaskRecord.EVENT_TYPES, + PprofTaskRecord.DURATION, + PprofTaskRecord.DUMP_PERIOD + ); + + private final int queryMaxSize; + + public BanyanDBPprofTaskQueryDAO(BanyanDBStorageClient client, int queryMaxSize) { + super(client); + this.queryMaxSize = queryMaxSize; + } + + @Override + public List getTaskList(String serviceId, Long startTimeBucket, Long endTimeBucket, Integer limit) throws IOException { + long startTS = LOWER_BOUND_TIME; + long endTS = UPPER_BOUND_TIME; + if (startTimeBucket != null) { + startTS = TimeBucket.getTimestamp(startTimeBucket); + } + if (endTimeBucket != null) { + endTS = TimeBucket.getTimestamp(endTimeBucket); + } + StreamQueryResponse resp = query(false, PprofTaskRecord.INDEX_NAME, TAGS, new TimestampRange(startTS, endTS), + new QueryBuilder() { + @Override + protected void apply(StreamQuery query) { + if (StringUtil.isNotEmpty(serviceId)) { + query.and(eq(PprofTaskRecord.SERVICE_ID, serviceId)); + } + + if (limit != null) { + query.setLimit(limit); + } else { + query.setLimit(BanyanDBPprofTaskQueryDAO.this.queryMaxSize); + } + query.setOrderBy(new AbstractQuery.OrderBy(AbstractQuery.Sort.DESC)); + } + }); + + List tasks = new ArrayList<>(resp.size()); + for (final RowEntity entity : resp.getElements()) { + tasks.add(buildPprofTask(entity)); + } + return tasks; + } + + @Override + public PprofTask getById(String id) throws IOException { + StreamQueryResponse resp = query(false, PprofTaskRecord.INDEX_NAME, TAGS, + new QueryBuilder() { + @Override + protected void apply(StreamQuery query) { + if (StringUtil.isNotEmpty(id)) { + query.and(eq(PprofTaskRecord.TASK_ID, id)); + } + query.setLimit(1); + } + }); + + if (resp.size() == 0) { + return null; + } + + return buildPprofTask(resp.getElements().get(0)); + } + + private PprofTask buildPprofTask(RowEntity data) { + Type listType = new TypeToken>() { + }.getType(); + + String serviceInstanceIds = data.getTagValue(PprofTaskRecord.SERVICE_INSTANCE_IDS); + List serviceInstanceIdList = GSON.fromJson(serviceInstanceIds, listType); + + // Convert string events to PprofEventType enum + String eventsStr = data.getTagValue(PprofTaskRecord.EVENT_TYPES); + PprofEventType eventType = null; + if (StringUtil.isNotEmpty(eventsStr)) { + try { + eventType = PprofEventType.valueOfString(eventsStr); + } catch (Exception e) { + // Default to CPU if conversion fails + eventType = PprofEventType.CPU; + log.warn("Failed to parse pprof event type: {}, using CPU as default", eventsStr, e); + } + } + + return PprofTask.builder() + .id(data.getTagValue(PprofTaskRecord.TASK_ID)) + .serviceId(data.getTagValue(PprofTaskRecord.SERVICE_ID)) + .serviceInstanceIds(serviceInstanceIdList) + .createTime(((Number) data.getTagValue(PprofTaskRecord.CREATE_TIME)).longValue()) + .events(eventType) + .duration(((Number) data.getTagValue(PprofTaskRecord.DURATION)).intValue()) + .dumpPeriod(((Number) data.getTagValue(PprofTaskRecord.DUMP_PERIOD)).intValue()) + .build(); + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java index b275acd65014..d8dc848be5c3 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java @@ -167,4 +167,9 @@ public class StorageModuleElasticsearchConfig extends ModuleConfig { * in a single request. */ private int asyncProfilerTaskQueryMaxSize; + /** + * Max size of {@link org.apache.skywalking.oap.server.core.query.type.PprofTask} to be fetched + * in a single request. + */ + private int pprofTaskQueryMaxSize; } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java index c5dc8d018ef6..3cfd5e014c6b 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java @@ -39,6 +39,9 @@ import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IJFRDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.continuous.IContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingDataDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingScheduleDAO; @@ -81,6 +84,7 @@ import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.AlarmQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.AsyncProfilerTaskLogQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.AsyncProfilerTaskQueryEsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.PprofTaskLogQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.BrowserLogQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ContinuousProfilingPolicyEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.EBPFProfilingDataEsDAO; @@ -89,12 +93,14 @@ import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ESEventQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.HierarchyQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.JFRDataQueryEsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.PprofDataQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.LogQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.MetadataQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.MetricsQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ProfileTaskLogEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ProfileTaskQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ProfileThreadSnapshotQueryEsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.PprofTaskQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.RecordsQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ServiceLabelEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.SpanAttachedEventEsDAO; @@ -276,6 +282,18 @@ IProfileThreadSnapshotQueryDAO.class, new ProfileThreadSnapshotQueryEsDAO(elasti IJFRDataQueryDAO.class, new JFRDataQueryEsDAO(elasticSearchClient) ); + this.registerServiceImplementation( + IPprofTaskQueryDAO.class, + new PprofTaskQueryEsDAO(elasticSearchClient, config.getPprofTaskQueryMaxSize()) + ); + this.registerServiceImplementation( + IPprofTaskLogQueryDAO.class, + new PprofTaskLogQueryEsDAO(elasticSearchClient, config.getPprofTaskQueryMaxSize()) + ); + this.registerServiceImplementation( + IPprofDataQueryDAO.class, + new PprofDataQueryEsDAO(elasticSearchClient) + ); this.registerServiceImplementation( StorageTTLStatusQuery.class, new DefaultStorageTTLStatusQuery() diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java new file mode 100644 index 000000000000..77ea0c55bb7d --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import com.google.common.collect.Lists; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Query; +import org.apache.skywalking.library.elasticsearch.requests.search.Search; +import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; +import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; +import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.ElasticSearchConverter; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; + +public class PprofDataQueryEsDAO extends EsDAO implements IPprofDataQueryDAO { + public PprofDataQueryEsDAO(ElasticSearchClient client) { + super(client); + } + + @Override + public List getByTaskIdAndInstances(String taskId, List instanceIds) { + if (StringUtil.isBlank(taskId) || CollectionUtils.isEmpty(instanceIds)) { + return new ArrayList<>(); + } + final String index = IndexController.LogicIndicesRegister.getPhysicalTableName( + PprofProfilingDataRecord.INDEX_NAME); + final BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isMergedTable(PprofProfilingDataRecord.INDEX_NAME)) { + query.must(Query.term( + IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, + PprofProfilingDataRecord.INDEX_NAME + )); + } + query.must(Query.term(PprofProfilingDataRecord.TASK_ID, taskId)); + query.must(Query.terms(PprofProfilingDataRecord.INSTANCE_ID, instanceIds)); + final SearchBuilder search = Search.builder().query(query); + final SearchResponse response = getClient().search(index, search.build()); + List dataRecords = Lists.newArrayList(); + for (SearchHit searchHit : response.getHits().getHits()) { + dataRecords.add(parseData(searchHit)); + } + return dataRecords; + } + + private PprofProfilingDataRecord parseData(SearchHit data) { + final Map sourceAsMap = data.getSource(); + final PprofProfilingDataRecord.Builder builder = new PprofProfilingDataRecord.Builder(); + return builder.storage2Entity( + new ElasticSearchConverter.ToEntity(PprofProfilingDataRecord.INDEX_NAME, sourceAsMap)); + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java new file mode 100644 index 000000000000..36e3f18f18a9 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Query; +import org.apache.skywalking.library.elasticsearch.requests.search.Search; +import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Sort; +import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; +import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskLogRecord; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; + +public class PprofTaskLogQueryEsDAO extends EsDAO implements IPprofTaskLogQueryDAO { + + private final int queryMaxSize; + + public PprofTaskLogQueryEsDAO(ElasticSearchClient client, int profileTaskQueryMaxSize) { + super(client); + // query log size use pprof task query max size * per log count + this.queryMaxSize = profileTaskQueryMaxSize * 50; + } + + @Override + public List getTaskLogList() throws IOException { + final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofTaskLogRecord.INDEX_NAME); + final BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isMergedTable(PprofTaskLogRecord.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofTaskLogRecord.INDEX_NAME)); + } + + final SearchBuilder search = + Search.builder().query(query) + .sort(PprofTaskLogRecord.OPERATION_TIME, Sort.Order.DESC) + .size(queryMaxSize); + final SearchResponse response = getClient().search(index, search.build()); + + List tasks = new LinkedList<>(); + for (SearchHit searchHit : response.getHits().getHits()) { + tasks.add(buildPprofTaskLog(searchHit)); + } + return tasks; + } + + private PprofTaskLog buildPprofTaskLog(SearchHit data) { + Map source = data.getSource(); + int operationTypeInt = ((Number) source.get(PprofTaskLogRecord.OPERATION_TYPE)).intValue(); + PprofTaskLogOperationType operationType = PprofTaskLogOperationType.parse(operationTypeInt); + return PprofTaskLog.builder() + .id((String) source.get(PprofTaskLogRecord.TASK_ID)) + .instanceId((String) source.get(PprofTaskLogRecord.INSTANCE_ID)) + .operationType(operationType) + .operationTime(((Number) source.get(PprofTaskLogRecord.OPERATION_TIME)).longValue()) + .build(); + } +} \ No newline at end of file diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java new file mode 100644 index 000000000000..719cd064d930 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Query; +import org.apache.skywalking.library.elasticsearch.requests.search.Search; +import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Sort; +import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; +import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; + +public class PprofTaskQueryEsDAO extends EsDAO implements IPprofTaskQueryDAO { + private static final Gson GSON = new Gson(); + + private final int queryMaxSize; + + public PprofTaskQueryEsDAO(ElasticSearchClient client, int queryMaxSize) { + super(client); + this.queryMaxSize = queryMaxSize; + } + + @Override + public List getTaskList(String serviceId, + Long startTimeBucket, + Long endTimeBucket, + Integer limit) throws IOException { + String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofTaskRecord.INDEX_NAME); + BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isMergedTable(PprofTaskRecord.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofTaskRecord.INDEX_NAME)); + } + + if (StringUtil.isNotEmpty(serviceId)) { + query.must(Query.term(PprofTaskRecord.SERVICE_ID, serviceId)); + } + + if (startTimeBucket != null) { + query.must(Query.range(PprofTaskRecord.TIME_BUCKET).gte(startTimeBucket)); + } + + if (endTimeBucket != null) { + query.must(Query.range(PprofTaskRecord.TIME_BUCKET).lte(endTimeBucket)); + } + SearchBuilder search = Search.builder().query(query); + search.size(Objects.requireNonNullElse(limit, queryMaxSize)); + + search.sort(PprofTaskRecord.CREATE_TIME, Sort.Order.DESC); + + final SearchResponse response = getClient().search(index, search.build()); + + List tasks = new LinkedList<>(); + for (SearchHit hit : response.getHits().getHits()) { + tasks.add(parseTask(hit)); + } + return tasks; + } + + @Override + public PprofTask getById(String id) { + if (StringUtil.isEmpty(id)) { + return null; + } + final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofTaskRecord.INDEX_NAME); + final BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isMergedTable(PprofTaskRecord.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofTaskRecord.INDEX_NAME)); + } + query.must(Query.term(PprofTaskRecord.TASK_ID, id)); + + final SearchBuilder search = Search.builder().query(query).size(1); + final SearchResponse response = getClient().search(index, search.build()); + + if (!response.getHits().getHits().isEmpty()) { + return parseTask(response.getHits().getHits().iterator().next()); + } + return null; + } + + private PprofTask parseTask(SearchHit data) { + Map source = data.getSource(); + Type listType = new TypeToken>() { + }.getType(); + + String serviceInstanceIds = (String) source.get(PprofTaskRecord.SERVICE_INSTANCE_IDS); + + List instanceIdList = GSON.fromJson(serviceInstanceIds, listType); + return PprofTask.builder() + .id((String) source.get(PprofTaskRecord.TASK_ID)) + .serviceId((String) source.get(PprofTaskRecord.SERVICE_ID)) + .serviceInstanceIds(instanceIdList) + .createTime(((Number) source.get(PprofTaskRecord.CREATE_TIME)).longValue()) + .events(PprofEventType.valueOfString((String) source.get(PprofTaskRecord.EVENT_TYPES))) + .duration(((Number) source.get(PprofTaskRecord.DURATION)).intValue()) + .dumpPeriod(((Number) source.get(PprofTaskRecord.DUMP_PERIOD)).intValue()) + .build(); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCStorageProvider.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCStorageProvider.java index 2fd5cbcb4927..7891c1c7b122 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCStorageProvider.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCStorageProvider.java @@ -33,6 +33,9 @@ import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IJFRDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.continuous.IContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingDataDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingScheduleDAO; @@ -66,6 +69,9 @@ import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCAlarmQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCAsyncProfilerTaskQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCPprofDataQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCPprofTaskQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCBatchDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCBrowserLogQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCContinuousProfilingPolicyDAO; @@ -255,6 +261,18 @@ public void prepare() throws ServiceNotProvidedException, ModuleStartException { IJFRDataQueryDAO.class, new JDBCJFRDataQueryDAO(jdbcClient, tableHelper) ); + this.registerServiceImplementation( + IPprofTaskQueryDAO.class, + new JDBCPprofTaskQueryDAO(jdbcClient, tableHelper) + ); + this.registerServiceImplementation( + IPprofTaskLogQueryDAO.class, + new JDBCPprofTaskLogQueryDAO(jdbcClient, tableHelper) + ); + this.registerServiceImplementation( + IPprofDataQueryDAO.class, + new JDBCPprofDataQueryDAO(jdbcClient, tableHelper) + ); this.registerServiceImplementation( StorageTTLStatusQuery.class, new DefaultStorageTTLStatusQuery() diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java new file mode 100644 index 000000000000..f391f050f5cf --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import java.io.IOException; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; +import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCClient; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCEntityConverters; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCTableInstaller; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.TableHelper; + +@RequiredArgsConstructor +public class JDBCPprofDataQueryDAO implements IPprofDataQueryDAO { + private final JDBCClient jdbcClient; + private final TableHelper tableHelper; + + @Override + @SneakyThrows + public List getByTaskIdAndInstances(String taskId, + List instanceIds) throws IOException { + if (StringUtil.isBlank(taskId)) { + return new ArrayList<>(); + } + List tables = tableHelper.getTablesWithinTTL(PprofProfilingDataRecord.INDEX_NAME); + List results = new ArrayList<>(); + for (final var table : tables) { + List condition = new ArrayList<>(4); + StringBuilder sql = new StringBuilder() + .append("select * from ").append(table) + .append(" where ").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); + condition.add(PprofProfilingDataRecord.INDEX_NAME); + + if (CollectionUtils.isNotEmpty(instanceIds)) { + sql.append(" and ").append(PprofProfilingDataRecord.INSTANCE_ID).append(" in (?) "); + String joinedInstanceIds = String.join(",", instanceIds); + condition.add(joinedInstanceIds); + } + + results.addAll( + jdbcClient.executeQuery( + sql.toString(), + resultSet -> { + final var result = new ArrayList(); + while (resultSet.next()) { + result.add(parseData(resultSet)); + } + return result; + }, + condition.toArray(new Object[0]) + ) + ); + } + return results; + } + + private PprofProfilingDataRecord parseData(ResultSet data) { + final PprofProfilingDataRecord.Builder builder = new PprofProfilingDataRecord.Builder(); + PprofProfilingDataRecord pprofProfilingDataRecord = builder.storage2Entity(JDBCEntityConverters.toEntity(data)); + byte[] dataBinary = pprofProfilingDataRecord.getDataBinary(); + if (dataBinary != null) { + byte[] decodeResult = Base64.getDecoder().decode(dataBinary); + pprofProfilingDataRecord.setDataBinary(decodeResult); + } + return pprofProfilingDataRecord; + } +} + \ No newline at end of file diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskLogQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskLogQueryDAO.java new file mode 100644 index 000000000000..0db85eca5035 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskLogQueryDAO.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskLogRecord; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCClient; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCTableInstaller; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.SQLAndParameters; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.TableHelper; + +@RequiredArgsConstructor +public class JDBCPprofTaskLogQueryDAO implements IPprofTaskLogQueryDAO { + private final JDBCClient jdbcClient; + private final TableHelper tableHelper; + + @Override + @SneakyThrows + public List getTaskLogList() { + List tables = tableHelper.getTablesWithinTTL(PprofTaskLogRecord.INDEX_NAME); + final List results = new ArrayList(); + for (String table : tables) { + SQLAndParameters sqlAndParameters = buildSQL(table); + List logs = jdbcClient.executeQuery( + sqlAndParameters.sql(), + resultSet -> { + final List tasks = new ArrayList<>(); + while (resultSet.next()) { + tasks.add(parseLog(resultSet)); + } + return tasks; + }, + sqlAndParameters.parameters() + ); + results.addAll(logs); + } + return results; + } + + private SQLAndParameters buildSQL(String table) { + StringBuilder sql = new StringBuilder(); + List parameters = new ArrayList<>(2); + sql.append("select * from ").append(table) + .append(" where ").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); + parameters.add(PprofTaskLogRecord.INDEX_NAME); + sql.append(" order by ").append(PprofTaskLogRecord.OPERATION_TIME).append(" desc"); + return new SQLAndParameters(sql.toString(), parameters); + } + + private PprofTaskLog parseLog(ResultSet data) throws SQLException { + return PprofTaskLog.builder() + .id(data.getString(PprofTaskLogRecord.TASK_ID)) + .instanceId(data.getString(PprofTaskLogRecord.INSTANCE_ID)) + .operationType( + PprofTaskLogOperationType.parse(data.getInt(PprofTaskLogRecord.OPERATION_TYPE))) + .operationTime(data.getLong(PprofTaskLogRecord.OPERATION_TIME)) + .build(); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskQueryDAO.java new file mode 100644 index 000000000000..1dd991a25279 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskQueryDAO.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.io.IOException; +import java.lang.reflect.Type; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCClient; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCTableInstaller; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.TableHelper; + +import static java.util.stream.Collectors.toList; + +@RequiredArgsConstructor +public class JDBCPprofTaskQueryDAO implements IPprofTaskQueryDAO { + private static final Gson GSON = new Gson(); + + private final JDBCClient jdbcClient; + private final TableHelper tableHelper; + + @Override + @SneakyThrows + public List getTaskList(String serviceId, + Long startTimeBucket, + Long endTimeBucket, + Integer limit) throws IOException { + final var results = new ArrayList(); + final var tables = startTimeBucket == null || endTimeBucket == null ? + tableHelper.getTablesWithinTTL(PprofTaskRecord.INDEX_NAME) : + tableHelper.getTablesForRead(PprofTaskRecord.INDEX_NAME, startTimeBucket, endTimeBucket); + for (final var table : tables) { + List condition = new ArrayList<>(4); + StringBuilder sql = new StringBuilder() + .append("select * from ").append(table) + .append(" where ").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); + condition.add(PprofTaskRecord.INDEX_NAME); + + if (StringUtil.isNotEmpty(serviceId)) { + sql.append(" and ").append(PprofTaskRecord.SERVICE_ID).append("=? "); + condition.add(serviceId); + } + + if (startTimeBucket != null) { + sql.append(" and ").append(PprofTaskRecord.TIME_BUCKET).append(" >= ? "); + condition.add(startTimeBucket); + } + + if (endTimeBucket != null) { + sql.append(" and ").append(PprofTaskRecord.TIME_BUCKET).append(" <= ? "); + condition.add(endTimeBucket); + } + + sql.append(" ORDER BY ").append(PprofTaskRecord.CREATE_TIME).append(" DESC "); + + if (limit != null) { + sql.append(" LIMIT ").append(limit); + } + + results.addAll( + jdbcClient.executeQuery( + sql.toString(), + resultSet -> { + final var tasks = new ArrayList(); + while (resultSet.next()) { + tasks.add(buildPprofTask(resultSet)); + } + return tasks; + }, + condition.toArray(new Object[0]) + ) + ); + } + return limit == null ? + results : + results + .stream() + .limit(limit) + .collect(toList()); + } + + @Override + @SneakyThrows + public PprofTask getById(String id) throws IOException { + final var tables = tableHelper.getTablesWithinTTL(PprofTaskRecord.INDEX_NAME); + for (String table : tables) { + final StringBuilder sql = new StringBuilder(); + final List condition = new ArrayList<>(1); + sql.append("select * from ").append(table) + .append(" where ") + .append(JDBCTableInstaller.TABLE_COLUMN).append(" = ? ") + .append(" and ") + .append(PprofTaskRecord.TASK_ID + "=? LIMIT 1"); + condition.add(PprofTaskRecord.INDEX_NAME); + condition.add(id); + + final var r = jdbcClient.executeQuery( + sql.toString(), + resultSet -> { + if (resultSet.next()) { + return buildPprofTask(resultSet); + } + return null; + }, + condition.toArray(new Object[0]) + ); + if (r != null) { + return r; + } + } + return null; + } + + private PprofTask buildPprofTask(ResultSet data) throws SQLException { + Type listType = new TypeToken>() { + }.getType(); + String events = data.getString(PprofTaskRecord.EVENT_TYPES); + String serviceInstanceIds = data.getString(PprofTaskRecord.SERVICE_INSTANCE_IDS); + List serviceInstanceIdList = GSON.fromJson(serviceInstanceIds, listType); + return PprofTask.builder() + .id(data.getString(PprofTaskRecord.TASK_ID)) + .serviceId(data.getString(PprofTaskRecord.SERVICE_ID)) + .serviceInstanceIds(serviceInstanceIdList) + .createTime(data.getLong(PprofTaskRecord.CREATE_TIME)) + .duration(data.getInt(PprofTaskRecord.DURATION)) + .events(PprofEventType.valueOfString(events)) + .dumpPeriod(data.getInt(PprofTaskRecord.DUMP_PERIOD)) + .build(); + } + +} diff --git a/oap-server/server-tools/profile-exporter/tool-profile-snapshot-server-mock/src/main/java/org/apache/skywalking/oap/server/tool/profile/core/MockCoreModuleProvider.java b/oap-server/server-tools/profile-exporter/tool-profile-snapshot-server-mock/src/main/java/org/apache/skywalking/oap/server/tool/profile/core/MockCoreModuleProvider.java index a5e98389984e..8e55a72df68d 100755 --- a/oap-server/server-tools/profile-exporter/tool-profile-snapshot-server-mock/src/main/java/org/apache/skywalking/oap/server/tool/profile/core/MockCoreModuleProvider.java +++ b/oap-server/server-tools/profile-exporter/tool-profile-snapshot-server-mock/src/main/java/org/apache/skywalking/oap/server/tool/profile/core/MockCoreModuleProvider.java @@ -25,6 +25,7 @@ import org.apache.skywalking.oap.server.core.annotation.AnnotationScan; import org.apache.skywalking.oap.server.core.cache.AsyncProfilerTaskCache; import org.apache.skywalking.oap.server.core.cache.NetworkAddressAliasCache; +import org.apache.skywalking.oap.server.core.cache.PprofTaskCache; import org.apache.skywalking.oap.server.core.cache.ProfileTaskCache; import org.apache.skywalking.oap.server.core.command.CommandService; import org.apache.skywalking.oap.server.core.config.ConfigService; @@ -44,6 +45,8 @@ import org.apache.skywalking.oap.server.core.profiling.continuous.ContinuousProfilingQueryService; import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingMutationService; import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingQueryService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofMutationService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofQueryService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskMutationService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskQueryService; import org.apache.skywalking.oap.server.core.query.AggregationQueryService; @@ -212,6 +215,12 @@ TTLStatusQuery.class, new TTLStatusQuery( AsyncProfilerQueryService.class, new AsyncProfilerQueryService(getManager())); this.registerServiceImplementation( AsyncProfilerTaskCache.class, new AsyncProfilerTaskCache(getManager(), moduleConfig)); + this.registerServiceImplementation( + PprofMutationService.class, new PprofMutationService(getManager())); + this.registerServiceImplementation( + PprofQueryService.class, new PprofQueryService(getManager())); + this.registerServiceImplementation( + PprofTaskCache.class, new PprofTaskCache(moduleConfig)); this.registerServiceImplementation( EBPFProfilingMutationService.class, new EBPFProfilingMutationService(getManager())); this.registerServiceImplementation( diff --git a/test/e2e-v2/cases/go/service/go.mod b/test/e2e-v2/cases/go/service/go.mod index 2c412645299f..1e2b1db6e369 100644 --- a/test/e2e-v2/cases/go/service/go.mod +++ b/test/e2e-v2/cases/go/service/go.mod @@ -20,7 +20,7 @@ module sw-e2e go 1.19 require ( - github.com/apache/skywalking-go v0.5.1-0.20250301084827-154de50628e8 + github.com/apache/skywalking-go v0.6.1-0.20251023090254-afa75a3cc8c3 github.com/apache/skywalking-go/toolkit v0.5.1-0.20250301084827-154de50628e8 github.com/gin-gonic/gin v1.10.0 ) @@ -40,13 +40,16 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/uuid v1.3.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.15.9 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/segmentio/kafka-go v0.4.43 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect golang.org/x/arch v0.8.0 // indirect diff --git a/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml new file mode 100644 index 000000000000..f8964fe7a8d3 --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/banyandb/docker-compose.yml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '3.8' + +services: + banyandb: + extends: + file: ../../../../script/docker-compose/base-compose.yml + service: banyandb + networks: + - e2e + + go-service: + build: + context: ../../../go/service + dockerfile: Dockerfile + args: + - SW_AGENT_GO_COMMIT=${SW_AGENT_GO_COMMIT} + networks: + - e2e + expose: + - 8080 + environment: + SW_AGENT_NAME: go-service + SW_AGENT_INSTANCE_NAME: provider1 + SW_AGENT_REPORTER_GRPC_BACKEND_SERVICE: oap:11800 + UPSTREAM_URL: http://localhost:8080/ignored.html + privileged: true + stop_grace_period: 30s + depends_on: + oap: + condition: service_healthy + healthcheck: + test: ["CMD", "sh", "-c", "nc -z 127.0.0.1 8080"] + interval: 5s + timeout: 60s + retries: 120 + ports: + - 8080 + + oap: + extends: + file: ../../../../script/docker-compose/base-compose.yml + service: oap + environment: + SW_STORAGE: banyandb + depends_on: + banyandb: + condition: service_healthy + ports: + - 12800 + +networks: + e2e: diff --git a/test/e2e-v2/cases/profiling/pprof/banyandb/e2e.yaml b/test/e2e-v2/cases/profiling/pprof/banyandb/e2e.yaml new file mode 100644 index 000000000000..956e87c2820e --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/banyandb/e2e.yaml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 20m + init-system-environment: ../../../../script/env + steps: + - name: set PATH + command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + +verify: + retry: + count: 20 + interval: 10s + cases: + - includes: + - ../profiling-cases.yaml diff --git a/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml new file mode 100644 index 000000000000..c7d2e9519de4 --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '3.8' + +services: + es: + image: elastic/elasticsearch:7.15.0 + expose: + - 9200 + networks: + - e2e + environment: + - discovery.type=single-node + - cluster.routing.allocation.disk.threshold_enabled=false + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"] + interval: 5s + timeout: 60s + retries: 120 + + go-service: + build: + context: ../../../go/service + dockerfile: Dockerfile + args: + - SW_AGENT_GO_COMMIT=${SW_AGENT_GO_COMMIT} + networks: + - e2e + expose: + - 8080 + environment: + SW_AGENT_NAME: go-service + SW_AGENT_INSTANCE_NAME: provider1 + SW_AGENT_REPORTER_GRPC_BACKEND_SERVICE: oap:11800 + UPSTREAM_URL: http://localhost:8080/ignored.html + privileged: true + stop_grace_period: 30s + depends_on: + oap: + condition: service_healthy + healthcheck: + test: ["CMD", "sh", "-c", "nc -z 127.0.0.1 8080"] + interval: 5s + timeout: 60s + retries: 120 + ports: + - 8080 + + oap: + extends: + file: ../../../../script/docker-compose/base-compose.yml + service: oap + environment: + SW_STORAGE: elasticsearch + depends_on: + es: + condition: service_healthy + ports: + - 12800 + +networks: + e2e: diff --git a/test/e2e-v2/cases/profiling/pprof/es/e2e.yaml b/test/e2e-v2/cases/profiling/pprof/es/e2e.yaml new file mode 100644 index 000000000000..956e87c2820e --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/es/e2e.yaml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 20m + init-system-environment: ../../../../script/env + steps: + - name: set PATH + command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + +verify: + retry: + count: 20 + interval: 10s + cases: + - includes: + - ../profiling-cases.yaml diff --git a/test/e2e-v2/cases/profiling/pprof/expected/analysis.yml b/test/e2e-v2/cases/profiling/pprof/expected/analysis.yml new file mode 100644 index 000000000000..31d6b8056290 --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/expected/analysis.yml @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tree: + elements: + {{- contains .tree.elements }} + - id: {{ notEmpty .id }} + parentid: {{ notEmpty .parentid }} + codesignature: {{ notEmpty .codesignature }} + total: {{ gt .total -1 }} + self: {{ gt .self -1 }} + {{- end }} \ No newline at end of file diff --git a/test/e2e-v2/cases/profiling/pprof/expected/create.yml b/test/e2e-v2/cases/profiling/pprof/expected/create.yml new file mode 100644 index 000000000000..85bd8870434e --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/expected/create.yml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +errorreason: null +code: SUCCESS +id: {{ notEmpty .id }} \ No newline at end of file diff --git a/test/e2e-v2/cases/profiling/pprof/expected/list.yml b/test/e2e-v2/cases/profiling/pprof/expected/list.yml new file mode 100644 index 000000000000..823d93eebdbe --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/expected/list.yml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +errorreason: null +tasks: + {{- contains .tasks }} +- id: {{ notEmpty .id }} + serviceid: {{ b64enc "go-service" }}.1 + serviceinstanceids: + - {{ b64enc "go-service" }}.1_{{ b64enc "provider1" }} + createtime: {{ gt .createtime 0 }} + dumpperiod: {{ ge .dumpperiod 0 }} + events: HEAP + duration: {{ ge .duration 0 }} + {{- end }} \ No newline at end of file diff --git a/test/e2e-v2/cases/profiling/pprof/expected/progress.yml b/test/e2e-v2/cases/profiling/pprof/expected/progress.yml new file mode 100644 index 000000000000..56de9d629489 --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/expected/progress.yml @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +logs: + {{- contains .logs }} +- id: {{ notEmpty .id}} + instanceid: {{ b64enc "go-service" }}.1_{{ b64enc "provider1" }} + instancename: provider1 + operationtype: EXECUTION_FINISHED + operationtime: {{ ge .operationtime 0 }} + {{- end }} +errorinstanceids: [] +successinstanceids: + - {{ b64enc "go-service" }}.1_{{ b64enc "provider1" }} diff --git a/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml b/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml new file mode 100644 index 000000000000..630d58ce03c4 --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/expected/service-instance.yml @@ -0,0 +1,28 @@ +# Licensed to Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Apache Software Foundation (ASF) licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{{- contains . }} +- id: {{ notEmpty .id }} + name: {{ notEmpty .name }} + attributes: + {{- contains .attributes }} + - name: ipv4s + value: {{ notEmpty .value }} + {{- end}} + language: GO + instanceuuid: {{ notEmpty .instanceuuid }} +{{- end}} diff --git a/test/e2e-v2/cases/profiling/pprof/expected/service.yml b/test/e2e-v2/cases/profiling/pprof/expected/service.yml new file mode 100644 index 000000000000..5744ee6c74b4 --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/expected/service.yml @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- contains . }} +- id: {{ b64enc "go-service" }}.1 + name: go-service + group: "" + shortname: go-service + normal: true + layers: + - GENERAL + - SO11Y_GO_AGENT +{{- end }} diff --git a/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml new file mode 100644 index 000000000000..828163d3eeda --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/mysql/docker-compose.yml @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + mysql: + image: mysql/mysql-server:8.0.13 + networks: + - e2e + expose: + - 3306 + environment: + - MYSQL_ROOT_PASSWORD=root@1234 + - MYSQL_DATABASE=swtest + - MYSQL_ROOT_HOST=% + healthcheck: + test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/3306" ] + interval: 5s + timeout: 60s + retries: 120 + go-service: + build: + context: ../../../go/service + dockerfile: Dockerfile + args: + - SW_AGENT_GO_COMMIT=${SW_AGENT_GO_COMMIT} + networks: + - e2e + expose: + - 8080 + environment: + SW_AGENT_NAME: go-service + SW_AGENT_INSTANCE_NAME: provider1 + SW_AGENT_REPORTER_GRPC_BACKEND_SERVICE: oap:11800 + UPSTREAM_URL: http://localhost:8080/ignored.html + privileged: true + stop_grace_period: 30s + depends_on: + oap: + condition: service_healthy + healthcheck: + test: ["CMD", "sh", "-c", "nc -z 127.0.0.1 8080"] + interval: 5s + timeout: 60s + retries: 120 + ports: + - 8080 + + oap: + extends: + file: ../../../../script/docker-compose/base-compose.yml + service: oap + environment: + SW_STORAGE: mysql + depends_on: + mysql: + condition: service_healthy + entrypoint: ['sh', '-c', '/download-mysql.sh /skywalking/oap-libs && /skywalking/docker-entrypoint.sh'] + ports: + - 12800 + +networks: + e2e: diff --git a/test/e2e-v2/cases/profiling/pprof/mysql/e2e.yaml b/test/e2e-v2/cases/profiling/pprof/mysql/e2e.yaml new file mode 100644 index 000000000000..956e87c2820e --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/mysql/e2e.yaml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 20m + init-system-environment: ../../../../script/env + steps: + - name: set PATH + command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + +verify: + retry: + count: 20 + interval: 10s + cases: + - includes: + - ../profiling-cases.yaml diff --git a/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml b/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml new file mode 100644 index 000000000000..8256a642241a --- /dev/null +++ b/test/e2e-v2/cases/profiling/pprof/profiling-cases.yaml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +cases: + # service list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls + expected: expected/service.yml + # service instance list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=go-service + expected: expected/service-instance.yml + # create task + - query: | + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ + profiling pprof create --service-name=go-service \ + --dump-period=1 --events=HEAP \ + --instance-name-list=provider1 + expected: expected/create.yml + # list task + - query: | + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ + profiling pprof list --service-name=go-service + expected: expected/list.yml + # get task progress finished + - query: | + sleep 15 && swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ + profiling pprof progress --task-id=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling pprof list --service-name=go-service | yq e '.tasks[0].id') + expected: expected/progress.yml + # get task analysis + - query: | + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql \ + profiling pprof analysis --service-name=go-service \ + --instance-name-list=provider1 \ + --task-id=$(swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql profiling pprof list --service-name=go-service | yq e '.tasks[0].id') + expected: expected/analysis.yml + diff --git a/test/e2e-v2/script/env b/test/e2e-v2/script/env index 480d3081fef1..343737522f92 100644 --- a/test/e2e-v2/script/env +++ b/test/e2e-v2/script/env @@ -17,7 +17,7 @@ SW_AGENT_JAVA_COMMIT=f0245864e4388a388fe7445b56b6ce7cedc94aaf SW_AGENT_SATELLITE_COMMIT=ea27a3f4e126a24775fe12e2aa2695bcb23d99c3 SW_AGENT_NGINX_LUA_COMMIT=c3cee4841798a147d83b96a10914d4ac0e11d0aa SW_AGENT_NODEJS_COMMIT=4f9a91dad3dfd8cfe5ba8f7bd06b39e11eb5e65e -SW_AGENT_GO_COMMIT=154de50628e82e590941585411299459e352317d +SW_AGENT_GO_COMMIT=afa75a3cc8c31f142102443af6164b825d63d8fc SW_AGENT_PYTHON_COMMIT=c76a6ec51a478ac91abb20ec8f22a99b8d4d6a58 SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449 SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016 @@ -27,4 +27,4 @@ SW_BANYANDB_COMMIT=7466afd2a2120f7fa311983be5a3077cb15d07e7 SW_AGENT_PHP_COMMIT=d1114e7be5d89881eec76e5b56e69ff844691e35 SW_PREDICTOR_COMMIT=54a0197654a3781a6f73ce35146c712af297c994 -SW_CTL_COMMIT=3b675df73824bbb80e6aabf6a95d110feb37b6b1 +SW_CTL_COMMIT=9a1beab08413ce415a00a8547a238a14691c5655