From 8b707385c2244ad9417a6a34bc957a2de9c9e116 Mon Sep 17 00:00:00 2001 From: jmccormick7 Date: Fri, 10 Oct 2025 13:22:00 -0400 Subject: [PATCH 1/4] plugin_proxy: enable event_type specification for proxy plugins This commit adds event_type to the flb_plugin_proxy_def struct. This allows for the setting of event_type to a value other than the default log only initialization. In the flb_plugin_proxy_register function default behaviour is enforced by checking for unset event_type vlaues. When unset (a value of 0) then log behaviour is defaulted. No need for incorrect value checking since this was not a set field in the past. Unset fields are set as current behavior dicates and future usecases using incorrect values can be treated as user error. Input use case does not use event-type and is unaffected by this change. Signed-off-by: jmccormick7 --- include/fluent-bit/flb_plugin_proxy.h | 1 + src/flb_plugin_proxy.c | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/include/fluent-bit/flb_plugin_proxy.h b/include/fluent-bit/flb_plugin_proxy.h index fb245291bd1..8e534ecdf75 100644 --- a/include/fluent-bit/flb_plugin_proxy.h +++ b/include/fluent-bit/flb_plugin_proxy.h @@ -40,6 +40,7 @@ struct flb_plugin_proxy_def { int flags; char *name; /* plugin short name */ char *description; /* plugin description */ + int event_type; /* event type (logs/metrics/traces) */ }; /* Proxy context */ diff --git a/src/flb_plugin_proxy.c b/src/flb_plugin_proxy.c index 2ec84b546c5..e97890d017f 100644 --- a/src/flb_plugin_proxy.c +++ b/src/flb_plugin_proxy.c @@ -362,6 +362,14 @@ static int flb_proxy_register_output(struct flb_plugin_proxy *proxy, out->flags = def->flags; out->name = flb_strdup(def->name); + /* If event_type is unset (0) then default to logs (this is the current behavior) */ + if (def->event_type == 0) { + out->event_type = FLB_OUTPUT_LOGS; + } + else { + out->event_type = def->event_type; + } + out->description = def->description; mk_list_add(&out->_head, &config->out_plugins); @@ -396,6 +404,7 @@ static int flb_proxy_register_input(struct flb_plugin_proxy *proxy, in->flags = def->flags | FLB_INPUT_THREADED; in->name = flb_strdup(def->name); in->description = def->description; + mk_list_add(&in->_head, &config->in_plugins); /* From 82d792a5c4ac43050a5c617c1dc56f646e816f46 Mon Sep 17 00:00:00 2001 From: jmccormick7 Date: Fri, 10 Oct 2025 15:13:02 -0400 Subject: [PATCH 2/4] plugin_proxy: change proxydef initialization to use calloc instead of malloc Changing intialization to use calloc instead of malloc so that emtpy fields are 0 initialized. Specifically for the event_type addition. Signed-off-by: jmccormick7 --- src/flb_plugin_proxy.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flb_plugin_proxy.c b/src/flb_plugin_proxy.c index e97890d017f..8a58181b803 100644 --- a/src/flb_plugin_proxy.c +++ b/src/flb_plugin_proxy.c @@ -621,7 +621,7 @@ struct flb_plugin_proxy *flb_plugin_proxy_create(const char *dso_path, int type, return NULL; } - proxy->def = flb_malloc(sizeof(struct flb_plugin_proxy_def)); + proxy->def = flb_calloc(1, sizeof(struct flb_plugin_proxy_def)); if (!proxy->def) { flb_errno(); dlclose(handle); From 74c83d133d4bdc4c8e5d85b4afc928f331d70921 Mon Sep 17 00:00:00 2001 From: jmccormick7 Date: Wed, 12 Nov 2025 12:17:07 -0500 Subject: [PATCH 3/4] runtime_shell: Adding test for go plugins This commit adds a runtime shell test for the go plugin interface. A dummy go output plugin is added to the directories, and a test that builds the plugin shared object and then tests that it works and writes the output. This test ensures that the go proxy interface is working as expected. This was tested in the devcontainer using ctest. Signed-off-by: jmccormick7 --- tests/runtime_shell/CMakeLists.txt | 4 +- tests/runtime_shell/conf/proxy_logs_test.conf | 21 ++++ .../conf/proxy_metrics_test.conf | 0 .../go_plugins/build_test_plugins.sh | 111 ++++++++++++++++++ tests/runtime_shell/go_plugins/go.mod | 8 ++ tests/runtime_shell/go_plugins/go.sum | 5 + tests/runtime_shell/go_plugins/logs_output.go | 47 ++++++++ tests/runtime_shell/go_plugins/test_logs_go.h | 92 +++++++++++++++ tests/runtime_shell/proxy_logs_expect.sh | 38 ++++++ 9 files changed, 325 insertions(+), 1 deletion(-) create mode 100644 tests/runtime_shell/conf/proxy_logs_test.conf create mode 100644 tests/runtime_shell/conf/proxy_metrics_test.conf create mode 100755 tests/runtime_shell/go_plugins/build_test_plugins.sh create mode 100644 tests/runtime_shell/go_plugins/go.mod create mode 100644 tests/runtime_shell/go_plugins/go.sum create mode 100644 tests/runtime_shell/go_plugins/logs_output.go create mode 100644 tests/runtime_shell/go_plugins/test_logs_go.h create mode 100755 tests/runtime_shell/proxy_logs_expect.sh diff --git a/tests/runtime_shell/CMakeLists.txt b/tests/runtime_shell/CMakeLists.txt index 30ae87a6b96..f201bb46910 100644 --- a/tests/runtime_shell/CMakeLists.txt +++ b/tests/runtime_shell/CMakeLists.txt @@ -15,6 +15,7 @@ set(UNIT_TESTS_SH in_syslog_uds_stream_plaintext_expect.sh processor_conditional.sh processor_invalid.sh + proxy_logs_expect.sh ) # Prepare list of unit tests @@ -27,6 +28,7 @@ foreach(script ${UNIT_TESTS_SH}) "FLB_ROOT=${PROJECT_SOURCE_DIR};\ FLB_RUNTIME_SHELL_PATH=${CMAKE_CURRENT_SOURCE_DIR};\ FLB_RUNTIME_SHELL_CONF=${CMAKE_CURRENT_SOURCE_DIR}/conf;\ -FLB_BIN=${CMAKE_BINARY_DIR}/bin/fluent-bit" +FLB_BIN=${CMAKE_BINARY_DIR}/bin/fluent-bit;\ +FLB_BUILD=${CMAKE_BINARY_DIR}" ) endforeach() diff --git a/tests/runtime_shell/conf/proxy_logs_test.conf b/tests/runtime_shell/conf/proxy_logs_test.conf new file mode 100644 index 00000000000..2946088e6f6 --- /dev/null +++ b/tests/runtime_shell/conf/proxy_logs_test.conf @@ -0,0 +1,21 @@ +[SERVICE] + Flush 1 + Grace 2 + Log_Level info + Daemon Off + +[INPUT] + Name dummy + Dummy {"message": "test log entry", "level": "info"} + Samples 3 + Tag test.logs + +[OUTPUT] + Name test_logs_go + Match test.logs + +[OUTPUT] + Name file + Match test.logs + File ${SIGNAL_FILE_PATH} + mkdir on diff --git a/tests/runtime_shell/conf/proxy_metrics_test.conf b/tests/runtime_shell/conf/proxy_metrics_test.conf new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/runtime_shell/go_plugins/build_test_plugins.sh b/tests/runtime_shell/go_plugins/build_test_plugins.sh new file mode 100755 index 00000000000..3b1ab4e882e --- /dev/null +++ b/tests/runtime_shell/go_plugins/build_test_plugins.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +# Build script for Go test plugins +set -e + +GO_PLUGIN_DIR="${FLB_ROOT}/tests/runtime_shell/go_plugins" +BUILD_DIR="${FLB_ROOT}/build" + +install_go_if_needed() { + if ! command -v go &> /dev/null; then + echo "Go not found, installing Go..." + + ARCH=$(uname -m) + case $ARCH in + x86_64) GO_ARCH="amd64" ;; + aarch64|arm64) GO_ARCH="arm64" ;; + *) echo "Unsupported architecture: $ARCH"; exit 1 ;; + esac + + OS=$(uname -s | tr '[:upper:]' '[:lower:]') + GO_VERSION="1.25.4" + GO_TARBALL="go${GO_VERSION}.${OS}-${GO_ARCH}.tar.gz" + GO_URL="https://golang.org/dl/${GO_TARBALL}" + + echo "Downloading Go from $GO_URL..." + + TEMP_DIR=$(mktemp -d) + cd "$TEMP_DIR" + + if command -v curl > /dev/null 2>&1; then + curl -L -O "$GO_URL" + else + echo "Neither wget nor curl is available to download Go." + exit 1 + fi + + echo "Extracting Go tarball..." + ls -la + + if [ ! -f "$GO_TARBALL" ]; then + echo "Failed to download Go tarball." + exit 1 + fi + + tar -xzf "$GO_TARBALL" + + if [ -w "/usr/local" ]; then + sudo rm -rf /usr/local/go + sudo mv go /usr/local/go + export PATH="/usr/local/go/bin:$PATH" + else + echo "No write permission to /usr/local. Installing Go to $HOME/.local/go" + mkdir -p "$HOME/.local" + rm -rf "$HOME/.local/go" + mv go "$HOME/.local/go" + export PATH="$HOME/.local/go/bin:$PATH" + fi + cd - > /dev/null + rm -rf "$TEMP_DIR" + echo "Go installed successfully." + go version +else + echo "Go is already installed." +fi +} + +verify_go_cgo() { + echo "Verifying Go CGO support..." + if ! go env CGO_ENABLED | grep -q "1"; then + echo "Warning: CGO is not enabled. Attempting to enable CGO..." + export CGO_ENABLED=1 + fi + + TEMP_GO_FILE=$(mktemp --suffix=.go) + cat > "$TEMP_GO_FILE" << 'EOF' +package main +import "C" +//export TestFunc +func TestFunc() {} +func main() {} +EOF + TEMP_SO_FILE=$(mktemp --suffix=.so) + if go build -buildmode=c-shared -o "$TEMP_SO_FILE" "$TEMP_GO_FILE" 2> /dev/null; then + echo "CGO is enabled and working." + rm -f "$TEMP_GO_FILE" "$TEMP_SO_FILE" + else + echo "Error: CGO is not enabled or not working properly. Please ensure you have a C compiler installed." + rm -f "$TEMP_GO_FILE" "$TEMP_SO_FILE" + exit 1 + fi +} + +build_go_plugins() { + echo "Building Go test plugins..." + + echo "Building logs output plugin..." + cd "$GO_PLUGIN_DIR" + CGO_ENABLED=1 GO111MODULE=on go build -buildmode=c-shared -v -ldflags="-s -w" -o $BUILD_DIR/test_logs_go.so logs_output.go + if [ $? -eq 0 ]; then + echo "Go test plugins built successfully!" + echo "Logs plugin: $BUILD_DIR/test_logs_go.so" + else + echo "Failed to build Go test plugins." + exit 1 + fi +} + +echo "Setting up Go build environment..." +install_go_if_needed +verify_go_cgo +build_go_plugins \ No newline at end of file diff --git a/tests/runtime_shell/go_plugins/go.mod b/tests/runtime_shell/go_plugins/go.mod new file mode 100644 index 00000000000..ff0a7df9ff6 --- /dev/null +++ b/tests/runtime_shell/go_plugins/go.mod @@ -0,0 +1,8 @@ +module github.com/fluent/fluent-bit/tests/runtime_shell/go_plugins + +go 1.25.1 + +require ( + github.com/fluent/fluent-bit-go v0.0.0-20230731091245-a7a013e2473c + github.com/ugorji/go/codec v1.1.7 // indirect +) \ No newline at end of file diff --git a/tests/runtime_shell/go_plugins/go.sum b/tests/runtime_shell/go_plugins/go.sum new file mode 100644 index 00000000000..523259f04ad --- /dev/null +++ b/tests/runtime_shell/go_plugins/go.sum @@ -0,0 +1,5 @@ +github.com/fluent/fluent-bit-go v0.0.0-20230731091245-a7a013e2473c h1:yKN46XJHYC/gvgH2UsisJ31+n4K3S7QYZSfU2uAWjuI= +github.com/fluent/fluent-bit-go v0.0.0-20230731091245-a7a013e2473c/go.mod h1:L92h+dgwElEyUuShEwjbiHjseW410WIcNz+Bjutc8YQ= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= diff --git a/tests/runtime_shell/go_plugins/logs_output.go b/tests/runtime_shell/go_plugins/logs_output.go new file mode 100644 index 00000000000..21ac6b5d6ef --- /dev/null +++ b/tests/runtime_shell/go_plugins/logs_output.go @@ -0,0 +1,47 @@ +package main + +import ( + "C" + "fmt" + "unsafe" + + "github.com/fluent/fluent-bit-go/output" +) + +//export FLBPluginRegister +func FLBPluginRegister(def unsafe.Pointer) int { + // Register as logs-only output plugin + return output.FLBPluginRegister(def, "test_logs_go", "Test Go Output Plugin for Logs") +} + +//export FLBPluginInit +func FLBPluginInit(plugin unsafe.Pointer) int { + return output.FLB_OK +} + +//export FLBPluginFlushCtx +func FLBPluginFlushCtx(ctx, data unsafe.Pointer, length C.int, tag *C.char) int { + // Write to a stdout to verify it received data + dec := output.NewDecoder(data, int(length)) + var logrecords []string + for { + ret, _, record := output.GetRecord(dec) + if ret != 0 { + break + } + logrecords = append(logrecords, fmt.Sprintf("%v", record)) + } + for _, record := range logrecords { + fmt.Printf("%s\n", record) + } + + return output.FLB_OK +} + +//export FLBPluginExit +func FLBPluginExit() int { + return output.FLB_OK +} + +func main() { +} diff --git a/tests/runtime_shell/go_plugins/test_logs_go.h b/tests/runtime_shell/go_plugins/test_logs_go.h new file mode 100644 index 00000000000..6b6b8bb27cf --- /dev/null +++ b/tests/runtime_shell/go_plugins/test_logs_go.h @@ -0,0 +1,92 @@ +/* Code generated by cmd/cgo; DO NOT EDIT. */ + +/* package command-line-arguments */ + + +#line 1 "cgo-builtin-export-prolog" + +#include + +#ifndef GO_CGO_EXPORT_PROLOGUE_H +#define GO_CGO_EXPORT_PROLOGUE_H + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef struct { const char *p; ptrdiff_t n; } _GoString_; +extern size_t _GoStringLen(_GoString_ s); +extern const char *_GoStringPtr(_GoString_ s); +#endif + +#endif + +/* Start of preamble from import "C" comments. */ + + + + +/* End of preamble from import "C" comments. */ + + +/* Start of boilerplate cgo prologue. */ +#line 1 "cgo-gcc-export-header-prolog" + +#ifndef GO_CGO_PROLOGUE_H +#define GO_CGO_PROLOGUE_H + +typedef signed char GoInt8; +typedef unsigned char GoUint8; +typedef short GoInt16; +typedef unsigned short GoUint16; +typedef int GoInt32; +typedef unsigned int GoUint32; +typedef long long GoInt64; +typedef unsigned long long GoUint64; +typedef GoInt64 GoInt; +typedef GoUint64 GoUint; +typedef size_t GoUintptr; +typedef float GoFloat32; +typedef double GoFloat64; +#ifdef _MSC_VER +#if !defined(__cplusplus) || _MSVC_LANG <= 201402L +#include +typedef _Fcomplex GoComplex64; +typedef _Dcomplex GoComplex128; +#else +#include +typedef std::complex GoComplex64; +typedef std::complex GoComplex128; +#endif +#else +typedef float _Complex GoComplex64; +typedef double _Complex GoComplex128; +#endif + +/* + static assertion to make sure the file is being used on architecture + at least with matching size of GoInt. +*/ +typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef _GoString_ GoString; +#endif +typedef void *GoMap; +typedef void *GoChan; +typedef struct { void *t; void *v; } GoInterface; +typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; + +#endif + +/* End of boilerplate cgo prologue. */ + +#ifdef __cplusplus +extern "C" { +#endif + +extern GoInt FLBPluginRegister(void* def); +extern GoInt FLBPluginInit(void* plugin); +extern GoInt FLBPluginFlushCtx(void* ctx, void* data, int length, char* tag); +extern GoInt FLBPluginExit(void); + +#ifdef __cplusplus +} +#endif diff --git a/tests/runtime_shell/proxy_logs_expect.sh b/tests/runtime_shell/proxy_logs_expect.sh new file mode 100755 index 00000000000..08148d7542c --- /dev/null +++ b/tests/runtime_shell/proxy_logs_expect.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +# Setup environment if not already set +if [ -z "$FLB_BIN" ]; then + FLB_ROOT=${FLB_ROOT:-$(cd $(dirname $0)/../.. && pwd)} + FLB_BIN=${FLB_BIN:-$FLB_ROOT/build/bin/fluent-bit} +fi + +echo "Using Fluent Bit at: $FLB_BIN" + +. $FLB_RUNTIME_SHELL_PATH/go_plugins/build_test_plugins.sh + +test_proxy_logs_compatibility() { + export SIGNAL_FILE_PATH="/tmp/flb_signal_logs_$$.txt" + STDOUT_OUTPUT_FILE="/tmp/test_logs_stdout_$$.txt" + + rm -f "$STDOUT_OUTPUT_FILE" "$SIGNAL_FILE_PATH" + + $FLB_BIN -e $FLB_ROOT/build/test_logs_go.so -c $FLB_RUNTIME_SHELL_CONF/proxy_logs_test.conf > "$STDOUT_OUTPUT_FILE" 2>&1 & + FLB_PID=$! + + sleep 3 + + if [ -f "$STDOUT_OUTPUT_FILE" ]; then + echo "SUCCESS: Captured Fluent Bit output" + echo "Output contents:" + cat "$STDOUT_OUTPUT_FILE" + else + echo "FAIL: No stdout output captured" + return 1 + fi + + # Clean up + rm -f "$STDOUT_OUTPUT_FILE" "$SIGNAL_FILE_PATH" +} + +# Load the runtime shell environment +. $FLB_RUNTIME_SHELL_PATH/runtime_shell.env From 68002150f70d52d5f6aa6d821adbe92d085da932 Mon Sep 17 00:00:00 2001 From: jmccormick7 Date: Wed, 12 Nov 2025 12:41:27 -0500 Subject: [PATCH 4/4] runtime_shell: Apply coderabit fixes Signed-off-by: jmccormick7 --- tests/runtime_shell/go_plugins/build_test_plugins.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/runtime_shell/go_plugins/build_test_plugins.sh b/tests/runtime_shell/go_plugins/build_test_plugins.sh index 3b1ab4e882e..cb6e21ce697 100755 --- a/tests/runtime_shell/go_plugins/build_test_plugins.sh +++ b/tests/runtime_shell/go_plugins/build_test_plugins.sh @@ -45,7 +45,9 @@ install_go_if_needed() { tar -xzf "$GO_TARBALL" if [ -w "/usr/local" ]; then - sudo rm -rf /usr/local/go + if [ -d /usr/local/go ]; then + sudo rm -rf /usr/local/go + fi sudo mv go /usr/local/go export PATH="/usr/local/go/bin:$PATH" else