rotate postgres logs WIP

Author: dsessler7

internal/collector/config.go

@@ -9,6 +9,7 @@ import (
 	_ "embed"
 	"fmt"
 	"math"
+	"strings"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -118,7 +119,7 @@ func NewConfig(spec *v1beta1.InstrumentationSpec) *Config {
 // AddLogrotateConfig generates a logrotate configuration and adds it to the
 // provided configmap
 func AddLogrotateConfig(ctx context.Context, spec *v1beta1.InstrumentationSpec,
-	outInstanceConfigMap *corev1.ConfigMap, logFilePath, postrotateScript string,
+	outInstanceConfigMap *corev1.ConfigMap, logFilesToRotate []string, postrotateScript string,
 ) error {
 	var err error
 	var retentionPeriod *v1beta1.Duration
@@ -138,22 +139,22 @@ func AddLogrotateConfig(ctx context.Context, spec *v1beta1.InstrumentationSpec,
 		}
 	}
 
-	outInstanceConfigMap.Data["logrotate.conf"] = generateLogrotateConfig(logFilePath,
+	outInstanceConfigMap.Data["logrotate.conf"] = generateLogrotateConfig(logFilesToRotate,
 		retentionPeriod, postrotateScript)
 
 	return err
 }
 
 // generateLogrotateConfig generates a configuration string for logrotate based
 // on the provided full log file path, retention period, and postrotate script
-func generateLogrotateConfig(logFilePath string, retentionPeriod *v1beta1.Duration,
+func generateLogrotateConfig(logFilesToRotate []string, retentionPeriod *v1beta1.Duration,
 	postrotateScript string,
 ) string {
 	number, interval := parseDurationForLogrotate(retentionPeriod)
 
 	return fmt.Sprintf(
 		logrotateConfigFormatString,
-		logFilePath,
+		strings.Join(logFilesToRotate, " "),
 		number,
 		interval,
 		postrotateScript,

internal/collector/postgres.go

@@ -37,6 +37,10 @@ func NewConfigForPostgresPod(ctx context.Context,
 //go:embed "generated/postgres_logs_transforms.json"
 var postgresLogsTransforms json.RawMessage
 
+// PostgresPostRotateScript is the script that is run after postgres's log
+// files have been rotated.
+const PostgresPostRotateScript = "pkill -USR1 --exact --parent=1 postgres"
+
 // postgresCSVNames returns the names of fields in the CSV logs for version.
 func postgresCSVNames(version int) string {
 	// JSON is the preferred format, so use those names.
@@ -100,11 +104,11 @@ func EnablePostgresLogging(
 		// probably requires another process that deletes the oldest files.
 		//
 		// The ".log" suffix is replaced by ".json" for JSON log files.
-		outParameters.Mandatory.Add("log_filename", "postgresql-%a.log")
+		outParameters.Mandatory.Add("log_filename", "postgresql")
 		outParameters.Mandatory.Add("log_file_mode", "0660")
-		outParameters.Mandatory.Add("log_rotation_age", "1d")
+		outParameters.Mandatory.Add("log_rotation_age", "0")
 		outParameters.Mandatory.Add("log_rotation_size", "0")
-		outParameters.Mandatory.Add("log_truncate_on_rotation", "on")
+		outParameters.Mandatory.Add("log_truncate_on_rotation", "off")
 
 		// Log in a timezone that the OpenTelemetry Collector will understand.
 		outParameters.Mandatory.Add("log_timezone", "UTC")
@@ -124,7 +128,7 @@ func EnablePostgresLogging(
 		// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/receiver/filelogreceiver#readme
 		outConfig.Receivers["filelog/postgres_csvlog"] = map[string]any{
 			// Read the CSV files and keep track of what has been processed.
-			"include": []string{directory + "/*.csv"},
+			"include": []string{directory + "/*.csv", directory + "/*.csv.1"},
 			"storage": "file_storage/postgres_logs",
 
 			// Postgres does not escape newlines in its CSV log format. Search for
@@ -155,7 +159,7 @@ func EnablePostgresLogging(
 		// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/receiver/filelogreceiver#readme
 		outConfig.Receivers["filelog/postgres_jsonlog"] = map[string]any{
 			// Read the JSON files and keep track of what has been processed.
-			"include": []string{directory + "/*.json"},
+			"include": []string{directory + "/*.json", directory + "/*.json.1"},
 			"storage": "file_storage/postgres_logs",
 
 			// Differentiate these from the CSV ones above.

internal/collector/postgres_test.go

@@ -190,6 +190,7 @@ receivers:
   filelog/postgres_csvlog:
     include:
     - /pgdata/logs/postgres/*.csv
+    - /pgdata/logs/postgres/*.csv.1
     multiline:
       line_start_pattern: ^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d.\d{3} UTC,(?:"[_\D](?:[^"]|"")*")?,(?:"[_\D](?:[^"]|"")*")?,\d*,(?:"(?:[^"]|"")+")?,[0-9a-f]+[.][0-9a-f]+,\d+,
     operators:
@@ -206,6 +207,7 @@ receivers:
   filelog/postgres_jsonlog:
     include:
     - /pgdata/logs/postgres/*.json
+    - /pgdata/logs/postgres/*.json.1
     operators:
     - from: body
       to: body.original
@@ -422,6 +424,7 @@ receivers:
   filelog/postgres_csvlog:
     include:
     - /pgdata/logs/postgres/*.csv
+    - /pgdata/logs/postgres/*.csv.1
     multiline:
       line_start_pattern: ^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d.\d{3} UTC,(?:"[_\D](?:[^"]|"")*")?,(?:"[_\D](?:[^"]|"")*")?,\d*,(?:"(?:[^"]|"")+")?,[0-9a-f]+[.][0-9a-f]+,\d+,
     operators:
@@ -438,6 +441,7 @@ receivers:
   filelog/postgres_jsonlog:
     include:
     - /pgdata/logs/postgres/*.json
+    - /pgdata/logs/postgres/*.json.1
     operators:
     - from: body
       to: body.original

internal/controller/postgrescluster/instance.go

@@ -1206,7 +1206,7 @@ func (r *Reconciler) reconcileInstance(
 		// TODO: Setting the includeLogrotate argument to false for now. This
 		// should be changed when we implement log rotation for postgres
 		collector.AddToPod(ctx, cluster.Spec.Instrumentation, cluster.Spec.ImagePullPolicy, instanceConfigMap, &instance.Spec.Template.Spec,
-			[]corev1.VolumeMount{postgres.DataVolumeMount()}, "", false)
+			[]corev1.VolumeMount{postgres.DataVolumeMount()}, "", true)
 	}
 
 	if err == nil &&
@@ -1427,6 +1427,15 @@ func (r *Reconciler) reconcileInstanceConfigMap(
 	if err == nil && (feature.Enabled(ctx, feature.OpenTelemetryLogs) || feature.Enabled(ctx, feature.OpenTelemetryMetrics)) {
 		err = collector.AddToConfigMap(ctx, otelConfig, instanceConfigMap)
 	}
+	// If OTel logging is enabled, add logrotate config
+	if err == nil && otelConfig != nil && feature.Enabled(ctx, feature.OpenTelemetryLogs) {
+		logFilesToRotate := []string{
+			postgres.LogDirectory() + "/postgresql.json",
+			postgres.LogDirectory() + "/postgresql.csv",
+		}
+		err = collector.AddLogrotateConfig(ctx, cluster.Spec.Instrumentation, instanceConfigMap,
+			logFilesToRotate, collector.PostgresPostRotateScript)
+	}
 	if err == nil {
 		err = patroni.InstanceConfigMap(ctx, cluster, spec, instanceConfigMap)
 	}

internal/controller/postgrescluster/pgbouncer.go

@@ -106,7 +106,7 @@ func (r *Reconciler) reconcilePGBouncerConfigMap(
 	// If OTel logging is enabled, add logrotate config
 	if err == nil && otelConfig != nil && feature.Enabled(ctx, feature.OpenTelemetryLogs) {
 		err = collector.AddLogrotateConfig(ctx, cluster.Spec.Instrumentation, configmap,
-			naming.PGBouncerFullLogPath, collector.PGBouncerPostRotateScript)
+			[]string{naming.PGBouncerFullLogPath}, collector.PGBouncerPostRotateScript)
 	}
 	if err == nil {
 		err = errors.WithStack(r.apply(ctx, configmap))