env.example

# Bento Flexprice Environment Configuration
# Copy this file to .env and fill in your actual values

# Flexprice API Configuration
# ===========================
# API host without protocol (e.g., api.cloud.flexprice.io or localhost:8080)
FLEXPRICE_API_HOST=api.cloud.flexprice.io

# Your Flexprice API key (get this from your Flexprice dashboard)
FLEXPRICE_API_KEY=your_api_key_here

# HTTP scheme (https for production, http for local development)
FLEXPRICE_SCHEME=https

# Kafka Configuration (Confluent Cloud )
# ================================================
# Kafka broker address (e.g., pkc-xxxxx.us-east-1.aws.confluent.cloud:9092)
FLEXPRICE_KAFKA_BROKERS=your-broker.confluent.cloud:9092

# Kafka topic to consume events from
FLEXPRICE_KAFKA_TOPIC=bento-testing

# SASL Authentication credentials
# Mechanism: PLAIN (Confluent Cloud) or SCRAM-SHA-512 (AWS MSK)
FLEXPRICE_KAFKA_SASL_MECHANISM=PLAIN
FLEXPRICE_KAFKA_SASL_USER=your_kafka_api_key
FLEXPRICE_KAFKA_SASL_PASSWORD=your_kafka_api_secret

# Consumer group name (change to reset offset position)
FLEXPRICE_KAFKA_CONSUMER_GROUP=bento-flexprice-v0

# Client ID for identification
FLEXPRICE_KAFKA_CLIENT_ID=bento-flexprice-collector

# Kafka Consumer Performance Tuning
# =================================
# Fetch buffer capacity (messages to pre-fetch)
# Increase for higher throughput (default: 1024)
KAFKA_FETCH_BUFFER_CAP=1024

# Checkpoint limit (messages to read before forced commit)
# Increase to allow larger batches and parallel processing
# IMPORTANT: Should be significantly larger than batch size * max_in_flight
# Default: 5000
KAFKA_CHECKPOINT_LIMIT=5000

# Logging Configuration
# =====================
# Log level: DEBUG, INFO, WARN, ERROR
LOG_LEVEL=INFO

# AWS S3 Bucket Output configuration
# ===================================================
FLEXPRICE_S3_BUCKET_NAME="flexprice-dev-testing"

# AWS S3 Bucket output path key prefix
# ===================================================
# FLEXPRICE_S3_KEY_PREFIX="data"

# S3 Write timeout (increase if seeing 'context deadline exceeded' errors)
# Default: 30s
S3_WRITE_TIMEOUT=30s


# Flexprice Output Kafka Configuration (for custom-kafka-to-flexprice-kafka.yaml)
# ================================================================================
# Tenant ID for Flexprice events (required)
FLEXPRICE_TENANT_ID=your-tenant-id

# Environment ID for Flexprice events (optional but recommended)
FLEXPRICE_ENVIRONMENT_ID=your-environment-id

# Kafka broker address for output (e.g., MSK cluster endpoint)
FLEXPRICE_OUTPUT_KAFKA_BROKERS=your-msk-broker.aws.amazonaws.com:9092

# Kafka topic to write transformed events to
FLEXPRICE_OUTPUT_KAFKA_TOPIC=bento-flexprice-output

# TLS enabled for output Kafka (default: true)
FLEXPRICE_OUTPUT_KAFKA_TLS_ENABLED=true

# SASL Authentication for output Kafka
# Mechanism: PLAIN (Confluent Cloud) or SCRAM-SHA-512 (AWS MSK)
FLEXPRICE_OUTPUT_KAFKA_SASL_MECHANISM=PLAIN
FLEXPRICE_OUTPUT_KAFKA_SASL_USER=your_output_kafka_user
FLEXPRICE_OUTPUT_KAFKA_SASL_PASSWORD=your_output_kafka_password

# Client ID for output Kafka producer
FLEXPRICE_OUTPUT_KAFKA_CLIENT_ID=bento-billing-flexprice-output

# Batching configuration for output Kafka
FLEXPRICE_OUTPUT_KAFKA_BATCH_SIZE=10
FLEXPRICE_OUTPUT_KAFKA_BATCH_PERIOD=2s

# Maximum concurrent requests for output Kafka
FLEXPRICE_OUTPUT_KAFKA_MAX_IN_FLIGHT=10

# Kafka DLQ Configuration (Dead Letter Queue)
# =============================================
# Topic for failed messages after all retries are exhausted
# Used by ClickHouse collectors (v1 and v2) for both ClickHouse and Kafka output failures
FLEXPRICE_KAFKA_DLQ_TOPIC=bento-clickhouse-dlq

# ClickHouse Configuration (for custom-kafka-to-flexprice-clickhouse.yaml)
# =========================================================================
# ClickHouse server address with port (native protocol uses port 9000)
# Example: clickhouse.example.com:9000
FLEXPRICE_CLICKHOUSE_ADDRESS=your-clickhouse-host:9000

# ClickHouse authentication
FLEXPRICE_CLICKHOUSE_USERNAME=default
FLEXPRICE_CLICKHOUSE_PASSWORD=your-clickhouse-password

# ClickHouse database name
FLEXPRICE_CLICKHOUSE_DATABASE=flexprice

# TLS/SSL configuration (set to true if using secure connection)
FLEXPRICE_CLICKHOUSE_TLS=false

# Complete ClickHouse DSN (Data Source Name) - recommended approach
# Format: clickhouse://username:password@host:port/database
# Note: Password must be URL-encoded if it contains special characters (?, ;, !, etc.)
# Example encoding: python3 -c "import urllib.parse; print(urllib.parse.quote('your-password', safe=''))"
FLEXPRICE_CLICKHOUSE_DSN=clickhouse://default:your-url-encoded-password@your-clickhouse-host:9000/flexprice

# Tenant and Environment IDs (required for proper data filtering)
# These should match your Flexprice tenant and environment
FLEXPRICE_TENANT_ID=tenant_your_tenant_id_here
FLEXPRICE_ENVIRONMENT_ID=env_your_environment_id_here

# ClickHouse Batching Configuration
# =================================
# Batch size for inserts (default: 1000, use 500 if traffic is low)
CLICKHOUSE_BATCH_COUNT=1000

# Flush period - flush every N seconds even if batch not full (default: 1s)
CLICKHOUSE_BATCH_PERIOD=1s

# Maximum parallel async inserts to ClickHouse (default: 8)
CLICKHOUSE_MAX_IN_FLIGHT=8

# AWS ECR Configuration (for Docker image publishing)
# ===================================================
# ECR registry URL (e.g., 123456789012.dkr.ecr.us-east-1.amazonaws.com)
ECR_REGISTRY=your-account-id.dkr.ecr.us-east-1.amazonaws.com

# ECR repository name
ECR_REPOSITORY=bento-collector

# AWS region for ECR (optional, defaults to us-east-1)
AWS_REGION=us-east-1

# Azure Event Hubs Input Configuration
# ====================================================
# Used by internal/azure-eh-to-flexprice.yaml (SAS) and
# internal/azure-eh-to-flexprice-mi.yaml (Managed Identity / OAUTHBEARER).

# EH namespace endpoint over the Kafka protocol: <namespace>.servicebus.windows.net:9093
AZURE_EH_BROKER=your-namespace.servicebus.windows.net:9093

# Event Hub name (the Kafka "topic")
AZURE_EH_TOPIC=your-event-hub

# Consumer group (default: flexprice-bento)
AZURE_EH_CONSUMER_GROUP=flexprice-bento

# --- Auth option A: SAS connection string (azure-eh-to-flexprice.yaml) -------
# Listen-only SAS connection string. Leave unset when using Managed Identity.
AZURE_EH_CONNECTION_STRING=Endpoint=sb://your-namespace.servicebus.windows.net/;SharedAccessKeyName=Listen;SharedAccessKey=...

# --- Auth option B: Managed Identity / OAUTHBEARER (azure-eh-to-flexprice-mi.yaml)
# Requires the collector to run on Azure compute with a User-Assigned Managed
# Identity granted the "Azure Event Hubs Data Receiver" role. No SAS key needed.
#
# EH namespace short name — used to build the AAD scope below.
AZURE_EH_NAMESPACE=your-namespace
# UAMI client ID. When set, that identity is used; when unset, DefaultAzureCredential.
AZURE_EH_MANAGED_IDENTITY_CLIENT_ID=00000000-0000-0000-0000-000000000000
# Optional explicit scope override (defaults to https://${AZURE_EH_NAMESPACE}.servicebus.windows.net/.default)
# AZURE_EH_OAUTH_SCOPE=https://your-namespace.servicebus.windows.net/.default

# =============================================================================
# AWS MSK -> Flexprice (internal/aws-kafka-to-flexprice.yaml)
# =============================================================================
# MSK bootstrap brokers. Use the SASL/SCRAM listener (port 9096), comma-free —
# one address per broker is set via the YAML list.
AWS_KAFKA_BROKERS=b-1.your-cluster.kafka.us-east-1.amazonaws.com:9096

# Topic to consume and the consumer group to join.
AWS_KAFKA_TOPIC=your-topic
AWS_KAFKA_CONSUMER_GROUP=flexprice-bento

# Start from the beginning of the topic on first run (no committed offset).
AWS_KAFKA_START_FROM_OLDEST=true

# --- Listener / auth mode ----------------------------------------------------
# These two variables select the MSK listener. The same config file serves all
# of them; nothing else needs to change.
#
#   Listener                     Port   TLS_ENABLED  SASL_MECHANISM
#   ---------------------------  -----  -----------  --------------
#   SASL/SCRAM over TLS          9096   true         SCRAM-SHA-512   (default)
#   TLS, no auth                 9094   true         none
#   PLAINTEXT (no TLS, no auth)  9092   false        none
#   SASL/PLAIN over TLS          9096   true         PLAIN
#
# PLAINTEXT sends event payloads unencrypted — VPC-internal brokers only.
# Never combine PLAIN with TLS disabled; that puts the password on the wire.
AWS_KAFKA_TLS_ENABLED=true

# SASL mechanism: SCRAM-SHA-512 | none | PLAIN | AWS_MSK_IAM
AWS_KAFKA_SASL_MECHANISM=SCRAM-SHA-512

# Credentials. Required only when the mechanism is SCRAM-* or PLAIN; leave unset
# when it is `none`. On MSK these come from the AWS Secrets Manager secret
# associated with the cluster. Inject at runtime — never commit real values.
AWS_KAFKA_SASL_USER=your_msk_scram_user
AWS_KAFKA_SASL_PASSWORD=your_msk_scram_password

# Consumer tuning (defaults shown).
AWS_KAFKA_CLIENT_ID=flexprice-bento
AWS_KAFKA_COMMIT_PERIOD=5s
AWS_KAFKA_MAX_PROCESSING_PERIOD=60s
AWS_KAFKA_FETCH_BUFFER_CAP=1000
AWS_KAFKA_CHECKPOINT_LIMIT=100

# --- Flexprice output batching: selects single vs bulk endpoint --------------
# Leaving these at the defaults below yields a no-op batch policy, so a flush
# holding one event uses the SINGLE event endpoint. Set count > 1 and/or a
# period to activate batching, which routes EVERY flush to the BULK endpoint.
#
# count must exceed 1 to activate — count=1 alone is still a no-op.
FLEXPRICE_BATCH_COUNT=1
FLEXPRICE_BATCH_PERIOD=
FLEXPRICE_BATCH_BYTE_SIZE=0

# =============================================================================
# AWS SQS -> Flexprice (internal/aws-sqs-to-flexprice.yaml)
# =============================================================================
# Full queue URL: https://sqs.<region>.amazonaws.com/<account-id>/<queue-name>
AWS_SQS_QUEUE_URL=https://sqs.us-east-1.amazonaws.com/123456789012/your-queue

# Region. Optional — the SDK infers it from the URL/environment when unset.
AWS_SQS_REGION=us-east-1

# Long-polling wait (0-20s) and messages per poll (1-10). Defaults shown.
AWS_SQS_WAIT_TIME_SECONDS=20
AWS_SQS_MAX_MESSAGES=10

# --- Credentials -------------------------------------------------------------
# Pick ONE. The queue policy must allow sqs:ReceiveMessage, sqs:DeleteMessage
# and sqs:GetQueueAttributes for the chosen identity.
#
# Option A: IAM role on the compute (preferred inside AWS — no static keys).
#   - ECS task role / EKS IRSA or Pod Identity: keep this false and leave the
#     access keys unset. The AWS SDK default chain resolves these automatically.
#   - EC2 instance profile ONLY: set this true (forces the EC2 provider).
AWS_SQS_FROM_EC2_ROLE=false
#
# Option B: static keys — only when running outside AWS. Leave unset for A.
AWS_SQS_ACCESS_KEY_ID=your_access_key_id
AWS_SQS_SECRET_ACCESS_KEY=your_secret_access_key
#
# Optional: assume-role ARN for cross-account access.
# AWS_SQS_ROLE_ARN=arn:aws:iam::123456789012:role/flexprice-sqs-reader