diff --git a/README.md b/README.md index 5ebcb88..68c6722 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,97 @@ +## Dependencies +- Install Python packages into your virtualenv: + +``` +python3 -m venv .venv +. .venv/bin/activate +pip install -r requirements.txt +``` + +## Configuration +- Redis connection is configured via CLI flags or environment variables using `redis_connector`. +- Supported env vars: + - `REDIS_HOST` (default: localhost) + - `REDIS_PORT` (default: 6379) + - `REDIS_DB` (default: 0) + - `REDIS_USERNAME` + - `REDIS_PASSWORD` + - `REDIS_TLS` (true/1/yes/on to enable) + - `REDIS_TLS_CA_CERTS` (path to CA bundle) + +## Installation +- Use the provided installer script to set up services with proper user/paths: +```bash +sudo ./install.sh [install_user] [install_path] +``` +- Defaults: `install_user=rdispatch`, `install_path=/opt/rdispatch/RedisActionDispatcher` +- The installer will: + - Create a system user (if needed) + - Set up the virtualenv and install dependencies + - Install and configure systemd services with proper paths + - Copy the example environment file to `/etc/rdispatch.env` + +## Services +- Sample systemd units are included: `action_dispatcher.service`, `action_server_my_server_1.service`, `dispatcher_webmonitor.service`. +- Service files use placeholders (`%INSTALL_USER%`, `%INSTALL_PATH%`) that are substituted during installation. +- Hardened defaults are applied (`NoNewPrivileges`, `PrivateTmp`, `ProtectSystem=full`, `ProtectHome=true`). + +### Templated server service +- Use `action_server@.service` to avoid per-ID unit files. +- Instance name must be `SERVER_CLASS:SERVER_ID`. Examples: + - Enable and start: `systemctl enable --now action_server@my_server:1` + - Check status: `systemctl status action_server@my_server:1` + +### Web monitor behind Nginx (optional) +- The web monitor binds to `127.0.0.1:5000` by default (4 workers, 120s timeout, 600s keepalive). +- A complete Nginx configuration is provided in [nginx-dispatcher.conf](nginx-dispatcher.conf). + +**To set up Nginx:** +```bash +# Install nginx +sudo apt install nginx # Debian/Ubuntu +# or +sudo yum install nginx # RHEL/CentOS + +# Copy and edit the config +sudo cp nginx-dispatcher.conf /etc/nginx/sites-available/dispatcher +sudo nano /etc/nginx/sites-available/dispatcher # Change server_name + +# Enable the site +sudo ln -s /etc/nginx/sites-available/dispatcher /etc/nginx/sites-enabled/ + +# Test and reload +sudo nginx -t +sudo systemctl reload nginx +``` + +**Features included:** +- Upstream connection pooling with keepalive +- WebSocket/SSE support on `/stream` endpoint +- Proper proxy headers and timeouts matching Gunicorn +- Static file caching optimization +- Optional HTTPS and basic auth examples (commented) +- Health check endpoint configuration + +**Minimal example for quick setup:** +```nginx +server { + listen 80; + server_name dispatcher.local; + + location / { + proxy_pass http://127.0.0.1:5000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} +``` + +After installation, edit `/etc/rdispatch.env` for Redis configuration, then enable/start services as shown by the installer. + +## Logging +- `action_dispatcher.py` and `action_server.py` use Python `logging` with `--log-level` CLI option. RedisActionDispatcher === New distributed dispatchless implementation of MDSplus action server based on REDIS diff --git a/action_dispatcher.py b/action_dispatcher.py old mode 100644 new mode 100755 index f45d4d5..8bc1328 --- a/action_dispatcher.py +++ b/action_dispatcher.py @@ -73,6 +73,7 @@ class ActionDispatcher: def __init__(self, red): + import logging self.seqActions = {} self.depActions = {} self.dependencies = {} @@ -98,6 +99,7 @@ def __init__(self, red): self.pendingSeqActions = {} self.pendingDepActions = {} + def printTables(self): print("******Sequential Actions") print(self.seqActions) @@ -601,35 +603,37 @@ def manageNotifications(actDisp): def manageWatchdog(actDisp): actDisp.serverWatchdog() +if __name__ == "__main__": + import argparse + import logging + import signal + import redis_connector -if len(sys.argv) != 1 and len(sys.argv) != 2: - print('usage: python action_dispatcher.py [redis server]') - sys.exit(0) - -if len(sys.argv) == 1: - red = redis.Redis(host='localhost') -else: - red = redis.Redis(host=sys.argv[1]) - -act = ActionDispatcher(red) -thread = Thread(target = manageNotifications, args = (act, )) -thread.start() -threadWatch = Thread(target = manageWatchdog, args = (act, )) -threadWatch.start() -act.handleCommands() - - - + parser = argparse.ArgumentParser() + redis_connector.add_redis_args(parser) - - - - - - - + parser.add_argument("--log-level", default="INFO", choices=["DEBUG","INFO","WARNING","ERROR","CRITICAL"], help="Logging level") + args = parser.parse_args() + logging.basicConfig(level=getattr(logging, args.log_level)) + red = redis_connector.connect_redis_from_args(args, prompt_on_auth_failure=True) + logging.info("Redis connected: %s", red.ping()) + act = ActionDispatcher(red) + # Basic graceful shutdown on SIGTERM/SIGINT + def _stop(*_): + try: + act.aborted = True + except Exception: + pass + signal.signal(signal.SIGTERM, _stop) + signal.signal(signal.SIGINT, _stop) + + thread = Thread(target=manageNotifications, args=(act, )) + thread.start() + threadWatch = Thread(target=manageWatchdog, args=(act, )) + threadWatch.start() + act.handleCommands() diff --git a/action_dispatcher.service b/action_dispatcher.service new file mode 100644 index 0000000..9dbbbab --- /dev/null +++ b/action_dispatcher.service @@ -0,0 +1,25 @@ +[Unit] +Description=Action Dispatcher Service +After=network.target redis.service + +[Service] +Type=simple +User=%INSTALL_USER% +WorkingDirectory=%INSTALL_PATH% +EnvironmentFile=/etc/rdispatch.env +ExecStart=/bin/bash -lc ' \ + source /etc/profile.d/mdsplus.sh && \ + exec %INSTALL_PATH%/.venv/bin/python -u action_dispatcher.py \ +' +Restart=always +RestartSec=5 +Environment=PYTHONUNBUFFERED=1 +StandardOutput=journal +StandardError=inherit +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=full +ProtectHome=true + +[Install] +WantedBy=multi-user.target diff --git a/action_server.py b/action_server.py old mode 100644 new mode 100755 index 4f27edc..f4faa6f --- a/action_server.py +++ b/action_server.py @@ -402,14 +402,13 @@ def reportServerOn(red, ident, id): red.hset('ACTION_SERVER_ACTIVE:'+ident, id, 'ON') time.sleep(1) -def main(serverClass, serverId, redisServer, sequential, process): - red = redis.Redis(host=redisServer) +def main(redisObject, serverClass, serverId, sequential, process): + import logging + red = redisObject ident = serverClass id = serverId - print('Action server started. Server class: '+ident+', Server Id: '+id) + logging.info('Action server started. Server class: %s, Server Id: %s', ident, id) act = ActionServer(ident, id, red) - #atexit.register(reportExit, red, ident, id) - #red.hset('ACTION_SERVER_ACTIVE:'+ident, id, 'ON') thread = threading.Thread(target = reportServerOn, args = (red, ident, id,)) thread.start() act.handleCommands(sequential != 0, process != 0) @@ -417,42 +416,23 @@ def main(serverClass, serverId, redisServer, sequential, process): if __name__ == "__main__": + import argparse + import logging + import redis_connector parser = argparse.ArgumentParser() - - # positional argument + redis_connector.add_redis_args(parser) + parser.add_argument("serverClass", help="Server Class") parser.add_argument("serverId", help="ServerId") - parser.add_argument("redisServer", help="REDIS server") parser.add_argument("--sequential", type=int, default=1, help="Force Mutual exclusion for log consistency") parser.add_argument("--process", type=int, default=0, help="Force Mutual exclusion for log consistency") - args = parser.parse_args() - print(args.serverClass, args.serverId, args.redisServer, args.sequential, args.process) - main(args.serverClass, args.serverId, args.redisServer, args.sequential, args.process) - - - - - - - - - - - - - - - - - - - - - - - - + parser.add_argument("--log-level", default="INFO", choices=["DEBUG","INFO","WARNING","ERROR","CRITICAL"], help="Logging level") + args = parser.parse_args() + logging.basicConfig(level=getattr(logging, args.log_level)) + red = redis_connector.connect_redis_from_args(args) + logging.info("Redis connected: %s", red.ping()) + main(red, args.serverClass, args.serverId, args.sequential, args.process) diff --git a/action_server@.service b/action_server@.service new file mode 100644 index 0000000..3262e50 --- /dev/null +++ b/action_server@.service @@ -0,0 +1,24 @@ +[Unit] +Description=Action Server (templated) — Class:ID from instance name +After=network.target redis.service + +[Service] +Type=simple +User=%INSTALL_USER% +WorkingDirectory=%INSTALL_PATH% +EnvironmentFile=/etc/rdispatch.env +# Usage example: systemctl start action_server@my_server:1 +# This shell trick splits %i as CLASS:ID into two args +ExecStart=/bin/sh -c '. /etc/profile.d/mdsplus.sh; exec %INSTALL_PATH%/.venv/bin/python -u action_server.py "${0%%:*}" "${0##*:}"' %i +Restart=always +RestartSec=5 +Environment=PYTHONUNBUFFERED=1 +StandardOutput=journal +StandardError=inherit +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=full +ProtectHome=true + +[Install] +WantedBy=multi-user.target diff --git a/dispatcher_command.py b/dispatcher_command.py old mode 100644 new mode 100755 diff --git a/dispatcher_monitor.py b/dispatcher_monitor.py old mode 100644 new mode 100755 diff --git a/dispatcher_webmonitor.py b/dispatcher_webmonitor.py old mode 100644 new mode 100755 index 4791506..0a3a09b --- a/dispatcher_webmonitor.py +++ b/dispatcher_webmonitor.py @@ -10,12 +10,17 @@ import json +import redis_connector +import logging + LOG_FILE = "redis_pubsub.log" -redishost= os.getenv("REDIS_HOST","localhost") #Permits setting a different redishost using env var - default is "localhost" -print(f"Redis host set to: {redishost}") +#redishost= os.getenv("REDIS_HOST","localhost") #Permits setting a different redishost using env var - default is "localhost" +#print(f"Redis host set to: {redishost}") app = Flask(__name__) -client = redis.StrictRedis(redishost, port=6379, decode_responses=True) +client = redis_connector.connect_from_env(decode_responses=True) +redishost = client.connection_pool.connection_kwargs.get("host") +logging.getLogger(__name__).info("Redis host set to: %s", redishost) redis_client = client lock = Lock() @@ -100,7 +105,7 @@ def message_handler(message): def log_all_channels(): setup_logger() - client = redis.StrictRedis(redishost, port=6379, decode_responses=True) + client = redis_connector.connect_from_env(decode_responses=True) pubsub = client.pubsub() channels = client.pubsub_channels() @@ -125,7 +130,7 @@ def log_all_channels(): def server_list(): setup_logger() - client = redis.StrictRedis(redishost, port=6379, decode_responses=True) + client = redis_connector.connect_from_env(decode_responses=True) pubsub = client.pubsub() channels = client.pubsub_channels() @@ -1020,7 +1025,7 @@ def publish_message(): #Initialize red to be able to call functions from dispatcher_monitor.py and action_server.py - red = redis.Redis(redishost) + red = redis_connector.connect_from_env() app.run(debug=True, host='0.0.0.0', port=3000, threaded=True) diff --git a/dispatcher_webmonitor.service b/dispatcher_webmonitor.service new file mode 100644 index 0000000..595adad --- /dev/null +++ b/dispatcher_webmonitor.service @@ -0,0 +1,31 @@ +[Unit] +Description=Dispatcher Web Monitor (Gunicorn) +After=network.target redis.service + +[Service] +Type=simple +User=%INSTALL_USER% +WorkingDirectory=%INSTALL_PATH% +EnvironmentFile=/etc/rdispatch.env +ExecStart=/bin/bash -lc ' \ + source /etc/profile.d/mdsplus.sh && \ + exec %INSTALL_PATH%/.venv/bin/gunicorn \ + -w 4 \ + -k gevent \ + --keep-alive 600 \ + --timeout 120 \ + -b 127.0.0.1:5000 \ + dispatcher_webmonitor:app \ +' +Restart=always +RestartSec=5 +Environment=PYTHONUNBUFFERED=1 +StandardOutput=journal +StandardError=inherit +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=full +ProtectHome=true + +[Install] +WantedBy=multi-user.target diff --git a/display_logs.py b/display_logs.py old mode 100644 new mode 100755 diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..0d23667 --- /dev/null +++ b/install.sh @@ -0,0 +1,161 @@ +#!/bin/bash +# Installer for RedisActionDispatcher systemd services +# Usage: sudo ./install.sh [install_user] [install_group] [install_path] + +set -e + +usage() { + cat <<'EOF' +Usage: sudo ./install.sh [install_user] [install_group] [install_path] + +Arguments (all optional): + install_user System user to own and run services (default: rdispatch) + install_group Group for files and services (default: rdispatch) + install_path Install location (default: /opt/rdispatch/RedisActionDispatcher) + +Environment flags: + SKIP_DISPATCHER=1 Skip installing action_dispatcher.service (non-interactive) + SKIP_WEB=1 Skip installing dispatcher_webmonitor.service (non-interactive) + +Interactive prompts: + - Whether to install action_dispatcher.service (if SKIP_DISPATCHER not set) + - Whether to install dispatcher_webmonitor.service (if SKIP_WEB not set) + - Which action_server@ instances to enable/start (comma-separated class:id) + +Examples: + sudo ./install.sh # defaults with prompts + SKIP_DISPATCHER=1 sudo ./install.sh # non-interactive skip dispatcher + SKIP_WEB=1 sudo ./install.sh # non-interactive skip web monitor +EOF +} + +if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then + usage + exit 0 +fi + +INSTALL_USER="${1:-rdispatch}" +INSTALL_GROUP="${2:-rdispatch}" +INSTALL_PATH="${3:-/opt/rdispatch/RedisActionDispatcher}" +SKIP_DISPATCHER="${SKIP_DISPATCHER:-0}" +SKIP_WEB="${SKIP_WEB:-0}" +SYSTEMD_DIR="/etc/systemd/system" +ENV_FILE="/etc/rdispatch.env" + +echo "==> RedisActionDispatcher Service Installer" +echo " Install user: $INSTALL_USER" +echo " Install group: $INSTALL_GROUP" +echo " Install path: $INSTALL_PATH" +echo " Skip action_dispatcher.service: $SKIP_DISPATCHER" +echo " Skip dispatcher_webmonitor.service: $SKIP_WEB" +echo "" + +# Check if running as root +if [ "$EUID" -ne 0 ]; then + echo "Error: This script must be run as root (use sudo)" + exit 1 +fi + +# Check if user exists, create if needed +if ! id "$INSTALL_USER" &>/dev/null; then + echo "==> Creating system user: $INSTALL_USER" + useradd --system --shell /bin/bash --home-dir "$INSTALL_PATH" --create-home "$INSTALL_USER" +else + echo "==> User $INSTALL_USER already exists" +fi + +# Create install directory if needed +if [ ! -d "$INSTALL_PATH" ]; then + echo "==> Creating install directory: $INSTALL_PATH" + mkdir -p "$INSTALL_PATH" +fi + +# Copy files if not installing in current directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [ "$SCRIPT_DIR" != "$INSTALL_PATH" ]; then + echo "==> Copying files to $INSTALL_PATH" + cp -r "$SCRIPT_DIR"/* "$INSTALL_PATH/" + chown -R "$INSTALL_USER":"$INSTALL_GROUP" "$INSTALL_PATH" +else + echo "==> Installing from current directory" + chown -R "$INSTALL_USER":"$INSTALL_GROUP" "$INSTALL_PATH" +fi + +# Create virtualenv if it doesn't exist +if [ ! -d "$INSTALL_PATH/.venv" ]; then + echo "==> Creating Python virtual environment" + sudo -u "$INSTALL_USER" python3 -m venv "$INSTALL_PATH/.venv" + sudo -u "$INSTALL_USER" "$INSTALL_PATH/.venv/bin/python" -m pip install --upgrade pip + sudo -u "$INSTALL_USER" "$INSTALL_PATH/.venv/bin/python" -m pip install -r "$INSTALL_PATH/requirements.txt" + sudo -u "$INSTALL_USER" echo "/usr/local/mdsplus/python" > $INSTALL_PATH/.venv/lib/$(python3 --version 2>&1 | awk '{split($2,v,"."); print "python" v[1] "." v[2]}')/site-packages/mdsplus.pth +else + echo "==> Virtual environment exists, updating dependencies" + sudo -u "$INSTALL_USER" "$INSTALL_PATH/.venv/bin/python" -m pip install -r "$INSTALL_PATH/requirements.txt" + sudo -u "$INSTALL_USER" echo "/usr/local/mdsplus/python" > $INSTALL_PATH/.venv/lib/$(python3 --version 2>&1 | awk '{split($2,v,"."); print "python" v[1] "." v[2]}')/site-packages/mdsplus.pth +fi + +# Install environment file if it doesn't exist +if [ ! -f "$ENV_FILE" ]; then + echo "==> Installing environment file: $ENV_FILE" + cp "$INSTALL_PATH/rdispatch.env.example" "$ENV_FILE" + chmod 600 "$ENV_FILE" + echo " ⚠️ Edit $ENV_FILE to configure Redis connection" +else + echo "==> Environment file $ENV_FILE already exists (not overwriting)" +fi + +# Install systemd service files (prompt for web monitor if not predefined) +if [ -z "${SKIP_WEB+x}" ]; then + read -r -p "Install dispatcher_webmonitor.service? [Y/n]: " _ans_web + case "$_ans_web" in + [nN]*) SKIP_WEB=1 ;; + *) SKIP_WEB=0 ;; + esac +fi + +echo "==> Installing systemd service files" +for service_file in "$INSTALL_PATH"/*.service; do + if [ -f "$service_file" ]; then + service_name=$(basename "$service_file") + if [ "$SKIP_DISPATCHER" = "1" ] && [ "$service_name" = "action_dispatcher.service" ]; then + echo " - skipping $service_name (SKIP_DISPATCHER=1)" + continue + fi + if [ "$SKIP_WEB" = "1" ] && [ "$service_name" = "dispatcher_webmonitor.service" ]; then + echo " - skipping $service_name (SKIP_WEB=1)" + continue + fi + echo " - $service_name" + # Substitute placeholders + sed -e "s|%INSTALL_USER%|$INSTALL_USER|g" \ + -e "s|%INSTALL_PATH%|$INSTALL_PATH|g" \ + "$service_file" > "$SYSTEMD_DIR/$service_name" + chmod 644 "$SYSTEMD_DIR/$service_name" + fi +done + +# Reload systemd +echo "==> Reloading systemd daemon" +systemctl daemon-reload + +echo "" +echo "✅ Installation complete!" +echo "" +echo "Next steps:" +echo " 1. Edit $ENV_FILE with your Redis configuration" +echo " 2. Enable and start services:" +echo " systemctl enable --now action_dispatcher # omit if SKIP_DISPATCHER=1" +echo " systemctl enable --now action_server@my_server:1" +echo " systemctl enable --now dispatcher_webmonitor # omit if SKIP_WEB=1" +echo " 3. Check status:" +echo " systemctl status action_dispatcher" +echo " journalctl -u action_dispatcher -f" +echo "" +echo "Optional - Nginx reverse proxy:" +echo " 1. Install nginx: apt install nginx (Debian/Ubuntu) or yum install nginx (RHEL/CentOS)" +echo " 2. Copy config: cp $INSTALL_PATH/nginx-dispatcher.conf /etc/nginx/sites-available/dispatcher" +echo " 3. Edit server_name in /etc/nginx/sites-available/dispatcher" +echo " 4. Enable: ln -s /etc/nginx/sites-available/dispatcher /etc/nginx/sites-enabled/" +echo " 5. Test: nginx -t" +echo " 6. Reload: systemctl reload nginx" +echo "" diff --git a/nginx-dispatcher.conf b/nginx-dispatcher.conf new file mode 100644 index 0000000..701f81b --- /dev/null +++ b/nginx-dispatcher.conf @@ -0,0 +1,112 @@ +# Nginx configuration for RedisActionDispatcher web monitor +# Install to: /etc/nginx/sites-available/dispatcher +# Enable: ln -s /etc/nginx/sites-available/dispatcher /etc/nginx/sites-enabled/ +# Test: nginx -t +# Reload: systemctl reload nginx + +upstream dispatcher_backend { + server 127.0.0.1:5000 fail_timeout=10s max_fails=3; + keepalive 32; +} + +server { + listen 80; + server_name dispatcher.local localhost; # Change first term to domain/hostname + + # Uncomment for HTTPS (requires SSL certificates) + # listen 443 ssl http2; + # ssl_certificate /etc/ssl/certs/dispatcher.crt; + # ssl_certificate_key /etc/ssl/private/dispatcher.key; + # ssl_protocols TLSv1.2 TLSv1.3; + # ssl_ciphers HIGH:!aNULL:!MD5; + + # Logging + access_log /var/log/nginx/dispatcher_access.log; + error_log /var/log/nginx/dispatcher_error.log; + + # Client request limits + client_max_body_size 10M; + client_body_timeout 60s; + + # Proxy settings + location / { + proxy_pass http://dispatcher_backend; + + # Standard proxy headers + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Port $server_port; + + # HTTP/1.1 for keepalive + proxy_http_version 1.1; + proxy_set_header Connection ""; + + # Timeouts (match Gunicorn settings) + proxy_connect_timeout 10s; + proxy_send_timeout 120s; + proxy_read_timeout 120s; + + # Buffer settings for better performance + proxy_buffering on; + proxy_buffer_size 4k; + proxy_buffers 8 4k; + proxy_busy_buffers_size 8k; + } + + # WebSocket support (if your app uses EventSource/WebSockets) + location /stream { + proxy_pass http://dispatcher_backend; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # WebSocket specific headers + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + # Longer timeouts for streaming connections + proxy_connect_timeout 10s; + proxy_send_timeout 3600s; + proxy_read_timeout 3600s; + + # Disable buffering for streaming + proxy_buffering off; + proxy_cache off; + } + + # Optional: Basic auth for additional security + # location / { + # auth_basic "Dispatcher Access"; + # auth_basic_user_file /etc/nginx/.htpasswd; + # proxy_pass http://dispatcher_backend; + # # ... include other proxy settings + # } + + # Static files optimization (if you serve static assets) + location ~* \.(jpg|jpeg|png|gif|ico|css|js|svg|woff|woff2|ttf|eot)$ { + proxy_pass http://dispatcher_backend; + proxy_cache_valid 200 1d; + add_header Cache-Control "public, immutable"; + } + + # Health check endpoint (optional) + location /health { + access_log off; + proxy_pass http://dispatcher_backend; + proxy_connect_timeout 2s; + proxy_read_timeout 2s; + } +} + +# Optional: Redirect HTTP to HTTPS +# server { +# listen 80; +# server_name dispatcher.local; +# return 301 https://$host$request_uri; +# } diff --git a/rdispatch.env.example b/rdispatch.env.example new file mode 100644 index 0000000..d9286cd --- /dev/null +++ b/rdispatch.env.example @@ -0,0 +1,10 @@ +# Example environment for RedisActionDispatcher (copy to /etc/rdispatch.env) +# Redis base settings +REDIS_HOST=localhost +REDIS_PORT=6379 +REDIS_DB=0 +# REDIS_USERNAME= +# REDIS_PASSWORD= +# TLS (optional) +# REDIS_TLS=true +# REDIS_TLS_CA_CERTS=/etc/ssl/certs/ca-certificates.crt diff --git a/redis_connector.py b/redis_connector.py new file mode 100644 index 0000000..b7c02cc --- /dev/null +++ b/redis_connector.py @@ -0,0 +1,171 @@ +import os +import sys +import getpass +import redis + + +def add_redis_args(parser): + g = parser.add_argument_group("redis") + g.add_argument("--redis-host", default=os.getenv("REDIS_HOST", "localhost")) + g.add_argument("--redis-port", type=int, default=int(os.getenv("REDIS_PORT", "6379"))) + g.add_argument("--redis-db", type=int, default=int(os.getenv("REDIS_DB", "0"))) + + g.add_argument("--redis-username", default=os.getenv("REDIS_USERNAME")) + g.add_argument("--redis-password-file", + help="Read password from file, or '-' for stdin") + g.add_argument("--prompt-password", action="store_true", + help="Prompt for password if authentication fails") + + g.add_argument("--tls", action="store_true") + g.add_argument("--tls-ca-certs", default=os.getenv("REDIS_TLS_CA_CERTS")) + + g.add_argument("--redis-opt", action="append", default=[], + metavar="KEY=VALUE", + help="Extra redis.Redis kwargs (repeatable)") + + +def _parse_opts(opts): + out = {} + for s in opts: + k, v = s.split("=", 1) + vl = v.lower() + if vl in ("true", "1", "yes", "on"): + v = True + elif vl in ("false", "0", "no", "off"): + v = False + else: + try: + v = int(v) + except ValueError: + try: + v = float(v) + except ValueError: + pass + out[k] = v + return out + + +def _get_password(args): + if args.redis_password_file: + if args.redis_password_file == "-": + return sys.stdin.read().strip() or None + try: + return open(args.redis_password_file, "r").read().strip() or None + except (FileNotFoundError, PermissionError) as e: + raise RuntimeError(f"Unable to read redis password file '{args.redis_password_file}': {e}") + return os.getenv("REDIS_PASSWORD") + + +def _get_common_kwargs(host, port, db, username, password, tls, tls_ca_certs, extra_opts=None): + kw = dict( + host=host, + port=port, + db=db, + username=username, + password=password, + ) + if extra_opts: + kw.update(_parse_opts(extra_opts)) + + if tls: + kw["ssl"] = True + if tls_ca_certs: + kw["ssl_cert_reqs"] = "required" + kw["ssl_ca_certs"] = tls_ca_certs + else: + kw["ssl_cert_reqs"] = "none" + + return kw + + +def connect_redis_from_args(args, *, prompt_on_auth_failure=False, **overrides): + password = _get_password(args) + + kw = _get_common_kwargs( + host=args.redis_host, + port=args.redis_port, + db=args.redis_db, + username=args.redis_username, + password=password, + tls=args.tls, + tls_ca_certs=args.tls_ca_certs, + extra_opts=args.redis_opt + ) + + kw.update(overrides) + kw = {k: v for k, v in kw.items() if v is not None} + + r = redis.Redis(**kw) + + try: + r.ping() + return r + except redis.exceptions.AuthenticationError: + if prompt_on_auth_failure and args.prompt_password and sys.stdin.isatty(): + pw = getpass.getpass("Redis password: ") + r = redis.Redis(**{**kw, "password": pw}) + r.ping() + return r + raise + except (redis.exceptions.ConnectionError, redis.exceptions.TimeoutError, redis.exceptions.RedisError) as e: + raise RuntimeError(f"Failed to connect to Redis at {kw.get('host')}:{kw.get('port')} db={kw.get('db')}: {e}") + + +def connect_from_env(**overrides): + """ + Connect to Redis using configuration from environment variables. + Supported variables: + REDIS_HOST (default: localhost) + REDIS_PORT (default: 6379) + REDIS_DB (default: 0) + REDIS_USERNAME + REDIS_PASSWORD + REDIS_TLS_CA_CERTS + REDIS_TLS (true/1/yes/on to enable) + """ + host = os.getenv("REDIS_HOST", "localhost") + port = int(os.getenv("REDIS_PORT", "6379")) + db = int(os.getenv("REDIS_DB", "0")) + username = os.getenv("REDIS_USERNAME") + password = os.getenv("REDIS_PASSWORD") + + tls_val = os.getenv("REDIS_TLS", "").lower() + tls = tls_val in ("true", "1", "yes", "on") + tls_ca_certs = os.getenv("REDIS_TLS_CA_CERTS") + + kw = _get_common_kwargs( + host=host, + port=port, + db=db, + username=username, + password=password, + tls=tls, + tls_ca_certs=tls_ca_certs + ) + + kw.update(overrides) + kw = {k: v for k, v in kw.items() if v is not None} + + r = redis.Redis(**kw) + try: + r.ping() + return r + except (redis.exceptions.AuthenticationError, redis.exceptions.ConnectionError, redis.exceptions.TimeoutError, redis.exceptions.RedisError) as e: + raise RuntimeError(f"Failed to connect to Redis at {kw.get('host')}:{kw.get('port')} db={kw.get('db')}: {e}") + +# ------------------------- +# Example usage +# ------------------------- +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + add_redis_args(parser) + + # Add your other arguments here... + parser.add_argument("--channel", default="mychannel") + + args = parser.parse_args() + + r = connect_redis_from_args(args) + print("Connected:", r.ping()) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..78a2f7e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +# Python dependencies for RedisActionDispatcher +redis>=4.5 +Flask>=2.3 +gunicorn>=21.2 +gevent>=23.9 +numpy diff --git a/server_kill.py b/server_kill.py old mode 100644 new mode 100755 diff --git a/server_start.py b/server_start.py old mode 100644 new mode 100755 diff --git a/show_log.py b/show_log.py old mode 100644 new mode 100755 index 5ebc4ae..651cd23 --- a/show_log.py +++ b/show_log.py @@ -1,26 +1,38 @@ -#!/usr/bin/python3 -u +#!/usr/bin/env python +import argparse +import redis_connector import redis import os import sys +import logging import MDSplus +parser = argparse.ArgumentParser() +redis_connector.add_redis_args(parser) -if len(sys.argv) != 5 and len(sys.argv) != 6: - print('usage: python show_log.py [redis server]') - sys.exit(0) -if len(sys.argv) == 4: - red = redis.Redis(host='localhost') -else: - red = redis.Redis(host=sys.argv[4]) +parser.add_argument("tree", type=str, default="test", help="Name of Tree") +parser.add_argument("shot", type=int, default=0, help="Shot number") +parser.add_argument("action", type=str, default="act1", help="Path of action to display") +parser.add_argument("--log-level", default="INFO", choices=["DEBUG","INFO","WARNING","ERROR","CRITICAL"], help="Logging level") + +args = parser.parse_args() + +logging.basicConfig(level=getattr(logging, args.log_level)) -treeName = sys.argv[1].upper() -shotStr = sys.argv[2] -t = MDSplus.Tree(treeName, -1) +red = redis_connector.connect_redis_from_args(args) +logging.info("Redis connected: %s", red.ping()) + +treeName = args.tree +shot = args.shot +t = MDSplus.Tree(treeName, shot) try: - actionPath = t.getNode(sys.argv[3]).getFullPath() + actionPath = t.getNode(args.action).getFullPath() except: - print('Cannot find node') - sys.exit(0) + logging.error('Cannot find node: %s', args.action) + sys.exit(1) -log = red.hget('ACTION_LOG:'+treeName+':'+shotStr, actionPath) -print(log.decode('utf-8')) +log = red.hget('ACTION_LOG:'+treeName+':'+str(shot), actionPath) +if log: + print(log.decode('utf-8')) +else: + logging.warning('No log found for %s:%s %s', treeName, shot, actionPath)