diff --git a/api/.env.example b/api/.env.example index 8195a3c074..40e1c2dfdf 100644 --- a/api/.env.example +++ b/api/.env.example @@ -737,24 +737,25 @@ SANDBOX_EXPIRED_RECORDS_RETENTION_DAYS=30 SANDBOX_EXPIRED_RECORDS_CLEAN_TASK_LOCK_TTL=90000 -# Redis URL used for PubSub between API and +# Redis URL used for event bus between API and # celery worker # defaults to url constructed from `REDIS_*` # configurations -PUBSUB_REDIS_URL= -# Pub/sub channel type for streaming events. -# valid options are: +EVENT_BUS_REDIS_URL= +# Event transport type. Options are: # -# - pubsub: for normal Pub/Sub -# - sharded: for sharded Pub/Sub +# - pubsub: normal Pub/Sub (at-most-once) +# - sharded: sharded Pub/Sub (at-most-once) +# - streams: Redis Streams (at-least-once, recommended to avoid subscriber races) # -# It's highly recommended to use sharded Pub/Sub AND redis cluster -# for large deployments. -PUBSUB_REDIS_CHANNEL_TYPE=pubsub -# Whether to use Redis cluster mode while running -# PubSub. +# Note: Before enabling 'streams' in production, estimate your expected event volume and retention needs. +# Configure Redis memory limits and stream trimming appropriately (e.g., MAXLEN and key expiry) to reduce +# the risk of data loss from Redis auto-eviction under memory pressure. +# Also accepts ENV: EVENT_BUS_REDIS_CHANNEL_TYPE. +EVENT_BUS_REDIS_CHANNEL_TYPE=pubsub +# Whether to use Redis cluster mode while use redis as event bus. # It's highly recommended to enable this for large deployments. -PUBSUB_REDIS_USE_CLUSTERS=false +EVENT_BUS_REDIS_USE_CLUSTERS=false # Whether to Enable human input timeout check task ENABLE_HUMAN_INPUT_TIMEOUT_TASK=true diff --git a/api/configs/middleware/cache/redis_pubsub_config.py b/api/configs/middleware/cache/redis_pubsub_config.py index 8cddc5677a..d30831a0ec 100644 --- a/api/configs/middleware/cache/redis_pubsub_config.py +++ b/api/configs/middleware/cache/redis_pubsub_config.py @@ -41,10 +41,10 @@ class RedisPubSubConfig(BaseSettings, RedisConfigDefaultsMixin): ) PUBSUB_REDIS_USE_CLUSTERS: bool = Field( - validation_alias=AliasChoices("EVENT_BUS_REDIS_CLUSTERS", "PUBSUB_REDIS_USE_CLUSTERS"), + validation_alias=AliasChoices("EVENT_BUS_REDIS_USE_CLUSTERS", "PUBSUB_REDIS_USE_CLUSTERS"), description=( "Enable Redis Cluster mode for pub/sub or streams transport. Recommended for large deployments. " - "Also accepts ENV: EVENT_BUS_REDIS_CLUSTERS." + "Also accepts ENV: EVENT_BUS_REDIS_USE_CLUSTERS." ), default=False, ) diff --git a/docker/.env.example b/docker/.env.example index 4ef856b431..9d6cd65318 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -1546,24 +1546,25 @@ SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_MAX_INTERVAL=200 SANDBOX_EXPIRED_RECORDS_RETENTION_DAYS=30 -# Redis URL used for PubSub between API and +# Redis URL used for event bus between API and # celery worker # defaults to url constructed from `REDIS_*` # configurations -PUBSUB_REDIS_URL= -# Pub/sub channel type for streaming events. -# valid options are: +EVENT_BUS_REDIS_URL= +# Event transport type. Options are: # -# - pubsub: for normal Pub/Sub -# - sharded: for sharded Pub/Sub +# - pubsub: normal Pub/Sub (at-most-once) +# - sharded: sharded Pub/Sub (at-most-once) +# - streams: Redis Streams (at-least-once, recommended to avoid subscriber races) # -# It's highly recommended to use sharded Pub/Sub AND redis cluster -# for large deployments. -PUBSUB_REDIS_CHANNEL_TYPE=pubsub -# Whether to use Redis cluster mode while running -# PubSub. +# Note: Before enabling 'streams' in production, estimate your expected event volume and retention needs. +# Configure Redis memory limits and stream trimming appropriately (e.g., MAXLEN and key expiry) to reduce +# the risk of data loss from Redis auto-eviction under memory pressure. +# Also accepts ENV: EVENT_BUS_REDIS_CHANNEL_TYPE. +EVENT_BUS_REDIS_CHANNEL_TYPE=pubsub +# Whether to use Redis cluster mode while use redis as event bus. # It's highly recommended to enable this for large deployments. -PUBSUB_REDIS_USE_CLUSTERS=false +EVENT_BUS_REDIS_USE_CLUSTERS=false # Whether to Enable human input timeout check task ENABLE_HUMAN_INPUT_TIMEOUT_TASK=true diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 689cc82a89..d14f0503e7 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -699,9 +699,9 @@ x-shared-env: &shared-api-worker-env SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_SIZE: ${SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_SIZE:-1000} SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_MAX_INTERVAL: ${SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_MAX_INTERVAL:-200} SANDBOX_EXPIRED_RECORDS_RETENTION_DAYS: ${SANDBOX_EXPIRED_RECORDS_RETENTION_DAYS:-30} - PUBSUB_REDIS_URL: ${PUBSUB_REDIS_URL:-} - PUBSUB_REDIS_CHANNEL_TYPE: ${PUBSUB_REDIS_CHANNEL_TYPE:-pubsub} - PUBSUB_REDIS_USE_CLUSTERS: ${PUBSUB_REDIS_USE_CLUSTERS:-false} + EVENT_BUS_REDIS_URL: ${EVENT_BUS_REDIS_URL:-} + EVENT_BUS_REDIS_CHANNEL_TYPE: ${EVENT_BUS_REDIS_CHANNEL_TYPE:-pubsub} + EVENT_BUS_REDIS_USE_CLUSTERS: ${EVENT_BUS_REDIS_USE_CLUSTERS:-false} ENABLE_HUMAN_INPUT_TIMEOUT_TASK: ${ENABLE_HUMAN_INPUT_TIMEOUT_TASK:-true} HUMAN_INPUT_TIMEOUT_TASK_INTERVAL: ${HUMAN_INPUT_TIMEOUT_TASK_INTERVAL:-1} SANDBOX_EXPIRED_RECORDS_CLEAN_TASK_LOCK_TTL: ${SANDBOX_EXPIRED_RECORDS_CLEAN_TASK_LOCK_TTL:-90000}