feat: add HA infrastructure, CI/CD pipeline, and Redis/Celery hardening
- Add docker-compose.ha.yml for PostgreSQL/Redis HA setup with Patroni and Sentinel - Add docker-compose.prod.yml for production deployment - Add GitHub Actions CI/CD workflow (build.yml) - Add install.cmd for Windows one-click setup - Harden Redis connection with retry logic and health checks - Add Celery HA config with Redis Sentinel support - Add HA operations runbook - Update README with deployment and architecture docs - Move landing page spec to .claude/specs/design/ - Update memory intelligence PRD
This commit is contained in:
@@ -5,11 +5,33 @@ from celery.schedules import crontab
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
celery_app = Celery(
|
||||
"dreamweaver",
|
||||
broker=settings.celery_broker_url,
|
||||
backend=settings.celery_result_backend,
|
||||
)
|
||||
if settings.redis_sentinel_enabled and settings.redis_sentinel_urls:
|
||||
sentinel_broker = ";".join(settings.redis_sentinel_urls)
|
||||
celery_app = Celery(
|
||||
"dreamweaver",
|
||||
broker=sentinel_broker,
|
||||
backend=sentinel_broker,
|
||||
)
|
||||
celery_app.conf.broker_transport_options = {
|
||||
"master_name": settings.redis_sentinel_master_name,
|
||||
"sentinel_kwargs": {
|
||||
"password": settings.redis_sentinel_password or None,
|
||||
"socket_timeout": settings.redis_sentinel_socket_timeout,
|
||||
},
|
||||
}
|
||||
celery_app.conf.result_backend_transport_options = {
|
||||
"master_name": settings.redis_sentinel_master_name,
|
||||
"sentinel_kwargs": {
|
||||
"password": settings.redis_sentinel_password or None,
|
||||
"socket_timeout": settings.redis_sentinel_socket_timeout,
|
||||
},
|
||||
}
|
||||
else:
|
||||
celery_app = Celery(
|
||||
"dreamweaver",
|
||||
broker=settings.celery_broker_url,
|
||||
backend=settings.celery_result_backend,
|
||||
)
|
||||
|
||||
celery_app.conf.update(
|
||||
task_track_started=True,
|
||||
|
||||
@@ -55,6 +55,18 @@ class Settings(BaseSettings):
|
||||
|
||||
# Generic Redis
|
||||
redis_url: str = Field("redis://localhost:6379/0", description="Redis connection URL")
|
||||
redis_sentinel_enabled: bool = Field(False, description="Whether to enable Redis Sentinel")
|
||||
redis_sentinel_nodes: str = Field(
|
||||
"",
|
||||
description="Comma-separated Redis Sentinel nodes, e.g. host1:26379,host2:26379",
|
||||
)
|
||||
redis_sentinel_master_name: str = Field("mymaster", description="Redis Sentinel master name")
|
||||
redis_sentinel_password: str = Field("", description="Password for Redis Sentinel (optional)")
|
||||
redis_sentinel_db: int = Field(0, description="Redis DB index when using Sentinel")
|
||||
redis_sentinel_socket_timeout: float = Field(
|
||||
0.5,
|
||||
description="Socket timeout in seconds for Sentinel clients",
|
||||
)
|
||||
|
||||
# Admin console
|
||||
enable_admin_console: bool = False
|
||||
@@ -71,9 +83,43 @@ class Settings(BaseSettings):
|
||||
missing.append("SECRET_KEY")
|
||||
if not self.database_url:
|
||||
missing.append("DATABASE_URL")
|
||||
if self.redis_sentinel_enabled and not self.redis_sentinel_nodes.strip():
|
||||
missing.append("REDIS_SENTINEL_NODES")
|
||||
if missing:
|
||||
raise ValueError(f"Missing required settings: {', '.join(missing)}")
|
||||
return self
|
||||
|
||||
@property
|
||||
def redis_sentinel_hosts(self) -> list[tuple[str, int]]:
|
||||
"""Parse Redis Sentinel nodes into (host, port) tuples."""
|
||||
nodes = []
|
||||
raw = self.redis_sentinel_nodes.strip()
|
||||
if not raw:
|
||||
return nodes
|
||||
|
||||
for item in raw.split(","):
|
||||
value = item.strip()
|
||||
if not value:
|
||||
continue
|
||||
if ":" not in value:
|
||||
raise ValueError(f"Invalid sentinel node format: {value}")
|
||||
host, port_text = value.rsplit(":", 1)
|
||||
if not host:
|
||||
raise ValueError(f"Invalid sentinel node host: {value}")
|
||||
try:
|
||||
port = int(port_text)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"Invalid sentinel node port: {value}") from exc
|
||||
nodes.append((host, port))
|
||||
return nodes
|
||||
|
||||
@property
|
||||
def redis_sentinel_urls(self) -> list[str]:
|
||||
"""Build Celery-compatible Sentinel URLs with DB index."""
|
||||
return [
|
||||
f"sentinel://{host}:{port}/{self.redis_sentinel_db}"
|
||||
for host, port in self.redis_sentinel_hosts
|
||||
]
|
||||
|
||||
|
||||
settings = Settings()
|
||||
|
||||
@@ -1,25 +1,46 @@
|
||||
"""Redis client module."""
|
||||
|
||||
from typing import AsyncGenerator
|
||||
|
||||
from redis.asyncio import Redis, from_url
|
||||
from redis.asyncio.sentinel import Sentinel
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.logging import get_logger
|
||||
|
||||
_redis_pool: Redis | None = None
|
||||
_sentinel_pool: Sentinel | None = None
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
async def get_redis() -> Redis:
|
||||
"""Get global Redis client instance."""
|
||||
global _redis_pool
|
||||
global _redis_pool, _sentinel_pool
|
||||
if _redis_pool is None:
|
||||
_redis_pool = from_url(settings.redis_url, encoding="utf-8", decode_responses=True)
|
||||
if settings.redis_sentinel_enabled:
|
||||
_sentinel_pool = Sentinel(
|
||||
settings.redis_sentinel_hosts,
|
||||
socket_timeout=settings.redis_sentinel_socket_timeout,
|
||||
password=settings.redis_sentinel_password or None,
|
||||
decode_responses=True,
|
||||
)
|
||||
_redis_pool = _sentinel_pool.master_for(
|
||||
settings.redis_sentinel_master_name,
|
||||
db=settings.redis_sentinel_db,
|
||||
decode_responses=True,
|
||||
)
|
||||
logger.info(
|
||||
"redis_connected_via_sentinel",
|
||||
master_name=settings.redis_sentinel_master_name,
|
||||
sentinel_nodes=settings.redis_sentinel_nodes,
|
||||
)
|
||||
else:
|
||||
_redis_pool = from_url(settings.redis_url, encoding="utf-8", decode_responses=True)
|
||||
return _redis_pool
|
||||
|
||||
|
||||
async def close_redis():
|
||||
"""Close Redis connection."""
|
||||
global _redis_pool
|
||||
global _redis_pool, _sentinel_pool
|
||||
if _redis_pool:
|
||||
await _redis_pool.close()
|
||||
_redis_pool = None
|
||||
_sentinel_pool = None
|
||||
|
||||
Reference in New Issue
Block a user