feat: add HA infrastructure, CI/CD pipeline, and Redis/Celery hardening

- Add docker-compose.ha.yml for PostgreSQL/Redis HA setup with Patroni and Sentinel
- Add docker-compose.prod.yml for production deployment
- Add GitHub Actions CI/CD workflow (build.yml)
- Add install.cmd for Windows one-click setup
- Harden Redis connection with retry logic and health checks
- Add Celery HA config with Redis Sentinel support
- Add HA operations runbook
- Update README with deployment and architecture docs
- Move landing page spec to .claude/specs/design/
- Update memory intelligence PRD
This commit is contained in:
zhangtuo
2026-02-28 14:57:02 +08:00
parent 9cdff18336
commit c82d408ea1
14 changed files with 1301 additions and 24 deletions

View File

@@ -5,11 +5,33 @@ from celery.schedules import crontab
from app.core.config import settings
celery_app = Celery(
"dreamweaver",
broker=settings.celery_broker_url,
backend=settings.celery_result_backend,
)
if settings.redis_sentinel_enabled and settings.redis_sentinel_urls:
sentinel_broker = ";".join(settings.redis_sentinel_urls)
celery_app = Celery(
"dreamweaver",
broker=sentinel_broker,
backend=sentinel_broker,
)
celery_app.conf.broker_transport_options = {
"master_name": settings.redis_sentinel_master_name,
"sentinel_kwargs": {
"password": settings.redis_sentinel_password or None,
"socket_timeout": settings.redis_sentinel_socket_timeout,
},
}
celery_app.conf.result_backend_transport_options = {
"master_name": settings.redis_sentinel_master_name,
"sentinel_kwargs": {
"password": settings.redis_sentinel_password or None,
"socket_timeout": settings.redis_sentinel_socket_timeout,
},
}
else:
celery_app = Celery(
"dreamweaver",
broker=settings.celery_broker_url,
backend=settings.celery_result_backend,
)
celery_app.conf.update(
task_track_started=True,

View File

@@ -55,6 +55,18 @@ class Settings(BaseSettings):
# Generic Redis
redis_url: str = Field("redis://localhost:6379/0", description="Redis connection URL")
redis_sentinel_enabled: bool = Field(False, description="Whether to enable Redis Sentinel")
redis_sentinel_nodes: str = Field(
"",
description="Comma-separated Redis Sentinel nodes, e.g. host1:26379,host2:26379",
)
redis_sentinel_master_name: str = Field("mymaster", description="Redis Sentinel master name")
redis_sentinel_password: str = Field("", description="Password for Redis Sentinel (optional)")
redis_sentinel_db: int = Field(0, description="Redis DB index when using Sentinel")
redis_sentinel_socket_timeout: float = Field(
0.5,
description="Socket timeout in seconds for Sentinel clients",
)
# Admin console
enable_admin_console: bool = False
@@ -71,9 +83,43 @@ class Settings(BaseSettings):
missing.append("SECRET_KEY")
if not self.database_url:
missing.append("DATABASE_URL")
if self.redis_sentinel_enabled and not self.redis_sentinel_nodes.strip():
missing.append("REDIS_SENTINEL_NODES")
if missing:
raise ValueError(f"Missing required settings: {', '.join(missing)}")
return self
@property
def redis_sentinel_hosts(self) -> list[tuple[str, int]]:
"""Parse Redis Sentinel nodes into (host, port) tuples."""
nodes = []
raw = self.redis_sentinel_nodes.strip()
if not raw:
return nodes
for item in raw.split(","):
value = item.strip()
if not value:
continue
if ":" not in value:
raise ValueError(f"Invalid sentinel node format: {value}")
host, port_text = value.rsplit(":", 1)
if not host:
raise ValueError(f"Invalid sentinel node host: {value}")
try:
port = int(port_text)
except ValueError as exc:
raise ValueError(f"Invalid sentinel node port: {value}") from exc
nodes.append((host, port))
return nodes
@property
def redis_sentinel_urls(self) -> list[str]:
"""Build Celery-compatible Sentinel URLs with DB index."""
return [
f"sentinel://{host}:{port}/{self.redis_sentinel_db}"
for host, port in self.redis_sentinel_hosts
]
settings = Settings()

View File

@@ -1,25 +1,46 @@
"""Redis client module."""
from typing import AsyncGenerator
from redis.asyncio import Redis, from_url
from redis.asyncio.sentinel import Sentinel
from app.core.config import settings
from app.core.logging import get_logger
_redis_pool: Redis | None = None
_sentinel_pool: Sentinel | None = None
logger = get_logger(__name__)
async def get_redis() -> Redis:
"""Get global Redis client instance."""
global _redis_pool
global _redis_pool, _sentinel_pool
if _redis_pool is None:
_redis_pool = from_url(settings.redis_url, encoding="utf-8", decode_responses=True)
if settings.redis_sentinel_enabled:
_sentinel_pool = Sentinel(
settings.redis_sentinel_hosts,
socket_timeout=settings.redis_sentinel_socket_timeout,
password=settings.redis_sentinel_password or None,
decode_responses=True,
)
_redis_pool = _sentinel_pool.master_for(
settings.redis_sentinel_master_name,
db=settings.redis_sentinel_db,
decode_responses=True,
)
logger.info(
"redis_connected_via_sentinel",
master_name=settings.redis_sentinel_master_name,
sentinel_nodes=settings.redis_sentinel_nodes,
)
else:
_redis_pool = from_url(settings.redis_url, encoding="utf-8", decode_responses=True)
return _redis_pool
async def close_redis():
"""Close Redis connection."""
global _redis_pool
global _redis_pool, _sentinel_pool
if _redis_pool:
await _redis_pool.close()
_redis_pool = None
_sentinel_pool = None