Remove MAX_CONCURRENT_DOWNLOADS configuration from various files and update documentation accordingly. Introduce security enhancements in the API and worker to enforce SSRF protection and client IP allowlisting. Update environment variables in .env.example and docker-compose files to reflect changes in worker settings.

asdfghj1237890 · asdfghj1237890 · commit 2e9965e91baf · 2025-12-16T00:15:11.000+08:00
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
@@ -559,7 +559,6 @@ The system deploys **2 independent download workers** by default to maximize thr
 
 **Per-Worker Configuration:**
 ```
-MAX_CONCURRENT_DOWNLOADS=3   # Jobs processed simultaneously per worker
 MAX_DOWNLOAD_WORKERS=10      # Threads per video for segment downloading
 ```
 
diff --git a/docs/SPECIFICATION.md b/docs/SPECIFICATION.md
@@ -180,7 +180,6 @@ services:
     environment:
       - REDIS_URL=redis://redis:6379
       - DATABASE_URL=postgresql://postgres:password@db:5432/m3u8_db
-      - MAX_CONCURRENT_DOWNLOADS=3
       - MAX_DOWNLOAD_WORKERS=10
     volumes:
       - /nas/downloads:/downloads
@@ -194,7 +193,6 @@ services:
     environment:
       - REDIS_URL=redis://redis:6379
       - DATABASE_URL=postgresql://postgres:password@db:5432/m3u8_db
-      - MAX_CONCURRENT_DOWNLOADS=3
       - MAX_DOWNLOAD_WORKERS=10
     volumes:
       - /nas/downloads:/downloads
@@ -401,7 +399,6 @@ API_KEY=your-secure-api-key-here
 DB_PASSWORD=your-secure-db-password-here
 # Storage is mounted to /downloads inside containers
 STORAGE_PATH=/downloads
-MAX_CONCURRENT_DOWNLOADS=3
 MAX_DOWNLOAD_WORKERS=10
 MAX_RETRY_ATTEMPTS=3
 FFMPEG_THREADS=4
diff --git a/m3u8-downloader/docker/.env.example b/m3u8-downloader/docker/.env.example
@@ -1,32 +1,52 @@
-﻿# API Configuration
-API_KEY=change-this-to-a-secure-random-key
-API_HOST=0.0.0.0
-API_PORT=8000
+﻿# WebVideo2NAS - Docker environment example
+#
+# Copy to .env and adjust values.
+#
 
-# NAS Storage Configuration
-NAS_STORAGE_PATH=/volume1/downloads/m3u8
+# =====================
+# Required
+# =====================
+# API authentication (must NOT be change-this-key)
+API_KEY=change-this-to-a-very-long-secure-key-minimum-32-chars
 
-# Database Configuration
-DATABASE_URL=postgresql://postgres:postgres_password@db:5432/m3u8_db
+# PostgreSQL password (used by db container and API/worker connection string)
+DB_PASSWORD=ChangeThisPassword123!
 
-# Redis Configuration
-REDIS_URL=redis://redis:6379/0
 
-# Download Worker Configuration
-MAX_CONCURRENT_DOWNLOADS=3
-MAX_DOWNLOAD_WORKERS=10
-MAX_RETRY_ATTEMPTS=3
-DOWNLOAD_TIMEOUT=300
+# =====================
+# Common
+# =====================
+LOG_LEVEL=INFO
 
-# FFmpeg Configuration
-FFMPEG_THREADS=4
-FFMPEG_PRESET=fast
-OUTPUT_FORMAT=mp4
+# CORS (API)
+# Example for Chrome extension:
+#   ALLOWED_ORIGINS=chrome-extension://<your-extension-id>
+ALLOWED_ORIGINS=chrome-extension://*
+# Optional: allow cookies/credentials (requires explicit origins; wildcard will be rejected)
+CORS_ALLOW_CREDENTIALS=false
 
-# Logging
-LOG_LEVEL=INFO
-LOG_FILE=/logs/app.log
 
-# Security
-ALLOWED_ORIGINS=chrome-extension://*,http://localhost:*
+# =====================
+# Performance
+# =====================
+MAX_DOWNLOAD_WORKERS=20
+MAX_RETRY_ATTEMPTS=3
+FFMPEG_THREADS=2
+
+
+# =====================
+# Security (recommended if exposed to public internet)
+# =====================
+# Per-client rate limit for protected endpoints (0 disables)
 RATE_LIMIT_PER_MINUTE=10
+
+# Restrict who can call the API (comma-separated CIDRs)
+# Examples:
+#   ALLOWED_CLIENT_CIDRS=203.0.113.8/32
+#   ALLOWED_CLIENT_CIDRS=203.0.113.8/32,198.51.100.0/24
+ALLOWED_CLIENT_CIDRS=
+
+# Basic SSRF guard for /api/download (blocks private/loopback/link-local/reserved destinations)
+# Set true for public deployments. Keep false if you intentionally download from LAN hosts.
+SSRF_GUARD=false
+
diff --git a/m3u8-downloader/docker/SYNOLOGY_DEPLOY_COMMANDS.md b/m3u8-downloader/docker/SYNOLOGY_DEPLOY_COMMANDS.md
@@ -22,7 +22,6 @@ cd /volume1/docker/m3u8-downloader/docker
 cat > .env << 'EOF'
 DB_PASSWORD=ChangeThisPassword123!
 API_KEY=change-this-to-a-very-long-secure-key-minimum-32-chars
-MAX_CONCURRENT_DOWNLOADS=3
 MAX_DOWNLOAD_WORKERS=10
 MAX_RETRY_ATTEMPTS=3
 FFMPEG_THREADS=4
@@ -318,7 +317,6 @@ docker-compose -f docker-compose.synology.yml restart
 vi /volume1/docker/m3u8-downloader/docker/.env
 
 # Change the following values:
-# MAX_CONCURRENT_DOWNLOADS=5  # Number of concurrent downloads
 # MAX_DOWNLOAD_WORKERS=15      # Number of threads per video
 
 # Restart Worker
diff --git a/m3u8-downloader/docker/api/main.py b/m3u8-downloader/docker/api/main.py
@@ -16,11 +16,16 @@
 from sqlalchemy import create_engine, text
 from sqlalchemy.orm import sessionmaker, Session
 import uuid
+import ipaddress
+import socket
 
 # Configuration
 DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@db:5432/m3u8_db")
 REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
 API_KEY = os.getenv("API_KEY")
+RATE_LIMIT_PER_MINUTE = int(os.getenv("RATE_LIMIT_PER_MINUTE", "0") or "0")
+ALLOWED_CLIENT_CIDRS_RAW = os.getenv("ALLOWED_CLIENT_CIDRS", "").strip()
+SSRF_GUARD_ENABLED = os.getenv("SSRF_GUARD", "false").strip().lower() in ("1", "true", "yes", "y", "on")
 
 # Backward-compatible default: allow all origins unless explicitly restricted.
 _allowed_origins_raw = os.getenv("ALLOWED_ORIGINS", "*").strip()
@@ -58,6 +63,102 @@
 # Redis setup
 redis_client = redis.from_url(REDIS_URL, decode_responses=True)
 
+# Security helpers
+def _get_client_ip(request: Request) -> str:
+    """
+    Best-effort client IP for rate limiting / allowlisting.
+    If you're behind a reverse proxy, ensure it is trusted before relying on X-Forwarded-For.
+    """
+    xff = (request.headers.get("x-forwarded-for") or "").strip()
+    if xff:
+        # Use the left-most (original) IP
+        return xff.split(",")[0].strip()
+    if request.client and request.client.host:
+        return request.client.host
+    return "unknown"
+
+
+def _parse_allowed_client_networks() -> list[ipaddress._BaseNetwork]:
+    if not ALLOWED_CLIENT_CIDRS_RAW:
+        return []
+    networks: list[ipaddress._BaseNetwork] = []
+    for token in [t.strip() for t in ALLOWED_CLIENT_CIDRS_RAW.split(",") if t.strip()]:
+        try:
+            networks.append(ipaddress.ip_network(token, strict=False))
+        except ValueError:
+            raise HTTPException(status_code=503, detail=f"Server misconfigured: invalid ALLOWED_CLIENT_CIDRS entry: {token}")
+    return networks
+
+
+_ALLOWED_CLIENT_NETWORKS = _parse_allowed_client_networks()
+
+
+def _enforce_client_allowlist(request: Request) -> None:
+    if not _ALLOWED_CLIENT_NETWORKS:
+        return
+    client_ip_str = _get_client_ip(request)
+    try:
+        client_ip = ipaddress.ip_address(client_ip_str)
+    except ValueError:
+        raise HTTPException(status_code=403, detail="Client IP not allowed")
+    for net in _ALLOWED_CLIENT_NETWORKS:
+        if client_ip in net:
+            return
+    raise HTTPException(status_code=403, detail="Client IP not allowed")
+
+
+def _rate_limit(request: Request, bucket: str) -> None:
+    if RATE_LIMIT_PER_MINUTE <= 0:
+        return
+    client_ip = _get_client_ip(request)
+    window = int(datetime.utcnow().timestamp() // 60)
+    key = f"rl:{bucket}:{client_ip}:{window}"
+    try:
+        count = redis_client.incr(key)
+        redis_client.expire(key, 90)
+    except Exception:
+        # If Redis is unavailable, skip rate limiting (avoid breaking core API).
+        return
+    if count > RATE_LIMIT_PER_MINUTE:
+        raise HTTPException(status_code=429, detail="Rate limit exceeded")
+
+
+def _resolve_host_ips(hostname: str) -> list[ipaddress._BaseAddress]:
+    # Resolve A/AAAA records; if it fails, we treat as invalid for SSRF protection.
+    infos = socket.getaddrinfo(hostname, None, proto=socket.IPPROTO_TCP)
+    ips: list[ipaddress._BaseAddress] = []
+    for info in infos:
+        sockaddr = info[4]
+        ip_str = sockaddr[0]
+        ips.append(ipaddress.ip_address(ip_str))
+    return ips
+
+
+def _is_ip_public(ip: ipaddress._BaseAddress) -> bool:
+    # Block common SSRF targets: loopback, link-local, RFC1918/ULA, multicast, reserved, etc.
+    if ip.is_loopback or ip.is_private or ip.is_link_local or ip.is_multicast or ip.is_reserved or ip.is_unspecified:
+        return False
+    return True
+
+
+def _enforce_ssrf_guard(url: HttpUrl) -> None:
+    if not SSRF_GUARD_ENABLED:
+        return
+    hostname = url.host
+    if not hostname:
+        raise HTTPException(status_code=400, detail="Invalid URL host")
+    if hostname.lower() in ("localhost",):
+        raise HTTPException(status_code=400, detail="URL host not allowed")
+    try:
+        ips = _resolve_host_ips(hostname)
+    except Exception:
+        raise HTTPException(status_code=400, detail="URL host could not be resolved")
+    if not ips:
+        raise HTTPException(status_code=400, detail="URL host could not be resolved")
+    for ip in ips:
+        if not _is_ip_public(ip):
+            raise HTTPException(status_code=400, detail="URL host not allowed")
+
 # Pydantic models
 class DownloadRequest(BaseModel):
     url: HttpUrl
@@ -73,6 +174,8 @@ def validate_video_url(cls, v):
         is_valid = '.m3u8' in url_str or '.mp4' in url_str
         if not is_valid:
             raise ValueError('URL must contain .m3u8 or .mp4')
+        # Optional SSRF protection for public deployments
+        _enforce_ssrf_guard(v)
         return v
 
 class JobResponse(BaseModel):
@@ -102,8 +205,10 @@ def get_db():
     finally:
         db.close()
 
-def verify_api_key(authorization: Optional[str] = Header(None)):
+def verify_api_key(request: Request, authorization: Optional[str] = Header(None)):
     """Verify API key from Authorization header"""
+    _enforce_client_allowlist(request)
+    _rate_limit(request, bucket="auth")
     if not API_KEY or API_KEY.strip() == "" or API_KEY.strip() == "change-this-key":
         raise HTTPException(status_code=503, detail="Server not configured: API_KEY is not set")
     if not authorization:
@@ -128,9 +233,15 @@ async def root():
     }
 
 @app.get("/api/health")
-async def health_check():
+async def health_check(request: Request, authorization: Optional[str] = Header(None)):
     """Health check endpoint"""
     try:
+        # Avoid exposing internal status to the public internet.
+        # Allow localhost checks (Docker healthcheck) without auth; require API key otherwise.
+        client_ip = _get_client_ip(request)
+        if client_ip not in ("127.0.0.1", "::1"):
+            verify_api_key(request=request, authorization=authorization)
+
         # Check database
         db = SessionLocal()
         db.execute(text("SELECT 1"))
diff --git a/m3u8-downloader/docker/docker-compose.synology.yml b/m3u8-downloader/docker/docker-compose.synology.yml
@@ -100,7 +100,6 @@ services:
       - DATABASE_URL=postgresql://postgres:${DB_PASSWORD:-postgres_password}@db:5432/m3u8_db
       - REDIS_URL=redis://redis:6379/0
       - LOG_LEVEL=${LOG_LEVEL:-INFO}
-      - MAX_CONCURRENT_DOWNLOADS=${MAX_CONCURRENT_DOWNLOADS:-3}
       - MAX_DOWNLOAD_WORKERS=${MAX_DOWNLOAD_WORKERS:-2}
       - MAX_RETRY_ATTEMPTS=${MAX_RETRY_ATTEMPTS:-3}
       - FFMPEG_THREADS=${FFMPEG_THREADS:-4}
@@ -130,7 +129,6 @@ services:
       - DATABASE_URL=postgresql://postgres:${DB_PASSWORD:-postgres_password}@db:5432/m3u8_db
       - REDIS_URL=redis://redis:6379/0
       - LOG_LEVEL=${LOG_LEVEL:-INFO}
-      - MAX_CONCURRENT_DOWNLOADS=${MAX_CONCURRENT_DOWNLOADS:-3}
       - MAX_DOWNLOAD_WORKERS=${MAX_DOWNLOAD_WORKERS:-2}
       - MAX_RETRY_ATTEMPTS=${MAX_RETRY_ATTEMPTS:-3}
       - FFMPEG_THREADS=${FFMPEG_THREADS:-4}
diff --git a/m3u8-downloader/docker/docker-compose.yml b/m3u8-downloader/docker/docker-compose.yml
@@ -90,7 +90,6 @@ services:
       - DATABASE_URL=postgresql://postgres:${DB_PASSWORD:-postgres_password}@db:5432/m3u8_db
       - REDIS_URL=redis://redis:6379/0
       - LOG_LEVEL=${LOG_LEVEL:-INFO}
-      - MAX_CONCURRENT_DOWNLOADS=${MAX_CONCURRENT_DOWNLOADS:-3}
       - MAX_DOWNLOAD_WORKERS=${MAX_DOWNLOAD_WORKERS:-2}
       - MAX_RETRY_ATTEMPTS=${MAX_RETRY_ATTEMPTS:-3}
       - FFMPEG_THREADS=${FFMPEG_THREADS:-4}
@@ -118,7 +117,6 @@ services:
       - DATABASE_URL=postgresql://postgres:${DB_PASSWORD:-postgres_password}@db:5432/m3u8_db
       - REDIS_URL=redis://redis:6379/0
       - LOG_LEVEL=${LOG_LEVEL:-INFO}
-      - MAX_CONCURRENT_DOWNLOADS=${MAX_CONCURRENT_DOWNLOADS:-3}
       - MAX_DOWNLOAD_WORKERS=${MAX_DOWNLOAD_WORKERS:-2}
       - MAX_RETRY_ATTEMPTS=${MAX_RETRY_ATTEMPTS:-3}
       - FFMPEG_THREADS=${FFMPEG_THREADS:-4}
diff --git a/m3u8-downloader/docker/worker/worker.py b/m3u8-downloader/docker/worker/worker.py
@@ -16,12 +16,15 @@
 from datetime import datetime
 from urllib.parse import urlparse
 import signal
+import ipaddress
+import socket
 
 # Configuration
 DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@db:5432/m3u8_db")
 REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
 LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
 MAX_RETRY_ATTEMPTS = int(os.getenv("MAX_RETRY_ATTEMPTS", "3"))
+SSRF_GUARD_ENABLED = os.getenv("SSRF_GUARD", "false").strip().lower() in ("1", "true", "yes", "y", "on")
 
 # Setup logging
 logging.basicConfig(
@@ -48,6 +51,41 @@ def signal_handler(sig, frame):
 signal.signal(signal.SIGINT, signal_handler)
 signal.signal(signal.SIGTERM, signal_handler)
 
+def _resolve_host_ips(hostname: str) -> list[ipaddress._BaseAddress]:
+    infos = socket.getaddrinfo(hostname, None, proto=socket.IPPROTO_TCP)
+    ips: list[ipaddress._BaseAddress] = []
+    for info in infos:
+        sockaddr = info[4]
+        ip_str = sockaddr[0]
+        ips.append(ipaddress.ip_address(ip_str))
+    return ips
+
+
+def _is_ip_public(ip: ipaddress._BaseAddress) -> bool:
+    if ip.is_loopback or ip.is_private or ip.is_link_local or ip.is_multicast or ip.is_reserved or ip.is_unspecified:
+        return False
+    return True
+
+
+def _enforce_ssrf_guard(url: str) -> None:
+    if not SSRF_GUARD_ENABLED:
+        return
+    parsed = urlparse(url)
+    hostname = parsed.hostname
+    if not hostname:
+        raise Exception("Invalid URL host")
+    if hostname.lower() in ("localhost",):
+        raise Exception("URL host not allowed")
+    try:
+        ips = _resolve_host_ips(hostname)
+    except Exception:
+        raise Exception("URL host could not be resolved")
+    if not ips:
+        raise Exception("URL host could not be resolved")
+    for ip in ips:
+        if not _is_ip_public(ip):
+            raise Exception("URL host not allowed")
+
 
 class DownloadWorker:
     """Worker class for processing download jobs"""
@@ -235,6 +273,8 @@ def _process_direct_download(self, job_id: str, job: dict):
         from ssl_adapter import create_legacy_session
         
         try:
+            _enforce_ssrf_guard(job["url"])
+
             # Update status to downloading
             self.update_job_status(job_id, "downloading", progress=0)
             logger.info(f"Starting direct download: {job['url']}")
@@ -587,6 +627,8 @@ def _process_m3u8_download(self, job_id: str, job: dict):
         temp_dir = None
         
         try:
+            _enforce_ssrf_guard(job["url"])
+
             # Update status to downloading
             self.update_job_status(job_id, "downloading", progress=0)
             logger.info(f"Starting m3u8 download: {job['url']}")
