Merge branch 'main' and resolve conflicts in submitter.py, parser.py and api.js

Author: atharvakarval

form-flow-backend/main.py

@@ -1,36 +1,20 @@
-"""
-Form Flow AI - Backend Application
-
-FastAPI application for voice-powered form automation.
-Provides endpoints for form scraping, voice processing, and form submission.
-
-Features:
-    - Form URL scraping with Playwright
-    - Voice-to-text with Vosk
-    - Text-to-speech with ElevenLabs
-    - AI-powered form field understanding with Gemini
-    - Automated form submission
-
-Run:
-    python main.py
-    # or
-    uvicorn main:app --reload
-"""
-
-
-import warnings
 import sys
 import asyncio
 
 # Fix for Playwright on Windows - ProactorEventLoop required for subprocess
-if sys.platform == 'win32' and 'pytest' not in sys.modules:
+if sys.platform == 'win32':
     asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+    # Explicitly confirm policy application
+    print(f"🔧 [Windows] Event loop policy set to: {type(asyncio.get_event_loop_policy()).__name__}")
+
+import warnings
 
 # Suppress Pydantic V1 warning from LangChain
 warnings.filterwarnings("ignore", message=".*Core Pydantic V1 functionality.*")
 
 from contextlib import asynccontextmanager
 from fastapi import FastAPI, Request
+from fastapi.exceptions import RequestValidationError
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
 from slowapi.errors import RateLimitExceeded
@@ -67,6 +51,15 @@ async def lifespan(app: FastAPI):
     logger.info(f"Starting {settings.APP_NAME} v{settings.APP_VERSION}")
     logger.info(f"Debug mode: {settings.DEBUG}")
 
+    # Log loop type for debugging
+    try:
+        current_loop = asyncio.get_running_loop()
+        logger.info(f"Running on event loop: {type(current_loop).__name__}")
+        if sys.platform == 'win32' and 'Proactor' not in type(current_loop).__name__:
+            logger.warning("❌ WARNING: Not using ProactorEventLoop on Windows. Subprocesses (Playwright) will fail!")
+    except Exception as e:
+        logger.warning(f"Could not determine running loop type: {e}")
+    
     # Create database tables
     async with database.engine.begin() as conn:
         await conn.run_sync(models.Base.metadata.create_all)
@@ -174,6 +167,15 @@ async def formflow_exception_handler(request: Request, exc: FormFlowError):
 # Rate limit exceeded handler
 app.add_exception_handler(RateLimitExceeded, rate_limit_exceeded_handler)
 
+# Log Pydantic/validation errors with full detail to diagnose 422s quickly
+@app.exception_handler(RequestValidationError)
+async def validation_exception_handler(request: Request, exc: RequestValidationError):
+    logger.error(f"❌ Validation error at {request.url.path}: {exc.errors()} | body={getattr(exc, 'body', 'n/a')}")
+    return JSONResponse(
+        status_code=422,
+        content={"detail": exc.errors()}
+    )
+
 
 # =============================================================================
 # Routers
@@ -332,5 +334,6 @@ async def metrics_dashboard():
         host="0.0.0.0",
         port=8001,
         reload=settings.DEBUG,
+        reload_excludes=["models/**", "**/__pycache__/**", ".venv/**", "chroma_db/**", "storage/**"],
         log_level="debug" if settings.DEBUG else "info"
     )

form-flow-backend/requirements.txt

@@ -76,8 +76,6 @@ python-docx>=1.1.0
 # Voice Processing
 numpy>=1.24.0
 webrtcvad>=2.0.10
-edge-tts>=7.0.0  # Free Microsoft TTS fallback
-
-# Logging
+edge-tts>=7.0.0  
 structlog>=23.1.0
 

form-flow-backend/routers/forms.py

@@ -23,11 +23,14 @@
 from config.settings import settings
 from sqlalchemy.future import select
 from services.ai.profile.service import generate_profile_background
-from utils.api_cache import get_cached_form_schema, cache_form_schema
+from utils.api_cache import get_cached_form_schema, cache_form_schema, invalidate_form_cache
+
+SUBMIT_TIMEOUT_SECONDS = 30
 
 # --- Pydantic Models ---
 class ScrapeRequest(BaseModel):
     url: str
+    refresh: bool = False  # Cache-busting: set True to force re-scrape
 
 class VoiceProcessRequest(BaseModel):
     transcript: str
@@ -178,7 +181,7 @@ async def _run_magic_fill_background(
                 if gemini_service:
                     t_mf = _time.time()
                     print("✨ [Background] Magic Fill starting...")
-                    filler = SmartFormFillerChain(gemini_service.llm)
+                    filler = SmartFormFillerChain(gemini_service)
                     magic_result = await filler.fill(
                         user_profile=user_profile,
                         form_schema=form_schema,
@@ -298,20 +301,29 @@ async def scrape_form(
 
         print(f"Normalized URL: {url}")
 
-        # ━━━ CACHE CHECK ━━━
-        try:
-            cached = await get_cached_form_schema(url)
-            if cached:
-                print(f"✅ Cache HIT for {url} — returning instantly")
-                return {
-                    "message": "Form loaded from cache",
-                    **cached,
-                    "cached": True,
-                    "gemini_ready": gemini_service is not None,
-                    "timing": {"total": round(_time.time() - t0, 2)}
-                }
-        except Exception as e:
-            print(f"⚠️ Cache lookup failed (proceeding without cache): {e}")
+        # ━━━ CACHE BUSTING ━━━
+        if data.refresh:
+            print(f"🔄 Cache bust requested for {url}")
+            try:
+                await invalidate_form_cache(url)
+            except Exception as e:
+                print(f"⚠️ Cache invalidation failed: {e}")
+        
+        # ━━━ CACHE CHECK (with validation guard) ━━━
+        if not data.refresh:
+            try:
+                cached = await get_cached_form_schema(url)
+                if cached:
+                    print(f"✅ Cache HIT for {url} — returning instantly")
+                    return {
+                        "message": "Form loaded from cache",
+                        **cached,
+                        "cached": True,
+                        "gemini_ready": gemini_service is not None,
+                        "timing": {"total": round(_time.time() - t0, 2)}
+                    }
+            except Exception as e:
+                print(f"⚠️ Cache lookup failed (proceeding without cache): {e}")
 
         # ━━━ SCRAPE + PROCESS (parallel smart prompts + hybrid TTS) ━━━
         t1 = _time.time()
@@ -341,9 +353,9 @@ async def scrape_form(
             "magic_fill_status": "processing" if settings.ENABLE_AI and auth_header and auth_header.startswith('Bearer ') else "skipped"
         }
 
-        # ━━━ CACHE RESULT (30 min TTL) ━━━
+        # ━━━ CACHE RESULT (30 min TTL, validated) ━━━
         try:
-            # Cache processed data (excluding magic fill — that's user-specific)
+            # cache_form_schema validates internally — won't cache 0-field schemas
             await cache_form_schema(url, processed_data, ttl=1800)
         except Exception as e:
             print(f"⚠️ Failed to cache result: {e}")
@@ -633,7 +645,7 @@ async def magic_fill(
         if not gemini_service:
             raise HTTPException(status_code=500, detail="Gemini service not available")
 
-        filler = SmartFormFillerChain(gemini_service.llm)
+        filler = SmartFormFillerChain(gemini_service)
         result = await filler.fill(
             user_profile=user_profile,
             form_schema=data.form_schema,
@@ -666,6 +678,8 @@ async def submit_form(
         schema = get_schema(data.url, form_data=data.form_schema)
         formatted_data = data.form_data
 
+        submit_coro = None
+
         if schema:
             formatted_data = schema.format_all(data.form_data)
             valid, errors = schema.validate_all(formatted_data)
@@ -677,21 +691,28 @@ async def submit_form(
                     "submitted_data": formatted_data
                 }
 
-            result = await form_submitter.submit_form_data(
+            submit_coro = form_submitter.submit_form_data(
                 url=data.url,
                 form_data=formatted_data,
                 form_schema=data.form_schema,
                 use_cdp=data.use_cdp,
                 human_like=data.human_like
             )
         else:
-            result = await form_submitter.submit_form_data(
+            submit_coro = form_submitter.submit_form_data(
                 url=data.url,
                 form_data=data.form_data,
                 form_schema=data.form_schema,
                 use_cdp=data.use_cdp,
                 human_like=data.human_like
             )
+            formatted_data = data.form_data
+
+        try:
+            result = await asyncio.wait_for(submit_coro, timeout=SUBMIT_TIMEOUT_SECONDS)
+        except asyncio.TimeoutError:
+            print(f"⏱️ Submit action timed out after {SUBMIT_TIMEOUT_SECONDS}s — aborting to keep UX responsive.")
+            raise HTTPException(status_code=504, detail=f"Form submission timed out after {SUBMIT_TIMEOUT_SECONDS} seconds")
 
         # --- History Tracking ---
         try:

form-flow-backend/routers/suggestions.py

@@ -1,6 +1,9 @@
 from fastapi import APIRouter, HTTPException, Depends, Request, BackgroundTasks
-from pydantic import BaseModel
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, Field, validator
 from typing import Optional, List, Dict, Any
+from difflib import SequenceMatcher
 import logging
 import uuid
 
@@ -41,9 +44,19 @@ class SuggestionRequest(BaseModel):
     field_name: str
     field_label: Optional[str] = None
     field_type: Optional[str] = "text"
+    field_options: Optional[List[Dict[str, Any]]] = None
+    is_dropdown: Optional[bool] = None
     current_value: Optional[str] = None
     n_results: int = 5
 
+    @validator('field_type', pre=True, always=True)
+    def normalize_field_type(cls, v):
+        if v is None or str(v).strip() == "":
+            return "text"
+        v = str(v).lower()
+        allowed = {"text", "email", "tel", "phone", "number", "select", "dropdown", "textarea", "url", "date", "radio", "checkbox"}
+        return v if v in allowed else "text"
+
 
 class SuggestionResponse(BaseModel):
     suggestions: List[str]
@@ -54,12 +67,23 @@ class IntelligentSuggestionRequest(BaseModel):
     """Request for profile-based intelligent suggestions."""
     field_name: str
     field_label: Optional[str] = None
-    field_type: Optional[str] = "text"
+    field_type: Optional[str] = Field(default=None, description="Field input type (text, email, select, textarea, etc.)")
     form_purpose: Optional[str] = "General"
-    previous_answers: Optional[Dict[str, str]] = None
+    previous_answers: Optional[Dict[str, Any]] = None
     form_url: Optional[str] = None
     all_field_labels: Optional[List[str]] = None
     session_id: Optional[str] = None
+    field_options: Optional[List[Dict[str, Any]]] = Field(default=None, description="Dropdown options for this field")
+    is_dropdown: Optional[bool] = Field(default=None, description="True if field has a constrained options list")
+
+    @validator('field_type', pre=True, always=True)
+    def normalize_field_type(cls, v):
+        # Accept None/unknown; default to text and explicitly allow textarea
+        if v is None or str(v).strip() == "":
+            return "text"
+        v = str(v).lower()
+        allowed = {"text", "email", "tel", "phone", "number", "select", "dropdown", "textarea", "url", "date"}
+        return v if v in allowed else "text"
 
 
 class IntelligentSuggestionItem(BaseModel):
@@ -104,6 +128,8 @@ async def get_suggestions(
             "name": data.field_name,
             "label": data.field_label or data.field_name,
             "type": data.field_type or "text",
+            "options": data.field_options or [],
+            "is_dropdown": data.is_dropdown if data.is_dropdown is not None else bool(data.field_options),
         }
 
         # Detect patterns from current value if provided
@@ -125,6 +151,12 @@ async def get_suggestions(
 
         # Extract suggestion values
         suggestion_values = [s.suggested_value for s in suggestions if s.target_field == data.field_name]
+
+        if data.is_dropdown or (data.field_options and len(data.field_options) > 0):
+            filtered_values = _filter_to_field_options(suggestion_values, data.field_options)
+            if len(filtered_values) != len(suggestion_values):
+                logger.info(f"🎛️ Dropdown guardrail filtered {len(suggestion_values) - len(filtered_values)} invalid pattern suggestions for {data.field_name}")
+            suggestion_values = filtered_values
 
         return SuggestionResponse(
             suggestions=suggestion_values[:data.n_results],
@@ -199,6 +231,7 @@ async def get_smart_suggestions(
             "name": data.field_name,
             "label": data.field_label or data.field_name,
             "type": data.field_type or "text",
+            "options": data.field_options or [],
         }
 
         form_context = {
@@ -225,6 +258,16 @@ async def get_smart_suggestions(
 
         # Determine tier used
         tier_used = suggestions[0].tier.value if suggestions else "pattern_only"
+
+        # Guardrail: for dropdowns, only allow values that match an existing option
+        filtered = suggestions
+        if data.is_dropdown or (data.field_options and len(data.field_options) > 0):
+            allowed_values = _filter_to_field_options([s.value for s in suggestions], data.field_options)
+            allowed_set = {str(v).strip().lower() for v in allowed_values}
+            filtered = [s for s in suggestions if str(s.value).strip().lower() in allowed_set]
+            if len(filtered) != len(suggestions):
+                logger.info(f"🎛️ Dropdown guardrail filtered {len(suggestions) - len(filtered)} invalid suggestions for {data.field_name}")
+        suggestions = filtered
 
         return IntelligentSuggestionResponse(
             suggestions=[
@@ -258,6 +301,13 @@ async def get_smart_suggestions(
 # Helper Functions
 # =============================================================================
 
+async def validation_exception_handler(request: Request, exc: RequestValidationError):
+    """Log full validation details for faster debugging."""
+    body = getattr(exc, "body", None)
+    logger.error(f"❌ Validation error on {request.url.path}: {exc.errors()} | body={body}")
+    return JSONResponse(status_code=422, content={"detail": exc.errors(), "body": body})
+
+
 def _infer_field_pattern(field_name: str, field_label: str, field_type: str) -> str:
     """
     Infer the field pattern/category for RAG lookup.
@@ -294,3 +344,41 @@ def _infer_field_pattern(field_name: str, field_label: str, field_type: str) ->
         return "website"
     else:
         return field_name.lower()
+
+
+def _filter_to_field_options(values: List[str], options: Optional[List[Dict[str, Any]]]) -> List[str]:
+    """
+    Restrict suggestions to provided options using case-insensitive fuzzy match.
+    Returns canonical option labels/values when matched.
+    """
+    if not options:
+        return values
+
+    option_labels = []
+    for opt in options:
+        label = opt.get("label")
+        value = opt.get("value")
+        if label:
+            option_labels.append(str(label).strip())
+        if value and value != label:
+            option_labels.append(str(value).strip())
+
+    if not option_labels:
+        return values
+
+    filtered: List[str] = []
+    for val in values:
+        sval = str(val or "").strip()
+        sval_lower = sval.lower()
+        for opt in option_labels:
+            opt_lower = opt.lower()
+            ratio = SequenceMatcher(None, sval_lower, opt_lower).ratio()
+            if (
+                sval_lower == opt_lower
+                or sval_lower in opt_lower
+                or opt_lower in sval_lower
+                or ratio >= 0.82
+            ):
+                filtered.append(opt)
+                break
+    return filtered