views.py

from django.db.utils import IntegrityError, OperationalError, ProgrammingError
from openpyxl import Workbook
from openpyxl.utils import get_column_letter
from openpyxl.styles import Alignment
import copy
import csv
import json
import mimetypes
import re
import yaml
from django_filters.filterset import filterset_factory
from django_filters.utils import try_dbfield
import regex
import os
import uuid
import zipfile
import tempfile
from datetime import date, datetime, timedelta
from typing import Dict, Any, List, Tuple
import time
from django.db.models import (
    F,
    CharField,
    Count,
    Q,
    ExpressionWrapper,
    FloatField,
    IntegerField,
    Value,
    Min,
    Subquery,
    OuterRef,
    When,
    Case,
    Exists,
    ForeignKey,
    OneToOneField,
    ManyToManyField,
    QuerySet,
    Prefetch,
)
from django.db.models.functions import Greatest, Coalesce

from collections import defaultdict
import pytz
from uuid import UUID
from itertools import chain, cycle
import django_filters as df

import shutil
from pathlib import Path
import humanize

from wsgiref.util import FileWrapper

import pandas as pd
import io

import random
from django.db.models.functions import Lower

from docxtpl import DocxTemplate
from integrations.models import SyncMapping
from integrations.tasks import sync_object_to_integrations
from webhooks.service import dispatch_webhook_event
from .generators import gen_audit_context
from .serializer_fields import FieldsRelatedField

from django.utils import timezone
from django.utils.text import slugify
from django.utils.decorators import method_decorator
from django.views.decorators.cache import cache_page
from django.views.decorators.vary import vary_on_cookie
from django.core.cache import cache


from django.apps import apps
from django.contrib.auth.models import Permission
from django.conf import settings
from django.core.exceptions import FieldDoesNotExist
from django.core.files.storage import default_storage

from django.db import models, transaction
from django.forms import ValidationError
from django.http import FileResponse, HttpResponse, StreamingHttpResponse
from django.middleware import csrf
from django.template.loader import render_to_string
from django.utils.functional import Promise
from django.shortcuts import get_object_or_404
from django_filters.rest_framework import DjangoFilterBackend
from iam.models import Folder, RoleAssignment, User, UserGroup
from rest_framework import filters, generics, permissions, status, viewsets
from django.utils.translation import gettext_lazy as _, get_language
from rest_framework.decorators import (
    action,
    api_view,
    permission_classes,
    renderer_classes,
)
from rest_framework.parsers import (
    FileUploadParser,
)
from rest_framework.renderers import JSONRenderer
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.utils.serializer_helpers import ReturnDict
from rest_framework.views import APIView
from rest_framework.exceptions import NotFound, PermissionDenied


from weasyprint import HTML

from core.helpers import *
from core.models import (
    AppliedControl,
    ComplianceAssessment,
    RequirementMappingSet,
    RiskAssessment,
    RiskMatrix,
    RiskScenario,
    AssetClass,
    Terminology,
)
from core.serializers import ComplianceAssessmentReadSerializer
from core.utils import (
    compare_schema_versions,
    get_auditee_filtered_folder_ids,
    _generate_occurrences,
    _create_task_dict,
)
from dateutil import relativedelta as rd

from ebios_rm.models import (
    EbiosRMStudy,
    FearedEvent,
    RoTo,
    StrategicScenario,
    Stakeholder,
    AttackPath,
)

from tprm.models import Entity, Solution, Contract
from privacy.models import Processing, DataBreach, RightRequest
from resilience.models import BusinessImpactAnalysis

from .models import *
from .serializers import *

from .models import Severity
from . import dora

from serdes.utils import (
    get_domain_export_objects,
    import_export_serializer_class,
    topological_sort,
    build_dependency_graph,
    get_self_referencing_field,
    sort_objects_by_self_reference,
)
from serdes.serializers import ExportSerializer
from django.contrib.admin.utils import NestedObjects
from django.db import router
from global_settings.models import GlobalSettings
from global_settings.utils import ff_is_enabled

import structlog

logger = structlog.get_logger(__name__)

SHORT_CACHE_TTL = 2  # mn
MED_CACHE_TTL = 5  # mn
LONG_CACHE_TTL = 60  # mn


MAPPING_MAX_DEPTH = 3

SETTINGS_MODULE = __import__(os.environ.get("DJANGO_SETTINGS_MODULE"))
MODULE_PATHS = SETTINGS_MODULE.settings.MODULE_PATHS


class NullableChoiceFilter(df.MultipleChoiceFilter):
    """
    A filter that supports filtering for null values using '--' as a special value.
    When '--' is in the filter values, it matches records where the field is null.
    """

    def __init__(self, *args, **kwargs):
        # Add "--" to the choices
        if "choices" in kwargs:
            original_choices = kwargs["choices"]
            # Add ("--", "--") to the beginning of choices
            kwargs["choices"] = [("--", "--")] + list(original_choices)
        super().__init__(*args, **kwargs)

    def filter(self, qs, value):
        if not value:
            return qs

        # Convert single value to list if needed
        if not isinstance(value, list):
            value = [value]

        # Separate "--" (null) from other actual values
        has_null = "--" in value
        real_values = [v for v in value if v != "--"]

        if has_null and real_values:
            # Both null and specific values: OR condition
            return qs.filter(
                Q(**{f"{self.field_name}__isnull": True})
                | Q(**{f"{self.field_name}__in": real_values})
            )
        elif has_null:
            # Only null values
            return qs.filter(**{f"{self.field_name}__isnull": True})
        elif real_values:
            # Only real values - filter by those values
            return qs.filter(**{f"{self.field_name}__in": real_values})
        else:
            # No valid values, return empty queryset
            return qs.none()


def add_unset_option(choices):
    """Add '--' (unset) option to choices dictionary or list"""
    # Handle both dict and list of tuples format
    if isinstance(choices, dict):
        # For dict format {value: label}, prepend with "--": "--"
        return {"--": "--", **choices}
    else:
        # For list of tuples like [(value, label), ...]
        return [("--", "--")] + list(choices)


def get_mapping_max_depth():
    """Get mapping max depth from general settings at runtime; safe during migrations."""
    try:
        gs = GlobalSettings.objects.filter(name="general").only("value").first()
        if not gs or not isinstance(getattr(gs, "value", None), dict):
            return MAPPING_MAX_DEPTH
        raw = gs.value.get("mapping_max_depth", MAPPING_MAX_DEPTH)
        try:
            val = int(raw)
        except (TypeError, ValueError):
            return MAPPING_MAX_DEPTH
        # Clamp to UI constraints
        return max(2, min(5, val))
    except (OperationalError, ProgrammingError):
        # DB not ready (e.g., migrate, makemigrations)
        return MAPPING_MAX_DEPTH


def escape_excel_formula(value):
    """
    Escape Excel formula injection by prefixing dangerous characters.
    Prevents CSV/Formula injection (OWASP) when values start with =+-@
    """
    if value is None:
        return ""
    s = str(value)
    if not s:
        return ""
    stripped = s.lstrip(" \t\r\n")
    if stripped and stripped[0] in ("=", "+", "-", "@"):
        return "'" + s
    return s


def create_xlsx_response(entries, filename, wrap_columns=None):
    """
    DRY helper to create XLSX response with consistent formatting.

    Args:
        entries: List of dictionaries with data
        filename: Output filename
        wrap_columns: List of column names to wrap text (default: ["name", "description"])

    Returns:
        HttpResponse with XLSX file
    """
    if wrap_columns is None:
        wrap_columns = ["name", "description"]

    df = pd.DataFrame(entries)
    buffer = io.BytesIO()

    with pd.ExcelWriter(buffer, engine="openpyxl") as writer:
        df.to_excel(writer, index=False)
        worksheet = writer.sheets["Sheet1"]

        wrap_indices = [
            df.columns.get_loc(col) + 1 for col in wrap_columns if col in df.columns
        ]

        for col_idx in wrap_indices:
            for row_idx in range(2, len(df) + 2):
                cell = worksheet.cell(row=row_idx, column=col_idx)
                cell.alignment = Alignment(wrap_text=True)

        for idx, col in enumerate(df.columns):
            column_width = 20
            if col in wrap_columns:
                column_width = 40
            worksheet.column_dimensions[
                worksheet.cell(row=1, column=idx + 1).column_letter
            ].width = column_width

    buffer.seek(0)
    response = HttpResponse(
        buffer.getvalue(),
        content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
    )
    response["Content-Disposition"] = f'attachment; filename="{filename}"'
    return response


class ExportMixin:
    """
    Generic export mixin for CSV/XLSX exports.
    ViewSets define export_config with fields, formatting, and query optimization hints.
    """

    export_config = None

    def _get_export_queryset(self):
        """Get filtered queryset with permissions applied."""
        (viewable_ids, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), self.request.user, self.model
        )
        queryset = self.model.objects.filter(id__in=viewable_ids)

        if self.export_config:
            if self.export_config.get("select_related"):
                queryset = queryset.select_related(
                    *self.export_config["select_related"]
                )
            if self.export_config.get("prefetch_related"):
                queryset = queryset.prefetch_related(
                    *self.export_config["prefetch_related"]
                )

        return queryset

    def _resolve_field_value(self, obj, field_config):
        """Resolve field value from object using dot notation or callable."""
        source = field_config.get("source")
        if not source:
            return ""

        parts = source.split(".")
        value = obj
        for part in parts:
            if value is None:
                return ""
            value = getattr(value, part, "")

        # Call methods but keep managers for format functions
        if callable(value):
            # Check if it's a Django manager/queryset (has .all() method)
            is_manager = hasattr(value, "all")
            # Call methods, but pass managers as-is
            if not is_manager:
                try:
                    value = value()
                except TypeError:
                    pass

        # Always apply if formatter exists, let formatter handle empties
        format_func = field_config.get("format")
        if format_func:
            value = format_func(value)

        if isinstance(value, str):
            if field_config.get("escape", True):
                value = escape_excel_formula(value)

        return value if value is not None else ""

    @action(detail=False, name="Export as CSV")
    def export_csv(self, request):
        if not self.export_config:
            return HttpResponse(
                status=501, content="Export not configured for this model"
            )

        try:
            queryset = self._get_export_queryset()
            response = HttpResponse(content_type="text/csv")
            filename = f"{self.export_config.get('filename', 'export')}.csv"
            response["Content-Disposition"] = f'attachment; filename="{filename}"'

            writer = csv.writer(response, delimiter=";")
            fields = self.export_config["fields"]
            writer.writerow([f.get("label", name) for name, f in fields.items()])

            has_prefetch = bool(self.export_config.get("prefetch_related"))
            if has_prefetch:
                for obj in queryset:
                    row = [
                        self._resolve_field_value(obj, field_config)
                        for field_config in fields.values()
                    ]
                    writer.writerow(row)
            else:
                for obj in queryset.iterator():
                    row = [
                        self._resolve_field_value(obj, field_config)
                        for field_config in fields.values()
                    ]
                    writer.writerow(row)

            return response

        except Exception as e:
            logger.error(f"Error exporting {self.model.__name__} to CSV: {str(e)}")
            return HttpResponse(
                status=500, content="An error occurred while generating the CSV export."
            )

    @action(detail=False, name="Export as XLSX")
    def export_xlsx(self, request):
        if not self.export_config:
            return HttpResponse(
                status=501, content="Export not configured for this model"
            )

        queryset = self._get_export_queryset()
        entries = []
        fields = self.export_config["fields"]

        for obj in queryset:
            entry = {}
            for field_name, field_config in fields.items():
                entry[field_config.get("label", field_name)] = (
                    self._resolve_field_value(obj, field_config)
                )
            entries.append(entry)

        filename = f"{self.export_config.get('filename', 'export')}.xlsx"
        wrap_columns = self.export_config.get("wrap_columns", ["name", "description"])
        return create_xlsx_response(entries, filename, wrap_columns)

    def _create_multi_sheet_xlsx(self, queryset, main_fields, detail_config):
        """
        Create multi-sheet XLSX with main summary sheet and per-object detail sheets.

        Args:
            queryset: Main objects queryset
            main_fields: Fields config for summary sheet
            detail_config: Dict with 'fields', 'related_queryset_method', 'sheet_name_field'

        Returns:
            HttpResponse with XLSX file
        """
        wb = Workbook()
        wb.remove(wb.active)  # Remove default sheet

        # Create summary sheet
        summary_ws = wb.create_sheet(title="Summary")
        main_headers = [f.get("label", name) for name, f in main_fields.items()]
        summary_ws.append(main_headers)

        # Wrap text alignment for summary sheet
        wrap_alignment = Alignment(wrap_text=True, vertical="top")
        wrap_columns = self.export_config.get("wrap_columns", ["name", "description"])

        for col_idx, header in enumerate(main_headers, 1):
            if header.lower() in wrap_columns:
                for row_idx in range(1, summary_ws.max_row + 1):
                    summary_ws.cell(
                        row=row_idx, column=col_idx
                    ).alignment = wrap_alignment

        # Track sheet names to avoid duplicates
        used_sheet_names = {"Summary"}

        for obj in queryset:
            # Add row to summary sheet
            row = [
                self._resolve_field_value(obj, field_config)
                for field_config in main_fields.values()
            ]
            summary_ws.append(row)

            # Get related objects for detail sheet
            related_getter = detail_config.get("related_queryset_method")
            if related_getter:
                related_objects = getattr(obj, related_getter)()

                # Skip empty detail sheets if configured
                if not related_objects and detail_config.get("skip_empty", True):
                    continue

                # Create unique sheet name
                sheet_name_field = detail_config.get("sheet_name_field", "name")
                base_name = str(
                    self._resolve_field_value(obj, {"source": sheet_name_field})
                )
                if not base_name:
                    base_name = f"Object_{obj.pk}"

                # Truncate and ensure uniqueness
                sheet_name = base_name[:31]
                counter = 1
                while sheet_name in used_sheet_names:
                    suffix = f" ({counter})"
                    sheet_name = base_name[: 31 - len(suffix)] + suffix
                    counter += 1
                used_sheet_names.add(sheet_name)

                # Create detail sheet
                detail_ws = wb.create_sheet(title=sheet_name)
                detail_fields = detail_config.get("fields", {})
                detail_headers = [
                    f.get("label", name) for name, f in detail_fields.items()
                ]
                detail_ws.append(detail_headers)

                # Add related objects
                for related_obj in related_objects:
                    detail_row = [
                        self._resolve_field_value(related_obj, field_config)
                        for field_config in detail_fields.values()
                    ]
                    detail_ws.append(detail_row)

                # Apply wrap text to detail sheet
                for col_idx, header in enumerate(detail_headers, 1):
                    if header.lower() in wrap_columns:
                        for row_idx in range(1, detail_ws.max_row + 1):
                            detail_ws.cell(
                                row=row_idx, column=col_idx
                            ).alignment = wrap_alignment

        # Save to buffer
        buffer = io.BytesIO()
        wb.save(buffer)
        buffer.seek(0)

        filename = f"{self.export_config.get('filename', 'export')}_multi_sheet.xlsx"
        response = HttpResponse(
            buffer.getvalue(),
            content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
        )
        response["Content-Disposition"] = f'attachment; filename="{filename}"'
        return response


class GenericFilterSet(df.FilterSet):
    class UUIDInFilter(df.BaseInFilter, df.UUIDFilter):
        pass

    id = UUIDInFilter(field_name="id", lookup_expr="in")

    @classmethod
    def filter_for_lookup(cls, field, lookup_type):
        DEFAULTS = dict(cls.FILTER_DEFAULTS)
        if hasattr(cls, "_meta"):
            DEFAULTS.update(cls._meta.filter_overrides)

        data = try_dbfield(DEFAULTS.get, field.__class__) or {}
        filter_class = data.get("filter_class")
        params = data.get("extra", lambda field: {})(field)

        # if there is no filter class, exit early
        if not filter_class:
            return None, {}

        # perform lookup specific checks
        if lookup_type == "exact" and getattr(field, "choices", None):
            return df.MultipleChoiceFilter, {"choices": field.choices, **params}

        if lookup_type == "isnull":
            data = try_dbfield(DEFAULTS.get, models.BooleanField)

            filter_class = data.get("filter_class")
            params = data.get("extra", lambda field: {})(field)
            return filter_class, params

        if lookup_type == "in":

            class ConcreteInFilter(df.BaseInFilter, filter_class):
                pass

            ConcreteInFilter.__name__ = cls._csv_filter_class_name(
                filter_class, lookup_type
            )

            return ConcreteInFilter, params

        if lookup_type == "range":

            class ConcreteRangeFilter(df.BaseRangeFilter, filter_class):
                pass

            ConcreteRangeFilter.__name__ = cls._csv_filter_class_name(
                filter_class, lookup_type
            )

            return ConcreteRangeFilter, params

        return filter_class, params

    class Meta:
        model = None  # This will be set dynamically via filterset_factory.
        filter_overrides = {
            models.ForeignKey: {
                "filter_class": df.ModelMultipleChoiceFilter,
                "extra": lambda f: {
                    "queryset": f.remote_field.model.objects.all(),
                },
            },
            models.ManyToManyField: {
                "filter_class": df.ModelMultipleChoiceFilter,
                "extra": lambda f: {
                    "queryset": f.remote_field.model.objects.all(),
                },
            },
        }


class FolderOrderingFilter(filters.OrderingFilter):
    def get_ordering(self, request, queryset, view):
        ordering = super().get_ordering(request, queryset, view)
        if ordering:
            return [
                "folder__name"
                if f == "folder"
                else "-folder__name"
                if f == "-folder"
                else f
                for f in ordering
            ]
        return ordering


class BaseModelViewSet(viewsets.ModelViewSet):
    filter_backends = [
        DjangoFilterBackend,
        filters.SearchFilter,
        FolderOrderingFilter,
    ]
    ordering = ["created_at"]
    ordering_fields = "__all__"
    search_fields = ["name", "description"]
    filterset_fields = []
    model: type[models.Model] | None = None

    serializers_module = "core.serializers"

    @property
    def filterset_class(self):
        # If you have defined filterset_fields, build the FilterSet on the fly.
        if self.filterset_fields:
            return filterset_factory(
                model=self.model,
                filterset=GenericFilterSet,
                fields=self.filterset_fields,
            )
        return None

    def get_queryset(self) -> models.query.QuerySet:
        if not self.model:
            return None
        object_ids_view = None
        if self.request.method == "GET":
            if q := re.match(
                r"/api/[\w-]+/([\w-]+/)?([0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}(,[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12})+)",
                self.request.path,
            ):
                """"get_queryset is called by Django even for an individual object via get_object
                https://stackoverflow.com/questions/74048193/why-does-a-retrieve-request-end-up-calling-get-queryset"""
                id = UUID(q.group(1))
                if RoleAssignment.is_object_readable(self.request.user, self.model, id):
                    object_ids_view = [id]

        if not object_ids_view:
            object_ids_view = RoleAssignment.get_accessible_object_ids(
                Folder.get_root_folder(), self.request.user, self.model
            )[0]

        queryset = self.model.objects.filter(id__in=object_ids_view)

        field_names = {f.name for f in self.model._meta.get_fields()}
        if "parent_folder" in field_names:
            queryset = queryset.select_related("parent_folder")
        if "filtering_labels" in field_names:
            queryset = queryset.prefetch_related("filtering_labels")

        return queryset

    def get_serializer_context(self):
        context = super().get_serializer_context()
        context["action"] = self.action
        return context

    def get_serializer_class(self, **kwargs):
        serializer_factory = SerializerFactory(
            self.serializers_module, MODULE_PATHS.get("serializers", [])
        )
        serializer_class = serializer_factory.get_serializer(
            self.model.__name__, kwargs.get("action", self.action)
        )
        logger.debug(
            "Serializer class",
            serializer_class=serializer_class,
            action=kwargs.get("action", self.action),
            viewset=self,
            module_paths=MODULE_PATHS,
        )

        return serializer_class

    COMMA_SEPARATED_UUIDS_REGEX = r"^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}(,[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12})*$"

    def _process_request_data(self, request: Request) -> None:
        """
        Process the request data to split comma-separated UUIDs into a list
        and handle empty list scenarios.
        """
        for field in request.data:
            # NOTE: This is due to sveltekit-superforms not coercing the value into a list when
            # the form's dataType is "form", rather than "json".
            # Typically, dataType is "form" when the form contains a file input (e.g. for evidence attachments).
            # I am not ruling out the possibility that I am doing something wrong in the frontend. (Nassim)
            # TODO: Come back to this once superForms v2 is out of alpha. https://github.com/ciscoheat/sveltekit-superforms/releases
            if isinstance(request.data[field], list) and len(request.data[field]) == 1:
                if isinstance(request.data[field][0], str) and re.match(
                    self.COMMA_SEPARATED_UUIDS_REGEX, request.data[field][0]
                ):
                    request.data[field] = request.data[field][0].split(",")
                elif not request.data[field][0]:
                    request.data[field] = []

    def _process_labels(self, labels):
        """
        Creates a FilteringLabel and replaces the value with the ID of the newly created label.
        """
        new_labels = []
        for label in labels:
            try:
                uuid.UUID(label, version=4)
                new_labels.append(label)
            except ValueError:
                new_label = FilteringLabel(label=label)
                new_label.full_clean()
                new_label.save()
                new_labels.append(str(new_label.id))
        return new_labels

    def _process_evidences(self, evidences, folder):
        new_evidences = []
        for value in evidences or []:
            try:
                uuid.UUID(str(value), version=4)
                new_evidences.append(str(value))
            except ValueError:
                new_evidence = Evidence(name=value, folder=folder)
                new_evidence.full_clean()
                new_evidence.save()
                new_evidences.append(str(new_evidence.id))
        return new_evidences

    def _resolve_related_model(self, source: str):
        """Resolve a dotted source path (e.g. 'asset.folder') to its related model."""
        model = self.model
        for part in source.split("."):
            if not model:
                return None
            try:
                field = model._meta.get_field(part)
            except FieldDoesNotExist:
                return None
            model = getattr(field, "related_model", None)
        return model

    def _get_fieldsrelated_map(self, serializer):
        """Map FieldsRelatedField serializer fields to their related models."""
        if hasattr(serializer, "child"):
            serializer = serializer.child
        if not hasattr(serializer, "fields"):
            return {}
        field_models = {}
        for name, field in serializer.fields.items():
            is_fields_related = isinstance(field, FieldsRelatedField)
            if not is_fields_related and hasattr(field, "child_relation"):
                is_fields_related = isinstance(field.child_relation, FieldsRelatedField)
            if not is_fields_related:
                continue
            source = field.source if field.source not in (None, "*") else name
            related_model = self._resolve_related_model(source)
            if related_model:
                field_models[name] = related_model
        return field_models

    def _get_accessible_ids_map(self, related_models):
        """Return visible object IDs per related model for the current user."""
        root_folder = Folder.get_root_folder()
        allowed = {}
        for model in related_models:
            try:
                ids = RoleAssignment.get_accessible_object_ids(
                    root_folder, self.request.user, model
                )[0]
            except (NotImplementedError, Permission.DoesNotExist):
                # Model does not support IAM scoping; skip filtering
                allowed[model] = None
                continue
            allowed[model] = {str(item_id) for item_id in ids}
        return allowed

    def _extract_related_id(self, item):
        """Extract a related object ID from a field payload."""
        if isinstance(item, dict):
            value = item.get("id")
        else:
            value = item
        if value is None:
            return None
        return str(value)

    def _placeholder_for(self, item):
        """Create a placeholder for a hidden related object."""
        if isinstance(item, dict):
            return {}
        return ""

    def _filter_related_fields(self, data, field_models, allowed_ids):
        """Mask related fields the current user is not allowed to see."""
        if isinstance(data, list):
            return [
                self._filter_related_fields(item, field_models, allowed_ids)
                for item in data
            ]
        if not isinstance(data, dict):
            return data

        current_user_id = (
            str(self.request.user.id) if self.request.user.is_authenticated else None
        )

        for field_name, related_model in field_models.items():
            allowed = allowed_ids.get(related_model)
            if allowed is None:
                continue
            value = data.get(field_name)
            if isinstance(value, list):
                filtered = []
                for item in value:
                    item_id = self._extract_related_id(item)
                    # Always include current user
                    if (
                        not item_id
                        or item_id in allowed
                        or (
                            related_model.__name__ == "User"
                            and item_id == current_user_id
                        )
                    ):
                        filtered.append(item)
                    else:
                        filtered.append(self._placeholder_for(item))
                data[field_name] = filtered
            elif isinstance(value, dict):
                item_id = self._extract_related_id(value)
                # Always include current user
                if (
                    item_id
                    and item_id not in allowed
                    and not (
                        related_model.__name__ == "User" and item_id == current_user_id
                    )
                ):
                    data[field_name] = self._placeholder_for(value)
        return data

    def list(self, request, *args, **kwargs):
        """
        Override the list method to inject optimized data into the serializer context.
        """
        queryset = self.filter_queryset(self.get_queryset())
        page = self.paginate_queryset(queryset)
        objects = page if page is not None else queryset
        # 1. Perform the bulk calculation for the current page (or entire set if not paginated)
        optimized_data = self._get_optimized_object_data(objects)
        # 2. Pass the data to the serializer via context
        context = self.get_serializer_context()
        context["optimized_data"] = optimized_data
        if page is not None:
            serializer = self.get_serializer(page, many=True, context=context)
            data = serializer.data
            field_models = self._get_fieldsrelated_map(serializer)
            if field_models:
                allowed_ids = self._get_accessible_ids_map(set(field_models.values()))
                data = self._filter_related_fields(data, field_models, allowed_ids)
            return self.get_paginated_response(data)
        serializer = self.get_serializer(queryset, many=True, context=context)
        data = serializer.data
        field_models = self._get_fieldsrelated_map(serializer)
        if field_models:
            allowed_ids = self._get_accessible_ids_map(set(field_models.values()))
            data = self._filter_related_fields(data, field_models, allowed_ids)
        return Response(data)

    def retrieve(self, request, *args, **kwargs):
        """Return a single object with unauthorized related fields masked."""
        instance = self.get_object()
        serializer = self.get_serializer(instance)
        data = serializer.data
        field_models = self._get_fieldsrelated_map(serializer)
        if field_models:
            allowed_ids = self._get_accessible_ids_map(set(field_models.values()))
            data = self._filter_related_fields(data, field_models, allowed_ids)
        return Response(data)

    def perform_create(self, serializer):
        instance = serializer.save()
        dispatch_webhook_event(instance, "created", serializer=serializer)
        return instance

    def perform_update(self, serializer):
        instance = serializer.save()
        dispatch_webhook_event(instance, "updated", serializer=serializer)
        return instance

    def perform_destroy(self, instance):
        serializer = self.get_serializer(instance)
        serializer.delete(instance)
        try:
            dispatch_webhook_event(instance, "deleted")
        except Exception:
            logger.error("Webhook dispatch failed on delete", exc_info=True)

    def create(self, request: Request, *args, **kwargs) -> Response:
        self._process_request_data(request)
        if request.data.get("filtering_labels"):
            request.data["filtering_labels"] = self._process_labels(
                request.data["filtering_labels"]
            )
        # Experimental: process evidences on TaskTemplate creation
        if request.data.get("evidences") and self.model == TaskTemplate:
            folder = Folder.objects.get(id=request.data.get("folder"))
            request.data["evidences"] = self._process_evidences(
                request.data.get("evidences"), folder=folder
            )
        return super().create(request, *args, **kwargs)

    def update(self, request: Request, *args, **kwargs) -> Response:
        # Experimental: process evidences on TaskTemplate update

        if request.data.get("evidences") and self.model == TaskTemplate:
            folder = Folder.objects.get(id=request.data.get("folder"))
            request.data["evidences"] = self._process_evidences(
                request.data["evidences"], folder=folder
            )

        # NOTE: Handle filtering_labels field - SvelteKit SuperForms behavior inconsistency:
        # Forms with file inputs (like Evidence attachments) use dataType="form" and omit empty fields
        # Forms without file inputs use dataType="json" and send empty arrays []
        # When the field is missing, we need to explicitly clear the labels by passing empty list to serializer
        if hasattr(self.model, "_meta") and "filtering_labels" in [
            f.name for f in self.model._meta.get_fields()
        ]:
            if "filtering_labels" in request.data:
                labels = request.data.get("filtering_labels")
                if labels:
                    # Make request.data mutable if needed (e.g., for multipart/form-data)
                    if hasattr(request.data, "_mutable"):
                        request.data._mutable = True
                    request.data["filtering_labels"] = self._process_labels(labels)

        return super().update(request, *args, **kwargs)

    def partial_update(self, request: Request, *args, **kwargs) -> Response:
        self._process_request_data(request)
        return super().partial_update(request, *args, **kwargs)

    def destroy(self, request: Request, *args, **kwargs) -> Response:
        self._process_request_data(request)
        return super().destroy(request, *args, **kwargs)

    @action(detail=False, methods=["post"], url_path="batch-action")
    def batch_action(self, request):
        """
        Perform a batch action on multiple objects.
        Uses the IAM-filtered queryset and serializers to respect permissions
        and validation, mirroring the standard partial_update / destroy flows.

        Payload: { "action": "delete"|"change_field"|"change_m2m"|"change_folder",
                   "ids": [...], "field": "<field_name>", "value": ... }
        """
        action_type = request.data.get("action")
        ids = request.data.get("ids", [])
        value = request.data.get("value")
        field_name = request.data.get("field")

        valid_actions = ("delete", "change_field", "change_m2m", "change_folder")
        if action_type not in valid_actions:
            return Response(
                {"error": "Invalid action type"},
                status=status.HTTP_400_BAD_REQUEST,
            )

        if not ids:
            return Response(
                {"error": "No ids provided"},
                status=status.HTTP_400_BAD_REQUEST,
            )

        BATCH_SIZE_LIMIT = 100
        if len(ids) > BATCH_SIZE_LIMIT:
            return Response(
                {"error": f"Too many ids (max {BATCH_SIZE_LIMIT})"},
                status=status.HTTP_400_BAD_REQUEST,
            )

        # Use the IAM-filtered queryset (same as get_object() in update/destroy)
        queryset = self.get_queryset()
        objects_by_id = {str(o.id): o for o in queryset.filter(id__in=ids)}

        # Resolve the required permission for the action type
        perm_codename = (
            f"delete_{self.model._meta.model_name}"
            if action_type == "delete"
            else f"change_{self.model._meta.model_name}"
        )
        required_perm = Permission.objects.get(codename=perm_codename)

        # Resolve the write serializer once for all update operations
        if action_type != "delete":
            serializer_class = self.get_serializer_class(action="partial_update")

        succeeded = []
        failed = []

        for obj_id in ids:
            obj = objects_by_id.get(str(obj_id))
            if obj is None:
                failed.append(
                    {
                        "id": str(obj_id),
                        "error": "Object not found or access denied",
                    }
                )
                continue

            if not RoleAssignment.is_access_allowed(
                user=request.user,
                perm=required_perm,
                folder=Folder.get_folder(obj),
            ):
                failed.append(
                    {
                        "id": str(obj_id),
                        "name": str(obj),
                        "error": "Permission denied",
                    }
                )
                continue

            try:
                if action_type == "delete":
                    if getattr(obj, "builtin", False) or getattr(obj, "urn", None):
                        failed.append(
                            {
                                "id": str(obj_id),
                                "name": str(obj),
                                "error": "Cannot delete builtin object",
                            }
                        )
                        continue
                    try:
                        dispatch_webhook_event(obj, "deleted")
                    except Exception:
                        logger.error(
                            "Webhook dispatch failed on batch delete", exc_info=True
                        )
                    obj.delete()

                else:
                    # Build data dict for the serializer
                    if action_type == "change_folder":
                        data = {"folder": value}
                    elif action_type == "change_m2m":
                        actor_ids = value if isinstance(value, list) else [value]
                        data = {field_name: actor_ids}
                    else:  # change_field
                        data = {field_name: value}

                    # Go through the serializer for validation + save,
                    # and perform_update for webhook dispatch
                    serializer = serializer_class(
                        obj,
                        data=data,
                        partial=True,
                        context=self.get_serializer_context(),
                    )
                    if not serializer.is_valid():
                        failed.append(
                            {
                                "id": str(obj_id),
                                "name": str(obj),
                                "error": serializer.errors,
                            }
                        )
                        continue
                    self.perform_update(serializer)

                succeeded.append({"id": str(obj_id), "name": str(obj)})

            except PermissionDenied as e:
                failed.append(
                    {
                        "id": str(obj_id),
                        "name": str(obj),
                        "error": e.detail
                        if hasattr(e, "detail")
                        else "Permission denied",
                    }
                )
            except Exception:
                logger.error("Batch action failed for %s", obj_id, exc_info=True)
                failed.append(
                    {
                        "id": str(obj_id),
                        "name": str(obj),
                        "error": "Unexpected error",
                    }
                )

        return Response({"succeeded": succeeded, "failed": failed})

    @action(detail=True, methods=["get"], url_path="cascade-info")
    def cascade_info(self, request, pk=None):
        """
        Cascade preview:
        - deleted: objects actually deleted by cascade
        - affected: objects not deleted but whose relationships will be removed (through rows, SET_NULL, local links)
        """
        instance = self.get_object()
        collector = NestedObjects(using=router.db_for_write(instance))
        collector.collect([instance])

        skip_model_names = {
            "Token",
            "AuthToken",
            "EmailAddress",
            "Session",
            "RequirementAssessment",
            "LogEntry",
            "Folder",
            "Permission",
            "LibraryFilteringLabel",
        }
        skip_model_classes = set()

        def is_hidden_model(model):
            return (skip_model_classes and model in skip_model_classes) or (
                not skip_model_classes and model.__name__ in skip_model_names
            )

        # Build index of concrete objects that will be deleted
        deleted_index = set()
        for model, objs in collector.model_objs.items():
            if model is type(instance):
                continue
            if getattr(model._meta, "auto_created", False):
                continue  # skip through rows here; we will bubble endpoints separately
            for o in objs:
                deleted_index.add((model.__name__, str(getattr(o, "pk", ""))))

        def add_grouped(bucket, obj):
            model = obj.__class__
            if is_hidden_model(model):
                return
            key = model.__name__
            pk = str(getattr(obj, "pk", "")) or ""
            if (key, pk) in bucket["_seen"]:
                return

            if key == "RoleAssignment":
                role_name = getattr(getattr(obj, "role", None), "name", "") or ""
                if getattr(obj, "user", None):
                    display = f"{role_name} → {getattr(obj.user, 'email', '')}"
                elif getattr(obj, "user_group", None):
                    display = f"{role_name} → {getattr(obj.user_group, 'name', '')}"
                else:
                    display = role_name
            else:
                display = (
                    getattr(obj, "name", None)
                    or getattr(obj, "email", None)
                    or str(obj)
                )

            group = bucket["by_model"].setdefault(
                key,
                {
                    "model": key,
                    "verbose_name": str(model._meta.verbose_name),
                    "objects": [],
                },
            )
            group["objects"].append({"id": pk, "name": display})
            bucket["_seen"].add((key, pk))

        def finalize(bucket):
            groups = list(bucket["by_model"].values())
            for g in groups:
                g["objects"].sort(key=lambda o: (o["name"] or "").lower())
            groups.sort(key=lambda g: (g.get("verbose_name") or g["model"]).lower())
            count = sum(len(g["objects"]) for g in groups)
            return groups, count

        deleted_bucket = {"by_model": {}, "_seen": set()}
        affected_bucket = {"by_model": {}, "_seen": set()}

        # 1) Concrete deletions
        through_rows = []
        for model, instances in collector.model_objs.items():
            if model is type(instance):
                continue
            if getattr(model._meta, "auto_created", False):
                through_rows.append((model, list(instances)))
                continue
            if is_hidden_model(model):
                continue
            for obj in instances:
                add_grouped(deleted_bucket, obj)

        # 2) Bubble endpoints from through rows (M2M join tables)
        for through_model, t_instances in through_rows:
            # Join tables have ≥2 FKs; skip others
            fk_fields = [
                f
                for f in through_model._meta.fields
                if isinstance(f, ForeignKey) and getattr(f, "remote_field", None)
            ]
            if len(fk_fields) < 2:
                continue

            for t in t_instances:
                endpoints = []
                for fk in fk_fields:
                    try:
                        rel_obj = getattr(t, fk.name, None)
                    except Exception:
                        rel_obj = None
                    if not rel_obj:
                        continue
                    if isinstance(rel_obj, type(instance)):
                        continue
                    if getattr(rel_obj._meta, "auto_created", False):
                        continue
                    endpoints.append(rel_obj)

                for ep in endpoints:
                    key = ep.__class__.__name__
                    pk = str(getattr(ep, "pk", "")) or ""
                    if (key, pk) in deleted_index:
                        add_grouped(deleted_bucket, ep)
                    else:
                        add_grouped(affected_bucket, ep)

        # 2b) Incoming updates (SET_NULL/DEFAULT) from Django's plan
        updates = getattr(collector, "field_updates", {})

        def _iter_objs(maybe_iter):
            if maybe_iter is None:
                return
            if isinstance(maybe_iter, QuerySet):
                for x in maybe_iter.iterator():
                    yield x
            else:
                for x in maybe_iter:
                    yield x

        for key, ops in updates.items():
            # Shape A: (field, value) -> [objs, ...]
            if isinstance(key, tuple) and len(key) == 2:
                field, value = key
                for objs in ops or []:
                    for o in _iter_objs(objs):
                        model = field.model  # model whose rows will be updated
                        pair = (model.__name__, str(getattr(o, "pk", "")) or "")
                        if pair not in deleted_index:
                            # If you want only SET_NULL cases: if value is None: ...
                            add_grouped(affected_bucket, o)
            # Shape B: model -> [(field, value, objs) ...]
            elif isinstance(ops, (list, tuple)) and ops and isinstance(ops[0], tuple):
                for item in ops:
                    if len(item) == 3:
                        field, value, objs = item
                    elif len(item) == 2:
                        field, objs = item
                        value = None
                    else:
                        continue
                    for o in _iter_objs(objs):
                        model = field.model
                        pair = (model.__name__, str(getattr(o, "pk", "")) or "")
                        if pair not in deleted_index:
                            add_grouped(affected_bucket, o)
            else:
                # Unknown shape; ignore safely
                pass

        # 2c) Outgoing links from the instance (FK/O2O/M2M)
        for f in instance._meta.get_fields():
            if getattr(f, "auto_created", False):
                continue  # skip reverse relations

            # Local FK/O2O targets: will remain but lose the link to this instance.
            if isinstance(f, (ForeignKey, OneToOneField)):
                try:
                    rel_obj = getattr(instance, f.name, None)
                except Exception:
                    rel_obj = None
                if rel_obj:
                    key = rel_obj.__class__.__name__
                    pk = str(getattr(rel_obj, "pk", "")) or ""
                    if (key, pk) not in deleted_index:
                        add_grouped(affected_bucket, rel_obj)

            # Local M2M targets: will remain; join rows are removed.
            elif isinstance(f, ManyToManyField):
                try:
                    manager = getattr(instance, f.name)
                    for rel_obj in manager.all():
                        key = rel_obj.__class__.__name__
                        pk = str(getattr(rel_obj, "pk", "")) or ""
                        if (key, pk) not in deleted_index:
                            add_grouped(affected_bucket, rel_obj)
                except Exception:
                    pass

        # 3) Sort and respond
        deleted_groups, deleted_count = finalize(deleted_bucket)
        affected_groups, affected_count = finalize(affected_bucket)

        def flatten(groups):
            return [
                {
                    "model": g["verbose_name"] or g["model"],
                    "name": o["name"],
                    "id": o["id"],
                }
                for g in groups
                for o in g["objects"]
            ]

        payload = {
            "deleted": {
                "count": deleted_count,
                "grouped_objects": deleted_groups,
                "related_objects": flatten(deleted_groups),
                "message": "The following objects will be permanently deleted.",
                "level": "warning",
            },
            "affected": {
                "count": affected_count,
                "grouped_objects": affected_groups,
                "related_objects": flatten(affected_groups),
                "message": "These objects will NOT be deleted but will lose one or more relationships.",
                "level": "info",
            },
        }
        return Response(payload)

    def _get_optimized_object_data(self, queryset):
        """
        Calculate folder full paths for objects in the queryset in 1 DB request.
        """
        initial_objects = list(queryset)
        if not initial_objects:
            return {}

        path_results = {}
        folders = {f.id: f for f in Folder.objects.all()}
        for obj in initial_objects:
            path = []
            if hasattr(obj, "folder"):
                queue = deque([obj.folder.id])
            elif hasattr(obj, "parent_folder") and obj.parent_folder:
                queue = deque([obj.parent_folder.id])
            else:
                continue
            while queue:
                folder_id = queue.popleft()
                folder = folders[folder_id]
                if folder.parent_folder:
                    path.append(
                        {
                            "str": str(folder),
                            "id": folder.id,
                            "parent_id": folder.parent_folder.id,
                        }
                    )
                    queue.append(folder.parent_folder.id)
            path_results[obj.id] = path[::-1]  # Reverse to get root to leaf order

        return {
            "paths": path_results,
        }

    class Meta:
        abstract = True

    @action(detail=True, name="Get write data")
    def object(self, request, pk):
        serializer_class = self.get_serializer_class(action="update")

        return Response(serializer_class(super().get_object()).data)


# Content types


class ContentTypeListView(APIView):
    permission_classes = [permissions.IsAuthenticated]

    @method_decorator(cache_page(60 * MED_CACHE_TTL))
    def get(self, request, format=None):
        content_types = []
        for model in apps.get_models():
            content_types.append(
                {"label": model.__name__, "value": model._meta.model_name}
            )
        content_types.sort(key=lambda x: x["label"].lower())
        return Response(content_types)


# Risk Assessment


class PerimeterFilter(GenericFilterSet):
    folder = df.ModelMultipleChoiceFilter(
        queryset=Folder.objects.all(),
    )
    lc_status = df.MultipleChoiceFilter(
        choices=Perimeter.PRJ_LC_STATUS, lookup_expr="icontains"
    )

    class Meta:
        model = Perimeter
        fields = ["name", "folder", "lc_status", "campaigns"]


class PerimeterViewSet(BaseModelViewSet):
    """
    API endpoint that allows perimeters to be viewed or edited.
    """

    model = Perimeter
    filterset_class = PerimeterFilter
    search_fields = ["name", "ref_id", "description"]
    filterset_fields = ["name", "folder", "campaigns"]

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get status choices")
    def lc_status(self, request):
        return Response(dict(Perimeter.PRJ_LC_STATUS))

    @action(detail=False, methods=["get"])
    def names(self, request):
        uuid_list = request.query_params.getlist("id[]", [])
        queryset = Perimeter.objects.filter(id__in=uuid_list)

        return Response({str(perimeter.id): perimeter.name for perimeter in queryset})

    @action(detail=False, methods=["get"])
    def quality_check(self, request):
        """
        Returns the quality check of the perimeters
        """
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            folder=Folder.get_root_folder(), user=request.user, object_type=Perimeter
        )
        perimeters = Perimeter.objects.filter(id__in=viewable_objects)
        res = {
            str(p.id): {
                "perimeter": PerimeterReadSerializer(p).data,
                "compliance_assessments": {"objects": {}},
                "risk_assessments": {"objects": {}},
            }
            for p in perimeters
        }
        for compliance_assessment in ComplianceAssessment.objects.filter(
            perimeter__in=perimeters
        ):
            res[str(compliance_assessment.perimeter.id)]["compliance_assessments"][
                "objects"
            ][str(compliance_assessment.id)] = {
                "object": ComplianceAssessmentReadSerializer(
                    compliance_assessment
                ).data,
                "quality_check": compliance_assessment.quality_check(),
            }
        for risk_assessment in RiskAssessment.objects.filter(perimeter__in=perimeters):
            res[str(risk_assessment.perimeter.id)]["risk_assessments"]["objects"][
                str(risk_assessment.id)
            ] = {
                "object": RiskAssessmentReadSerializer(risk_assessment).data,
                "quality_check": risk_assessment.quality_check(),
            }
        return Response({"results": res})

    @action(detail=True, methods=["get"], url_path="quality_check")
    def quality_check_detail(self, request, pk):
        """
        Returns the quality check of the perimeter
        """
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            folder=Folder.get_root_folder(), user=request.user, object_type=Perimeter
        )
        if UUID(pk) in viewable_objects:
            perimeter = self.get_object()
            res = {
                "perimeter": PerimeterReadSerializer(perimeter).data,
                "compliance_assessments": {"objects": {}},
                "risk_assessments": {"objects": {}},
            }
            for compliance_assessment in ComplianceAssessment.objects.filter(
                perimeter=perimeter
            ):
                res["compliance_assessments"]["objects"][
                    str(compliance_assessment.id)
                ] = {
                    "object": ComplianceAssessmentReadSerializer(
                        compliance_assessment
                    ).data,
                    "quality_check": compliance_assessment.quality_check(),
                }
            for risk_assessment in RiskAssessment.objects.filter(perimeter=perimeter):
                res["risk_assessments"]["objects"][str(risk_assessment.id)] = {
                    "object": RiskAssessmentReadSerializer(risk_assessment).data,
                    "quality_check": risk_assessment.quality_check(),
                }
            return Response(res)
        else:
            return Response(status=status.HTTP_403_FORBIDDEN)

    @action(detail=False, methods=["get"])
    def ids(self, request):
        my_map = dict()

        (viewable_items, _, _) = RoleAssignment.get_accessible_object_ids(
            folder=Folder.get_root_folder(),
            user=request.user,
            object_type=Perimeter,
        )
        for item in Perimeter.objects.filter(id__in=viewable_items):
            if my_map.get(item.folder.name) is None:
                my_map[item.folder.name] = {}
            my_map[item.folder.name].update({item.name: item.id})

        return Response(my_map)


class ThreatViewSet(BaseModelViewSet):
    """
    API endpoint that allows threats to be viewed or edited.
    """

    model = Threat
    filterset_fields = [
        "folder",
        "provider",
        "library",
        "risk_scenarios",
        "filtering_labels",
    ]
    search_fields = ["name", "provider", "description"]

    def get_queryset(self):
        return (
            super()
            .get_queryset()
            .select_related(
                "folder",
                "folder__parent_folder",  # For get_folder_full_path() optimization
                "library",  # FieldsRelatedField includes library
            )
            .prefetch_related(
                "filtering_labels__folder",  # FieldsRelatedField includes folder
            )
        )

    def list(self, request, *args, **kwargs):
        return super().list(request, *args, **kwargs)

    def retrieve(self, request, *args, **kwargs):
        return super().retrieve(request, *args, **kwargs)

    @action(detail=False, name="Get provider choices")
    def provider(self, request):
        providers = set(
            Threat.objects.filter(provider__isnull=False).values_list(
                "provider", flat=True
            )
        )
        return Response({p: p for p in providers})

    @action(detail=False, name="Get threats count")
    def threats_count(self, request):
        folder_id = request.query_params.get("folder", None)
        return Response({"results": threats_count_per_name(request.user, folder_id)})

    @action(detail=False, methods=["get"])
    def ids(self, request):
        my_map = dict()

        (viewable_items, _, _) = RoleAssignment.get_accessible_object_ids(
            folder=Folder.get_root_folder(),
            user=request.user,
            object_type=Threat,
        )
        for item in Threat.objects.filter(id__in=viewable_items):
            if my_map.get(item.folder.name) is None:
                my_map[item.folder.name] = {}
            my_map[item.folder.name].update({item.name: item.id})
        return Response(my_map)


class AssetFilter(GenericFilterSet):
    folder = df.ModelMultipleChoiceFilter(queryset=Folder.objects.all())
    asset_class = df.ModelMultipleChoiceFilter(queryset=AssetClass.objects.all())

    exclude_children = df.ModelChoiceFilter(
        queryset=Asset.objects.all(),
        method="filter_exclude_children",
        label="Exclude children",
    )
    exclude_parents = df.ModelChoiceFilter(
        queryset=Asset.objects.all(),
        method="filter_exclude_parents",
        label="Exclude parents",
    )

    def filter_exclude_children(self, queryset, name, value):
        descendants = value.get_descendants()
        return queryset.exclude(id__in=[descendant.id for descendant in descendants])

    def filter_exclude_parents(self, queryset, name, value):
        ancestors = value.ancestors_plus_self()
        return queryset.exclude(id__in=[ancestor.id for ancestor in ancestors])

    class Meta:
        model = Asset
        fields = [
            "name",
            "type",
            "parent_assets",
            "exclude_children",
            "exclude_parents",
            "ebios_rm_studies",
            "risk_scenarios",
            "security_exceptions",
            "applied_controls",
            "filtering_labels",
            "personal_data",
            "is_business_function",
            "dora_licenced_activity",
            "dora_criticality_assessment",
            "dora_discontinuing_impact",
            "organisation_objectives",
        ]


class AssetCapabilityViewSet(BaseModelViewSet):
    model = AssetCapability
    search_fields = ["name"]


class AssetViewSet(ExportMixin, BaseModelViewSet):
    """
    API endpoint that allows assets to be viewed or edited.
    """

    model = Asset
    filterset_class = AssetFilter
    search_fields = ["name", "description", "ref_id", "folder__name"]
    ordering = ["folder__name", "name"]

    def get_queryset(self) -> models.query.QuerySet:
        qs = super().get_queryset().select_related("asset_class", "folder")
        if self.action == "autocomplete":
            return qs
        return qs.prefetch_related(
            "parent_assets",
            "child_assets",
            "owner",
            "security_exceptions",
            "filtering_labels",
            "personal_data",
            "overridden_children_capabilities",
        )

    def _get_optimized_object_data(self, queryset):
        """
        Extends the base optimization to add asset-specific data for objectives
        and descendants, ensuring maximum query efficiency.
        """
        optimized_data = super()._get_optimized_object_data(queryset)
        initial_assets = list(queryset)
        if not initial_assets:
            return optimized_data

        graph_data = Asset._prefetch_graph_data(initial_assets)
        child_to_parents = graph_data["child_to_parents"]
        parent_to_children = graph_data["parent_to_children"]

        scale = Asset._get_security_objective_scale()
        sec_obj_results = {}
        dro_obj_results = {}
        sec_cap_results = {}
        rec_cap_results = {}
        descendant_results = {}

        for asset in initial_assets:
            # Calculate and store descendants.
            descendants = Asset._get_all_descendants(asset, parent_to_children)
            descendant_results[asset.id] = [
                {"id": str(d.id), "str": str(d)} for d in descendants
            ]

            # Calculate and store objectives.
            if asset.is_primary:
                sec_obj = asset.security_objectives.get("objectives", {})
                dro_obj = asset.disaster_recovery_objectives.get("objectives", {})

                # For primary assets, aggregate capabilities from supporting descendants
                supporting_descendants = {d for d in descendants if not d.is_primary}
                sec_cap = Asset._aggregate_security_capabilities(
                    supporting_descendants, asset
                )
                rec_cap = Asset._aggregate_recovery_capabilities(
                    supporting_descendants, asset
                )
            else:
                ancestors = Asset._get_all_ancestors(asset, child_to_parents)
                primary_ancestors = {anc for anc in ancestors if anc.is_primary}
                sec_obj = Asset._aggregate_security_objectives(primary_ancestors)
                dro_obj = Asset._aggregate_dro_objectives(primary_ancestors)

                # For supporting assets, use stored capabilities
                sec_cap = asset.security_capabilities.get("objectives", {})
                rec_cap = asset.recovery_capabilities.get("objectives", {})

            sec_obj_results[asset.id] = self._format_security_objectives(sec_obj, scale)
            dro_obj_results[asset.id] = self._format_disaster_recovery_objectives(
                dro_obj
            )
            sec_cap_results[asset.id] = self._format_security_objectives(sec_cap, scale)
            rec_cap_results[asset.id] = self._format_disaster_recovery_objectives(
                rec_cap
            )

        optimized_data.update(
            {
                "security_objectives": sec_obj_results,
                "disaster_recovery_objectives": dro_obj_results,
                "security_capabilities": sec_cap_results,
                "recovery_capabilities": rec_cap_results,
                "descendants": descendant_results,
            }
        )
        return optimized_data

    def _format_security_objectives(self, objectives, scale):
        if not objectives:
            return []
        return [
            {key: Asset.SECURITY_OBJECTIVES_SCALES[scale][content.get("value", 0)]}
            for key, content in sorted(
                objectives.items(),
                key=lambda x: (
                    Asset.DEFAULT_SECURITY_OBJECTIVES.index(x[0])
                    if x[0] in Asset.DEFAULT_SECURITY_OBJECTIVES
                    else -1
                ),
            )
            if content.get("is_enabled", False)
            and content.get("value", -1) in range(0, 5)
        ]

    def _format_disaster_recovery_objectives(self, objectives):
        if not objectives:
            return []

        def format_seconds(s: int) -> str:
            if not isinstance(s, int) or s < 0:
                return "0s"
            h, r = divmod(s, 3600)
            m, s_rem = divmod(r, 60)
            parts = []
            if h > 0:
                parts.append(f"{h}h")
            if m > 0:
                parts.append(f"{m:02d}m")
            if s_rem > 0 or not parts:
                parts.append(f"{s_rem:02d}s")
            return "".join(parts)

        return [
            {"str": f"{key.upper()}: {format_seconds(content.get('value', 0))}"}
            for key, content in sorted(
                objectives.items(),
                key=lambda x: (
                    Asset.DEFAULT_DISASTER_RECOVERY_OBJECTIVES.index(x[0])
                    if x[0] in Asset.DEFAULT_DISASTER_RECOVERY_OBJECTIVES
                    else -1
                ),
            )
            if content.get("value") is not None and content.get("value") > 0
        ]

    @action(detail=False, name="Lightweight autocomplete search")
    def autocomplete(self, request):
        """Minimal endpoint for autocomplete selects — skips graph traversal."""
        from core.serializers import AssetAutocompleteSerializer

        qs = self.filter_queryset(self.get_queryset())
        page = self.paginate_queryset(qs)
        objects = page if page is not None else qs
        serializer = AssetAutocompleteSerializer(objects, many=True)
        data = serializer.data
        field_models = self._get_fieldsrelated_map(serializer)
        if field_models:
            allowed_ids = self._get_accessible_ids_map(set(field_models.values()))
            data = self._filter_related_fields(data, field_models, allowed_ids)
        if page is not None:
            return self.get_paginated_response(data)
        return Response(data)

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get type choices")
    def type(self, request):
        return Response(dict(Asset.Type.choices))

    @action(detail=False, name="Get asset class choices")
    def asset_class(self, request):
        # This endpoint is exclusively for filters.
        return Response(
            [
                {"id": ac.id, "name": ac.full_path}
                for ac in AssetClass.objects.filter(assets__isnull=False).distinct()
            ]
        )

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get DORA licensed activity choices")
    def dora_licenced_activity(self, request):
        return Response(dict(dora.DORA_LICENSED_ACTIVITY_CHOICES))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get DORA criticality assessment choices")
    def dora_criticality_assessment(self, request):
        return Response(dict(dora.DORA_YES_NO_ASSESSMENT_CHOICES))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get DORA discontinuing impact choices")
    def dora_discontinuing_impact(self, request):
        return Response(dict(dora.DORA_DISCONTINUING_IMPACT_CHOICES))

    @action(detail=True, name="Get asset write data")
    def object(self, request, pk):
        serializer_class = self.get_serializer_class(action="update")
        asset_data = serializer_class(super().get_object()).data

        general_settings = GlobalSettings.objects.filter(name="general").first()
        scale_key = (
            general_settings.value.get("security_objective_scale", "1-4")
            if general_settings
            else "1-4"
        )
        objective_scale = Asset.SECURITY_OBJECTIVES_SCALES[scale_key]

        objectives = asset_data["security_objectives"].get("objectives", {})
        security_capabilities = asset_data["security_capabilities"].get(
            "objectives", {}
        )
        reduced_objective_scale: dict[str, int] = {
            value: index
            for index, value in enumerate(objective_scale)
            if value not in objective_scale[:index]
        }

        for objective_dict in [objectives, security_capabilities]:
            for objective_data in objective_dict.values():
                if (objective_value := objective_data.get("value")) is None:
                    continue
                objective_label = objective_scale[objective_value]
                reduced_value = reduced_objective_scale[objective_label]
                objective_data["value"] = reduced_value

        return Response(asset_data)

    @action(detail=False, name="Get assets graph")
    def graph(self, request):
        nodes = []
        links = []
        nodes_idx = dict()
        categories = []
        N = 0
        hide_domains = (
            request.query_params.get("hide_domains", "false").lower() == "true"
        )

        (viewable_folders, _, _) = RoleAssignment.get_accessible_object_ids(
            folder=Folder.get_root_folder(),
            user=request.user,
            object_type=Folder,
        )
        (viewable_assets, _, _) = RoleAssignment.get_accessible_object_ids(
            folder=Folder.get_root_folder(),
            user=request.user,
            object_type=Asset,
        )
        # Build category index mapping first (key by UUID to avoid name collisions)
        domain_to_category = {}
        for domain in Folder.objects.filter(id__in=viewable_folders):
            categories.append({"name": domain.name})
            domain_to_category[domain.id] = len(categories) - 1

            if not hide_domains:
                nodes_idx[domain.id] = N
                nodes.append(
                    {
                        "name": domain.name,
                        "category": domain_to_category[domain.id],
                        "symbol": "roundRect",
                        "symbolSize": 30,
                        "value": "Domain",
                        "pk": str(domain.id),
                    }
                )
                N += 1
        for asset in Asset.objects.filter(id__in=viewable_assets):
            # Only include assets whose folders are also viewable to avoid KeyError
            if asset.folder.id not in viewable_folders:
                continue

            symbol = "circle"
            if asset.type == "PR":
                symbol = "diamond"
            # Use domain.name/asset.name as unique key to handle duplicate asset names across domains
            asset_key = f"{asset.folder.name}/{asset.name}"
            nodes.append(
                {
                    "name": asset.name,
                    "symbol": symbol,
                    "symbolSize": 25,
                    "category": domain_to_category[asset.folder.id],
                    "value": "Primary" if asset.type == "PR" else "Support",
                    "pk": str(asset.id),
                }
            )
            nodes_idx[asset_key] = N
            N += 1

        # Add links between domains (folders) based on parent-child relationships
        if not hide_domains:
            for domain in Folder.objects.filter(id__in=viewable_folders):
                if domain.parent_folder and domain.parent_folder.id in viewable_folders:
                    links.append(
                        {
                            "source": nodes_idx[domain.parent_folder.id],
                            "target": nodes_idx[domain.id],
                            "value": "contains",
                        }
                    )

        # Add links between assets and their domains
        if not hide_domains:
            for asset in Asset.objects.filter(id__in=viewable_assets):
                # Only include assets whose folders are also viewable to avoid KeyError
                if asset.folder.id not in viewable_folders:
                    continue

                asset_key = f"{asset.folder.name}/{asset.name}"
                links.append(
                    {
                        "source": nodes_idx[asset.folder.id],
                        "target": nodes_idx[asset_key],
                        "value": "contains",
                    }
                )

        # Add links between assets (existing relationships)
        for asset in Asset.objects.filter(id__in=viewable_assets):
            # Only include assets whose folders are also viewable to avoid KeyError
            if asset.folder.id not in viewable_folders:
                continue

            asset_key = f"{asset.folder.name}/{asset.name}"
            for relationship in asset.parent_assets.all():
                # Only include relationship if both assets and their folders are viewable
                if (
                    relationship.id not in viewable_assets
                    or relationship.folder.id not in viewable_folders
                ):
                    continue

                relationship_key = f"{relationship.folder.name}/{relationship.name}"
                links.append(
                    {
                        "source": nodes_idx[relationship_key],
                        "target": nodes_idx[asset_key],
                        "value": "depends on",
                    }
                )
        meta = {"display_name": "Assets Explorer"}
        return Response(
            {"nodes": nodes, "links": links, "categories": categories, "meta": meta}
        )

    @action(detail=False, methods=["get"])
    def ids(self, request):
        my_map = dict()

        (viewable_items, _, _) = RoleAssignment.get_accessible_object_ids(
            folder=Folder.get_root_folder(),
            user=request.user,
            object_type=Asset,
        )
        for item in Asset.objects.filter(id__in=viewable_items):
            if my_map.get(item.folder.name) is None:
                my_map[item.folder.name] = {}
            my_map[item.folder.name].update({item.name: item.id})
        return Response(my_map)

    @action(detail=False, name="Get security objectives")
    def security_objectives(self, request):
        return Response({"results": Asset.DEFAULT_SECURITY_OBJECTIVES})

    @action(detail=False, name="Get disaster recovery objectives")
    def disaster_recovery_objectives(self, request):
        return Response({"results": Asset.DEFAULT_DISASTER_RECOVERY_OBJECTIVES})

    export_config = {
        "fields": {
            "internal_id": {"source": "id", "label": "internal_id"},
            "name": {"source": "name", "label": "name", "escape": True},
            "description": {
                "source": "description",
                "label": "description",
                "escape": True,
            },
            "ref_id": {"source": "ref_id", "label": "ref_id", "escape": True},
            "type": {"source": "type", "label": "type"},
            "asset_class": {
                "source": "asset_class",
                "label": "asset_class",
                # Handle missing asset_class safely and trim a leading 'assetClass/' segment if present
                "format": lambda ac: (
                    (
                        ac.full_path.replace("assetClass/", "", 1)
                        if ac.full_path.startswith("assetClass/")
                        else ac.full_path.replace("assetClass", "")
                    )
                    if ac
                    else ""
                ),
                "escape": True,
            },
            "folder": {"source": "folder.name", "label": "folder", "escape": True},
            "security_objectives": {
                "source": "get_security_objectives_display",
                "label": "security_objectives",
                "format": lambda objs: ",".join(
                    f"{k}: {v}" for obj in objs for k, v in obj.items()
                ),
                "escape": True,
            },
            "disaster_recovery_objectives": {
                "source": "get_disaster_recovery_objectives_display",
                "label": "disaster_recovery_objectives",
                "format": lambda objs: ",".join(i["str"] for i in objs),
                "escape": True,
            },
            "security_capabilities": {
                "source": "get_security_capabilities_display",
                "label": "security_capabilities",
                "format": lambda objs: ",".join(
                    f"{k}: {v}" for obj in objs for k, v in obj.items()
                ),
                "escape": True,
            },
            "recovery_capabilities": {
                "source": "get_recovery_capabilities_display",
                "label": "recovery_capabilities",
                "format": lambda objs: ",".join(i["str"] for i in objs),
                "escape": True,
            },
            "link": {"source": "reference_link", "label": "link", "escape": True},
            "owners": {
                "source": "owner",
                "label": "owners",
                "format": lambda qs: ",".join(
                    escape_excel_formula(str(o)) for o in qs.all()
                ),
            },
            "parent_assets": {
                "source": "parent_assets",
                "label": "parent_assets",
                "format": lambda qs: ",".join(
                    escape_excel_formula(o.name) for o in qs.all()
                ),
            },
            "labels": {
                "source": "filtering_labels",
                "label": "labels",
                "format": lambda qs: ",".join(
                    escape_excel_formula(o.label) for o in qs.all()
                ),
            },
            "observation": {
                "source": "observation",
                "label": "observation",
                "escape": True,
            },
        },
        "wrap_columns": ["name", "description", "observation"],
        "filename": "assets_export",
        "select_related": ["folder", "asset_class"],
        "prefetch_related": [
            "owner",
            "parent_assets",
            "overridden_children_capabilities",
            "filtering_labels",
        ],
    }

    @action(detail=False, methods=["post"], url_path="batch-create")
    def batch_create(self, request):
        """
        Batch create multiple assets from a text list with parent-child relationships.
        Expected format:
        {
            "assets_text": "PR:Parent Asset\\n  SP:Child Asset 1\\n  SP:Child Asset 2",
            "folder": "folder-uuid"
        }
        Lines with leading spaces (2+) are children of the previous non-indented line.
        """
        try:
            assets_text = request.data.get("assets_text", "")
            folder_id = request.data.get("folder")

            if not assets_text:
                return Response(
                    {"error": "assets_text is required"},
                    status=status.HTTP_400_BAD_REQUEST,
                )

            if not folder_id:
                return Response(
                    {"error": "folder is required"},
                    status=status.HTTP_400_BAD_REQUEST,
                )

            try:
                folder = Folder.objects.get(id=uuid.UUID(str(folder_id)))
            except (ValueError, AttributeError, Folder.DoesNotExist):
                return Response(
                    {"error": "Folder not found"},
                    status=status.HTTP_404_NOT_FOUND,
                )

            # Parse the assets text with indentation (2 spaces per level)
            lines = [line.rstrip() for line in assets_text.split("\n") if line.strip()]
            created_assets = []
            reused_assets = []
            errors = []
            depth_stack = []  # Stack of assets at each depth level

            for line in lines:
                # Count leading spaces and calculate depth (2 spaces = 1 level)
                leading_spaces = len(line) - len(line.lstrip())
                depth = leading_spaces // 2
                line_content = line.strip()

                # Check for type prefix (SP: or PR:)
                asset_type = Asset.Type.SUPPORT  # default
                asset_name = line_content

                if line_content.upper().startswith("SP:"):
                    asset_type = Asset.Type.SUPPORT
                    asset_name = line_content[3:].strip()
                elif line_content.upper().startswith("PR:"):
                    asset_type = Asset.Type.PRIMARY
                    asset_name = line_content[3:].strip()

                if not asset_name:
                    errors.append({"line": line_content, "error": "Empty asset name"})
                    continue

                # Trim stack to current depth
                depth_stack = depth_stack[:depth]

                # Parent is the asset at the previous depth level (skip None entries from errors)
                parent_asset = None
                if depth_stack:
                    # Find the last non-None entry in the stack
                    for i in range(len(depth_stack) - 1, -1, -1):
                        if depth_stack[i] is not None:
                            parent_asset = depth_stack[i]
                            break

                # Check if asset already exists in the folder
                existing_asset = Asset.objects.filter(
                    name=asset_name, folder=folder
                ).first()

                if existing_asset:
                    # Reuse existing asset
                    asset = existing_asset

                    # Update parent relationship if needed and parent exists
                    if parent_asset and parent_asset not in asset.parent_assets.all():
                        asset.parent_assets.add(parent_asset)

                    # Add to stack for potential children
                    depth_stack.append(asset)

                    reused_assets.append(
                        {
                            "id": str(asset.id),
                            "name": asset.name,
                            "type": asset.get_type_display(),
                            "parent": parent_asset.name if parent_asset else None,
                            "depth": depth,
                        }
                    )
                else:
                    # Create new asset using the serializer to respect IAM
                    asset_data = {
                        "name": asset_name,
                        "type": asset_type,
                        "folder": str(folder.id),
                    }

                    # Add parent relationship if exists
                    if parent_asset:
                        asset_data["parent_assets"] = [parent_asset.id]

                    serializer = AssetWriteSerializer(
                        data=asset_data, context={"request": request}
                    )

                    if serializer.is_valid():
                        try:
                            asset = serializer.save()
                        except PermissionDenied as e:
                            return Response(
                                {"error": e.detail},
                                status=status.HTTP_403_FORBIDDEN,
                            )

                        # Add to stack for potential children
                        depth_stack.append(asset)

                        created_assets.append(
                            {
                                "id": str(asset.id),
                                "name": asset.name,
                                "type": asset.get_type_display(),
                                "parent": parent_asset.name if parent_asset else None,
                                "depth": depth,
                            }
                        )
                    else:
                        # Error creating asset - add None to stack to maintain depth
                        depth_stack.append(None)
                        errors.append(
                            {
                                "line": line_content,
                                "errors": serializer.errors,
                            }
                        )

            return Response(
                {
                    "created": len(created_assets),
                    "reused": len(reused_assets),
                    "assets": created_assets,
                    "reused_assets": reused_assets,
                    "errors": errors,
                },
                status=status.HTTP_201_CREATED
                if created_assets or reused_assets
                else status.HTTP_400_BAD_REQUEST,
            )

        except Exception as e:
            logger.error(f"Error in batch asset creation: {str(e)}")
            return Response(
                {"error": str(e)},
                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
            )


class AssetClassViewSet(BaseModelViewSet):
    model = AssetClass
    filterset_fields = ["parent"]

    ordering = ["parent", "name"]
    search_fields = ["name", "description"]

    @action(detail=False, name="Get Asset Class Tree")
    def tree(self, request):
        return Response(AssetClass.build_tree())


class ReferenceControlViewSet(BaseModelViewSet):
    """
    API endpoint that allows reference controls to be viewed or edited.
    """

    model = ReferenceControl
    filterset_fields = [
        "folder",
        "category",
        "csf_function",
        "provider",
        "findings",
        "filtering_labels",
    ]
    search_fields = ["name", "description", "provider", "ref_id"]

    @action(detail=False, name="Get provider choices")
    def provider(self, request):
        providers = set(
            ReferenceControl.objects.filter(provider__isnull=False).values_list(
                "provider", flat=True
            )
        )
        return Response({p: p for p in providers})

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get category choices")
    def category(self, request):
        return Response(dict(ReferenceControl.CATEGORY))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get function choices")
    def csf_function(self, request):
        return Response(dict(ReferenceControl.CSF_FUNCTION))


class RiskMatrixViewSet(BaseModelViewSet):
    """
    API endpoint that allows risk matrices to be viewed or edited.
    """

    model = RiskMatrix
    filterset_fields = ["folder", "is_enabled", "provider"]

    @action(detail=False)  # Set a name there
    def colors(self, request):
        return Response({"results": get_risk_color_ordered_list(request.user)})

    @action(detail=False, name="Get provider choices")
    def provider(self, request):
        providers = set(
            RiskMatrix.objects.filter(provider__isnull=False).values_list(
                "provider", flat=True
            )
        )
        return Response({p: p for p in providers})

    @action(detail=False, name="Get risk level choices")
    def risk(self, request):
        current_language = get_language()
        viewable_matrices: list[RiskMatrix] = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, RiskMatrix
        )[0]
        undefined = {-1: "--"}
        options = undefined
        for matrix in RiskMatrix.objects.filter(id__in=viewable_matrices):
            _choices = {}
            for i, risk in enumerate(matrix.json_definition["risk"]):
                translations = risk.get("translations")
                if not isinstance(translations, dict):
                    translations = {}
                translated = translations.get(current_language, {})

                # Use the translated name if available, otherwise fall back to the default name
                name = translated.get("name") or risk.get("name", "")
                _choices[risk.get("id", i)] = name
            options = options | _choices

        res = [{"value": k, "label": v} for k, v in options.items()]
        return Response(res)

    @action(detail=False, name="Get impact choices")
    def impact(self, request):
        viewable_matrices: list[RiskMatrix] = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, RiskMatrix
        )[0]
        undefined = {-1: "--"}
        options = []
        for matrix in RiskMatrix.objects.filter(id__in=viewable_matrices):
            _choices = {
                i: name
                for i, name in enumerate(
                    x["name"] for x in matrix.json_definition["impact"]
                )
            }
            choices = undefined | _choices
            options = options | choices.items()
        return Response(
            [{k: v for k, v in zip(("value", "label"), o)} for o in options]
        )

    @action(detail=False, name="Get probability choices")
    def probability(self, request):
        viewable_matrices: list[RiskMatrix] = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, RiskMatrix
        )[0]
        undefined = {-1: "--"}
        options = []
        for matrix in RiskMatrix.objects.filter(id__in=viewable_matrices):
            _choices = {
                i: name
                for i, name in enumerate(
                    x["name"] for x in matrix.json_definition["probability"]
                )
            }
            choices = undefined | _choices
            options = options | choices.items()
        return Response(
            [{k: v for k, v in zip(("value", "label"), o)} for o in options]
        )

    @action(detail=False, name="Get used risk matrices")
    def used(self, request):
        viewable_matrices = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, RiskMatrix
        )[0]
        viewable_assessments = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, RiskAssessment
        )[0]
        _used_matrices = (
            RiskMatrix.objects.filter(riskassessment__isnull=False)
            .filter(id__in=viewable_matrices)
            .filter(riskassessment__id__in=viewable_assessments)
            .distinct()
        )
        used_matrices = _used_matrices.values("id", "name")
        for i in range(len(used_matrices)):
            used_matrices[i]["risk_assessments_count"] = (
                RiskAssessment.objects.filter(risk_matrix=_used_matrices[i].id)
                .filter(id__in=viewable_assessments)
                .count()
            )
        return Response({"results": used_matrices})

    @action(detail=False, methods=["get"])
    def ids(self, request):
        my_map = dict()

        (viewable_items, _, _) = RoleAssignment.get_accessible_object_ids(
            folder=Folder.get_root_folder(),
            user=request.user,
            object_type=RiskMatrix,
        )
        for item in RiskMatrix.objects.filter(id__in=viewable_items):
            if my_map.get(item.folder.name) is None:
                my_map[item.folder.name] = {}
            my_map[item.folder.name].update({item.name: item.id})

        return Response(my_map)

    # --- Visual Matrix Editor actions ---

    def _check_change_permission(self, request, matrix):
        """Check that the user has change_riskmatrix permission on the matrix's folder."""
        if not RoleAssignment.is_access_allowed(
            user=request.user,
            perm=Permission.objects.get(codename="change_riskmatrix"),
            folder=matrix.folder,
        ):
            raise PermissionDenied({"error": "Permission denied."})

    @action(detail=True, methods=["post"], url_path="start-editing")
    def start_editing(self, request, pk=None):
        """Copy json_definition into editing_draft to begin editing."""
        matrix = self.get_object()
        self._check_change_permission(request, matrix)
        if matrix.urn:
            return Response(
                {
                    "error": "Library matrices cannot be edited directly. Use Clone instead."
                },
                status=status.HTTP_403_FORBIDDEN,
            )
        if matrix.editing_draft is not None:
            return Response(
                {"status": "already_editing", "editing_draft": matrix.editing_draft}
            )
        draft = copy.deepcopy(matrix.json_definition)
        # Include current metadata in the draft
        meta = {
            "name": matrix.name,
            "description": matrix.description or "",
            "provider": matrix.provider or "",
            "locale": matrix.locale or "en",
            "folder": str(matrix.folder_id),
        }
        if matrix.translations:
            meta["translations"] = copy.deepcopy(matrix.translations)
        draft["_meta"] = meta
        matrix.editing_draft = draft
        matrix.save(update_fields=["editing_draft", "updated_at"])
        return Response(
            {"status": "editing_started", "editing_draft": matrix.editing_draft}
        )

    @action(detail=True, methods=["patch"], url_path="save-draft")
    def save_draft(self, request, pk=None):
        """Update editing_draft with current WIP. Metadata stays draft-scoped until publish."""
        matrix = self.get_object()
        self._check_change_permission(request, matrix)
        editing_draft = request.data.get("editing_draft")
        if editing_draft is None:
            return Response(
                {"error": "editing_draft is required."},
                status=status.HTTP_400_BAD_REQUEST,
            )

        # Store metadata inside editing_draft so it stays draft-scoped
        meta = {}
        for field in ("name", "description", "provider", "locale", "folder"):
            if field in request.data:
                meta[field] = request.data[field]
        if request.data.get("metaTranslations"):
            meta["translations"] = request.data["metaTranslations"]
        if meta:
            editing_draft["_meta"] = meta

        matrix.editing_draft = editing_draft
        matrix.save(update_fields=["editing_draft", "updated_at"])
        return Response({"status": "draft_saved"})

    @action(detail=True, methods=["post"], url_path="publish-draft")
    def publish_draft(self, request, pk=None):
        """Publish editing_draft → json_definition, snapshot history, bump version."""
        matrix = self.get_object()
        self._check_change_permission(request, matrix)
        if matrix.editing_draft is None:
            return Response(
                {"error": "No active draft to publish."},
                status=status.HTTP_400_BAD_REQUEST,
            )

        errors = self._validate_json_definition(matrix.editing_draft)
        if errors:
            return Response({"errors": errors}, status=status.HTTP_400_BAD_REQUEST)

        # Snapshot current live definition into history (only if it has real content)
        if matrix.json_definition and matrix.json_definition.get("grid"):
            history = list(matrix.editing_history or [])
            history.append(
                {
                    "version": matrix.editing_version,
                    "definition": copy.deepcopy(matrix.json_definition),
                    "published_at": timezone.now().isoformat(),
                }
            )
            matrix.editing_history = history

        # Extract draft metadata and clean it from the definition
        draft_data = copy.deepcopy(matrix.editing_draft)
        meta = draft_data.pop("_meta", {})

        # Swap draft → live
        matrix.json_definition = draft_data
        matrix.editing_draft = None
        matrix.editing_version += 1
        matrix.is_enabled = True

        # Apply draft metadata to the model
        update_fields = [
            "json_definition",
            "editing_draft",
            "editing_version",
            "editing_history",
            "is_enabled",
            "updated_at",
        ]
        for field in ("name", "description", "provider", "locale"):
            if field in meta:
                setattr(matrix, field, meta[field])
                update_fields.append(field)
        if "folder" in meta:
            matrix.folder_id = meta["folder"]
            update_fields.append("folder_id")
        if "translations" in meta:
            matrix.translations = meta["translations"]
            update_fields.append("translations")

        matrix.save(update_fields=update_fields)
        return Response(
            {"status": "published", "editing_version": matrix.editing_version}
        )

    @action(detail=True, methods=["post"], url_path="discard-draft")
    def discard_draft(self, request, pk=None):
        """Discard editing_draft without affecting json_definition."""
        matrix = self.get_object()
        self._check_change_permission(request, matrix)
        matrix.editing_draft = None
        matrix.save(update_fields=["editing_draft", "updated_at"])
        return Response({"status": "draft_discarded"})

    @action(detail=False, methods=["post"], url_path="create-draft")
    def create_draft(self, request):
        """Create a new unpublished RiskMatrix with an editing_draft for the visual editor."""
        folder_id = request.data.get("folder")
        try:
            folder = (
                Folder.objects.get(id=folder_id)
                if folder_id
                else Folder.get_root_folder()
            )
        except Folder.DoesNotExist:
            return Response(
                {"error": "Invalid folder."},
                status=status.HTTP_400_BAD_REQUEST,
            )

        if not RoleAssignment.is_access_allowed(
            user=request.user,
            perm=Permission.objects.get(codename="add_riskmatrix"),
            folder=folder,
        ):
            return Response(
                {"error": "Permission denied for this folder."},
                status=status.HTTP_403_FORBIDDEN,
            )

        matrix = RiskMatrix.objects.create(
            name=request.data.get("name", "Untitled Matrix"),
            description=request.data.get("description", ""),
            folder=folder,
            json_definition={},
            editing_draft=request.data.get("editing_draft", {}),
            is_enabled=False,
        )
        return Response(
            {
                "id": str(matrix.id),
                "name": matrix.name,
                "status": "draft_created",
            },
            status=status.HTTP_201_CREATED,
        )

    @action(detail=True, methods=["post"], url_path="create-draft-from")
    def create_draft_from(self, request, pk=None):
        """Clone an existing matrix into a new unpublished RiskMatrix with editing_draft."""

        source = self.get_object()

        if not RoleAssignment.is_access_allowed(
            user=request.user,
            perm=Permission.objects.get(codename="add_riskmatrix"),
            folder=source.folder,
        ):
            return Response(
                {"error": "Permission denied for this folder."},
                status=status.HTTP_403_FORBIDDEN,
            )

        matrix = RiskMatrix.objects.create(
            name=f"{source.name} (copy)",
            description=source.description or "",
            folder=source.folder,
            json_definition={},
            editing_draft=copy.deepcopy(source.json_definition),
            is_enabled=False,
            locale=source.locale,
            default_locale=source.default_locale,
            provider=source.provider or "",
            translations=copy.deepcopy(source.translations)
            if source.translations
            else {},
        )
        return Response(
            {
                "id": str(matrix.id),
                "name": matrix.name,
                "status": "draft_created_from",
                "editing_draft": matrix.editing_draft,
            },
            status=status.HTTP_201_CREATED,
        )

    @action(detail=True, methods=["get"], url_path="export-yaml")
    def export_yaml(self, request, pk=None):
        """Export a matrix as a library-compatible YAML file."""
        matrix = self.get_object()
        # Use editing_draft if present (WIP), otherwise published json_definition
        definition = matrix.editing_draft or matrix.json_definition
        if not definition or not definition.get("grid"):
            return Response(
                {"error": "No matrix definition to export."},
                status=status.HTTP_400_BAD_REQUEST,
            )

        ref_id = (matrix.name or "untitled").lower().replace(" ", "-").replace("_", "-")
        ref_id = "".join(c for c in ref_id if c.isalnum() or c == "-")

        library_data = {
            "urn": f"urn:custom:risk:library:risk-matrix-{ref_id}",
            "locale": matrix.locale or "en",
            "ref_id": ref_id,
            "name": matrix.name,
            "description": matrix.description or "",
            "version": matrix.editing_version,
            "provider": matrix.provider or "custom",
            "packager": "custom",
            "objects": {
                "risk_matrix": [
                    {
                        "urn": f"urn:custom:risk:matrix:{ref_id}",
                        "ref_id": ref_id,
                        "name": matrix.name,
                        "description": matrix.description or "",
                        **definition,
                    }
                ]
            },
        }

        # Add translations if present
        if matrix.translations:
            library_data["translations"] = matrix.translations
            library_data["objects"]["risk_matrix"][0]["translations"] = (
                matrix.translations
            )

        yaml_content = yaml.dump(
            library_data, allow_unicode=True, default_flow_style=False, sort_keys=False
        )

        response = HttpResponse(yaml_content, content_type="application/x-yaml")
        response["Content-Disposition"] = (
            f'attachment; filename="risk-matrix-{ref_id}.yaml"'
        )
        return response

    @action(detail=False, methods=["post"], url_path="import-yaml")
    def import_yaml(self, request):
        """Import a library YAML file and create a new draft matrix from it."""
        uploaded_file = request.FILES.get("file")
        if not uploaded_file:
            return Response(
                {"error": "No file provided."},
                status=status.HTTP_400_BAD_REQUEST,
            )

        try:
            content = uploaded_file.read()
            library_data = yaml.safe_load(content)
        except yaml.YAMLError:
            return Response(
                {"error": "Invalid YAML file."},
                status=status.HTTP_400_BAD_REQUEST,
            )

        if not isinstance(library_data, dict):
            return Response(
                {"error": "YAML must be a dictionary."},
                status=status.HTTP_400_BAD_REQUEST,
            )

        # Extract matrix definition
        matrix_def = None
        matrix_meta = {}
        risk_matrix_obj = library_data.get("objects", {}).get("risk_matrix")
        if isinstance(risk_matrix_obj, list) and risk_matrix_obj:
            first = risk_matrix_obj[0]
            if isinstance(first, dict):
                matrix_def = first
                matrix_meta = {
                    "name": library_data.get(
                        "name", matrix_def.get("name", "Imported Matrix")
                    ),
                    "description": library_data.get(
                        "description", matrix_def.get("description", "")
                    ),
                    "provider": library_data.get("provider", ""),
                    "locale": library_data.get("locale", "en"),
                }
        elif all(k in library_data for k in ("probability", "impact", "risk", "grid")):
            # Direct matrix definition (not wrapped in library structure)
            matrix_def = library_data
            matrix_meta = {
                "name": library_data.get("name", "Imported Matrix"),
                "description": library_data.get("description", ""),
                "provider": library_data.get("provider", ""),
                "locale": library_data.get("locale", "en"),
            }

        if not isinstance(matrix_def, dict) or "grid" not in matrix_def:
            return Response(
                {"error": "No valid matrix definition found in file."},
                status=status.HTTP_400_BAD_REQUEST,
            )

        # Extract only the matrix fields
        editing_draft = {}
        for key in ("probability", "impact", "risk", "grid", "strength_of_knowledge"):
            if key in matrix_def:
                editing_draft[key] = matrix_def[key]

        folder_id = request.data.get("folder")
        try:
            folder = (
                Folder.objects.get(id=folder_id)
                if folder_id
                else Folder.get_root_folder()
            )
        except Folder.DoesNotExist:
            return Response(
                {"error": "Invalid folder."},
                status=status.HTTP_400_BAD_REQUEST,
            )

        if not RoleAssignment.is_access_allowed(
            user=request.user,
            perm=Permission.objects.get(codename="add_riskmatrix"),
            folder=folder,
        ):
            return Response(
                {"error": "Permission denied for this folder."},
                status=status.HTTP_403_FORBIDDEN,
            )

        matrix = RiskMatrix.objects.create(
            name=matrix_meta["name"],
            description=matrix_meta["description"],
            provider=matrix_meta["provider"],
            locale=matrix_meta["locale"],
            folder=folder,
            json_definition={},
            editing_draft=editing_draft,
            is_enabled=False,
            translations=library_data.get("translations", {}),
        )

        return Response(
            {
                "id": str(matrix.id),
                "name": matrix.name,
                "editing_draft": matrix.editing_draft,
                "status": "imported",
            },
            status=status.HTTP_201_CREATED,
        )

    @staticmethod
    def _validate_json_definition(json_def) -> list[str]:
        """Validate the matrix JSON definition before publishing.

        Defensive against malformed payloads — returns errors instead of raising.
        """
        errors = []

        if not isinstance(json_def, dict):
            return ["Matrix definition must be a JSON object."]

        probability = json_def.get("probability")
        impact = json_def.get("impact")
        risk = json_def.get("risk")
        grid = json_def.get("grid")

        if not isinstance(probability, list):
            return ["'probability' must be a list."]
        if not isinstance(impact, list):
            return ["'impact' must be a list."]
        if not isinstance(risk, list):
            return ["'risk' must be a list."]
        if not isinstance(grid, list):
            return ["'grid' must be a list."]

        if len(probability) < 2:
            errors.append("At least 2 probability levels are required.")
        if len(impact) < 2:
            errors.append("At least 2 impact levels are required.")
        if len(risk) < 2:
            errors.append("At least 2 risk levels are required.")

        if len(grid) != len(probability):
            errors.append(
                f"Grid has {len(grid)} rows but there are {len(probability)} probability levels."
            )

        for i, row in enumerate(grid):
            if not isinstance(row, list):
                errors.append(f"Grid row {i} is not a list.")
                continue
            if len(row) != len(impact):
                errors.append(
                    f"Grid row {i} has {len(row)} columns but there are {len(impact)} impact levels."
                )
            for j, val in enumerate(row):
                if not isinstance(val, int) or val < 0 or val >= len(risk):
                    errors.append(f"Grid cell [{i}][{j}] has invalid risk index {val}.")

        for category_name, levels in [
            ("probability", probability),
            ("impact", impact),
            ("risk", risk),
        ]:
            for k, level in enumerate(levels):
                if not isinstance(level, dict):
                    errors.append(f"{category_name}[{k}] is not a valid object.")
                    continue
                if not level.get("name"):
                    errors.append(f"{category_name}[{k}] is missing a name.")
                if not level.get("hexcolor"):
                    errors.append(
                        f"{category_name}[{k}] is missing a color (hexcolor)."
                    )

        return errors


class VulnerabilityViewSet(BaseModelViewSet):
    """
    API endpoint that allows vulnerabilities to be viewed or edited.
    """

    model = Vulnerability
    filterset_fields = [
        "folder",
        "assets",
        "status",
        "severity",
        "risk_scenarios",
        "applied_controls",
        "security_exceptions",
        "filtering_labels",
        "findings",
    ]
    search_fields = ["name", "description", "ref_id"]

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get status choices")
    def status(self, request):
        return Response(dict(Vulnerability.Status.choices))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get severity choices")
    def severity(self, request):
        return Response(dict(Severity.choices))

    @action(detail=False, methods=["get"], name="Get sankey data")
    def sankey_data(self, request):
        """
        Returns vulnerability data structured for Sankey diagram:
        Folders -> Severity -> Status (as links)
        """
        folder_id = request.query_params.get("folder", None)

        # Get viewable vulnerabilities
        scoped_folder = (
            Folder.objects.get(id=folder_id) if folder_id else Folder.get_root_folder()
        )
        (object_ids, _, _) = RoleAssignment.get_accessible_object_ids(
            scoped_folder, request.user, Vulnerability
        )

        vulnerabilities = Vulnerability.objects.filter(
            id__in=object_ids
        ).select_related("folder")

        # Build link structure for Sankey: folder -> severity -> status
        links = []
        folder_severity_counts = {}
        severity_status_counts = {}

        for vuln in vulnerabilities:
            # Get folder name
            if vuln.folder:
                folder_name = vuln.folder.name
            else:
                folder_name = "No Folder"

            # Get severity label
            severity_value = vuln.severity
            severity_label = dict(Severity.choices).get(severity_value, "undefined")

            # Get status label
            status_value = vuln.status
            status_label = status_value if status_value else "--"

            # Count folder -> severity links
            folder_severity_key = f"{folder_name}||{severity_label}"
            folder_severity_counts[folder_severity_key] = (
                folder_severity_counts.get(folder_severity_key, 0) + 1
            )

            # Count severity -> status links
            severity_status_key = f"{severity_label}||{status_label}"
            severity_status_counts[severity_status_key] = (
                severity_status_counts.get(severity_status_key, 0) + 1
            )

        # Convert to Sankey link format
        for key, value in folder_severity_counts.items():
            folder, severity = key.split("||")
            links.append({"source": folder, "target": severity, "value": value})

        for key, value in severity_status_counts.items():
            severity, status_label = key.split("||")
            links.append({"source": severity, "target": status_label, "value": value})

        return Response(links)

    @action(detail=False, methods=["get"], name="Get treemap data")
    def treemap_data(self, request):
        """
        Returns vulnerability data structured for treemap visualization:
        Folders -> Severity -> Status
        """
        folder_id = request.query_params.get("folder", None)

        # Get viewable vulnerabilities
        scoped_folder = (
            Folder.objects.get(id=folder_id) if folder_id else Folder.get_root_folder()
        )
        (object_ids, _, _) = RoleAssignment.get_accessible_object_ids(
            scoped_folder, request.user, Vulnerability
        )

        vulnerabilities = Vulnerability.objects.filter(
            id__in=object_ids
        ).select_related("folder")

        # Build hierarchical structure: folder -> severity -> status
        folder_data = {}

        for vuln in vulnerabilities:
            # Get folder name
            if vuln.folder:
                folder_name = vuln.folder.name
                folder_id = str(vuln.folder.id)
            else:
                folder_name = "No Folder"
                folder_id = "no-folder"

            # Get severity label (lowercase for frontend translation)
            severity_value = vuln.severity
            severity_label = dict(Severity.choices).get(severity_value, "undefined")

            # Get status label (from the choice value, not display)
            status_value = vuln.status
            status_label = status_value if status_value else "--"

            # Initialize folder structure if needed
            if folder_name not in folder_data:
                folder_data[folder_name] = {"folder_id": folder_id, "severities": {}}

            # Initialize severity structure if needed
            if severity_label not in folder_data[folder_name]["severities"]:
                folder_data[folder_name]["severities"][severity_label] = {}

            # Count status occurrences
            if (
                status_label
                not in folder_data[folder_name]["severities"][severity_label]
            ):
                folder_data[folder_name]["severities"][severity_label][status_label] = 0

            folder_data[folder_name]["severities"][severity_label][status_label] += 1

        # Convert to treemap structure
        treemap_data = []
        for folder_name, folder_info in folder_data.items():
            folder_node = {
                "name": folder_name,
                "id": folder_info["folder_id"],
                "children": [],
            }

            for severity_name, statuses in folder_info["severities"].items():
                severity_node = {
                    "name": severity_name,
                    "id": f"{folder_info['folder_id']}-{severity_name.lower()}",
                    "children": [],
                }

                for status_name, count in statuses.items():
                    status_node = {
                        "name": status_name,
                        "id": f"{folder_info['folder_id']}-{severity_name.lower()}-{status_name.lower().replace(' ', '-')}",
                        "value": count,
                    }
                    severity_node["children"].append(status_node)

                folder_node["children"].append(severity_node)

            treemap_data.append(folder_node)

        return Response(treemap_data, status=status.HTTP_200_OK)


class FilteringLabelViewSet(BaseModelViewSet):
    """
    API endpoint that allows labels to be viewed or edited.
    """

    model = FilteringLabel
    filterset_fields = ["folder"]
    search_fields = ["label"]
    ordering = ["label"]


class LibraryFilteringLabelViewSet(BaseModelViewSet):
    """
    API endpoint that allows library labels to be viewed or edited.
    """

    model = LibraryFilteringLabel
    filterset_fields = ["folder"]
    search_fields = ["label"]
    ordering = ["label"]


class RiskAssessmentFilterSet(GenericFilterSet):
    folder = df.ModelMultipleChoiceFilter(queryset=Folder.objects.all())
    perimeter = df.ModelMultipleChoiceFilter(queryset=Perimeter.objects.all())
    risk_matrix = df.ModelMultipleChoiceFilter(queryset=RiskMatrix.objects.all())
    ebios_rm_study = df.ModelMultipleChoiceFilter(queryset=EbiosRMStudy.objects.all())

    status = df.MultipleChoiceFilter(
        choices=RiskAssessment.get_status_choices(), method="filter_status"
    )

    class Meta:
        model = RiskAssessment
        fields = {
            "name": ["exact"],
            "ref_id": ["exact"],
            "authors": ["exact"],
            "reviewers": ["exact"],
            "genericcollection": ["exact"],
        }

    def filter_status(self, queryset, name, value):
        ra_undefined_status = queryset.filter(status__isnull=True)
        if "--" in value:
            return queryset.filter(status__in=value) | ra_undefined_status
        return queryset.filter(status__in=value)


class RiskAssessmentViewSet(BaseModelViewSet):
    """
    API endpoint that allows risk assessments to be viewed or edited.
    """

    model = RiskAssessment
    filterset_class = RiskAssessmentFilterSet

    def get_queryset(self):
        queryset = super().get_queryset()
        return queryset.select_related(
            "folder",
            "perimeter",
            "perimeter__folder",
            "risk_matrix",
            "ebios_rm_study",
        ).prefetch_related(
            "authors",
            "reviewers",
            "risk_scenarios",
        )

    def perform_create(self, serializer):
        instance: RiskAssessment = serializer.save()
        if instance.ebios_rm_study:
            # Unlink all previous risk assessments from this EBIOS RM study
            RiskAssessment.objects.filter(
                ebios_rm_study=instance.ebios_rm_study
            ).exclude(id=instance.id).update(ebios_rm_study=None)

            instance.risk_matrix = instance.ebios_rm_study.risk_matrix
            ebios_rm_study = EbiosRMStudy.objects.get(id=instance.ebios_rm_study.id)
            for operational_scenario in [
                operational_scenario
                for operational_scenario in ebios_rm_study.operational_scenarios.all()
                if operational_scenario.is_selected
            ]:
                # Build comprehensive description from EBIOS RM components
                description_parts = []

                # Feared events
                ro_to = operational_scenario.ro_to
                feared_events = ro_to.feared_events.filter(is_selected=True)
                if feared_events.exists():
                    feared_events_list = []
                    for fe in feared_events:
                        gravity_display = fe.get_gravity_display()
                        gravity_text = (
                            f" [{_('Gravity').capitalize()}: {gravity_display['name']}]"
                            if gravity_display["value"] >= 0
                            else ""
                        )
                        fe_text = f"- {fe.name}{gravity_text}"
                        if fe.description:
                            fe_text += f": {fe.description}"
                        feared_events_list.append(fe_text)
                    feared_events_text = (
                        f"**{_('Feared events').capitalize()}:**\n"
                        + "\n".join(feared_events_list)
                    )
                    description_parts.append(feared_events_text)

                # Risk origin and target objective
                risk_origin_name = (
                    ro_to.risk_origin.get_name_translated
                    if hasattr(ro_to.risk_origin, "get_name_translated")
                    else str(ro_to.risk_origin)
                )
                ro_to_text = f"**{_('Risk origin').capitalize()}:** {risk_origin_name}\n**{_('Target objective').capitalize()}:** {ro_to.target_objective}"
                description_parts.append(ro_to_text)

                # Strategic scenario
                strategic_scenario = operational_scenario.attack_path.strategic_scenario
                if strategic_scenario.description:
                    description_parts.append(
                        f"**{_('Strategic Scenario').capitalize()}:** {strategic_scenario.name}\n{strategic_scenario.description}"
                    )
                elif strategic_scenario.name:
                    description_parts.append(
                        f"**{_('Strategic Scenario').capitalize()}:** {strategic_scenario.name}"
                    )

                # Attack path
                attack_path = operational_scenario.attack_path
                if attack_path.description:
                    description_parts.append(
                        f"**{_('Attack path').capitalize()}:** {attack_path.name}\n{attack_path.description}"
                    )
                elif attack_path.name:
                    description_parts.append(
                        f"**{_('Attack path').capitalize()}:** {attack_path.name}"
                    )

                # Operating modes
                operating_modes = operational_scenario.operating_modes.all()
                if operating_modes.exists():
                    operating_modes_list = []
                    for om in operating_modes:
                        likelihood_display = om.get_likelihood_display()
                        likelihood_text = (
                            f" [{_('Likelihood').capitalize()}: {likelihood_display['name']}]"
                            if likelihood_display["value"] >= 0
                            else ""
                        )
                        om_text = f"- {om.name}{likelihood_text}"
                        if om.description:
                            om_text += f": {om.description}"
                        operating_modes_list.append(om_text)
                    operating_modes_text = (
                        f"**{_('operating modes').capitalize()}:**\n"
                        + "\n".join(operating_modes_list)
                    )
                    description_parts.append(operating_modes_text)
                elif operational_scenario.operating_modes_description:
                    description_parts.append(
                        f"**{_('operating modes').capitalize()}:**\n{operational_scenario.operating_modes_description}"
                    )

                risk_scenario = RiskScenario(
                    risk_assessment=instance,
                    operational_scenario=operational_scenario,
                    name=operational_scenario.name,
                    ref_id=operational_scenario.ref_id
                    if operational_scenario.ref_id
                    else RiskScenario.get_default_ref_id(instance),
                    description="\n\n".join(description_parts),
                )
                if ff_is_enabled("inherent_risk"):
                    risk_scenario.inherent_proba = operational_scenario.likelihood
                    risk_scenario.inherent_impact = operational_scenario.gravity
                else:
                    risk_scenario.current_proba = operational_scenario.likelihood
                    risk_scenario.current_impact = operational_scenario.gravity
                risk_scenario.save()

                risk_scenario.assets.set(operational_scenario.get_assets())
                risk_scenario.threats.set(operational_scenario.threats.all())
                risk_scenario.existing_applied_controls.set(
                    operational_scenario.get_applied_controls()
                )
        instance.save()
        return super().perform_create(serializer)

    @action(detail=True, methods=["post"], name="Sync with EBIOS RM")
    def sync_from_ebios_rm(self, request, pk=None):
        """
        Synchronize an existing risk assessment with its linked EBIOS RM study.
        Updates existing risk scenarios, adds new ones, and archives outdated ones.
        """
        risk_assessment = self.get_object()

        if not risk_assessment.ebios_rm_study:
            return Response(
                {"error": "Risk assessment is not linked to an EBIOS RM study"},
                status=status.HTTP_400_BAD_REQUEST,
            )

        ebios_rm_study = risk_assessment.ebios_rm_study
        selected_operational_scenarios = [
            os for os in ebios_rm_study.operational_scenarios.all() if os.is_selected
        ]

        # Track which operational scenarios we've processed
        processed_os_ids = set()
        updated_count = 0
        created_count = 0
        archived_count = 0

        # Helper function to build description (DRY)
        def build_description(operational_scenario):
            description_parts = []

            # Feared events
            ro_to = operational_scenario.ro_to
            feared_events = ro_to.feared_events.filter(is_selected=True)
            if feared_events.exists():
                feared_events_list = []
                for fe in feared_events:
                    gravity_display = fe.get_gravity_display()
                    gravity_text = (
                        f" [{_('Gravity').capitalize()}: {gravity_display['name']}]"
                        if gravity_display["value"] >= 0
                        else ""
                    )
                    fe_text = f"- {fe.name}{gravity_text}"
                    if fe.description:
                        fe_text += f": {fe.description}"
                    feared_events_list.append(fe_text)
                feared_events_text = (
                    f"**{_('Feared events').capitalize()}:**\n"
                    + "\n".join(feared_events_list)
                )
                description_parts.append(feared_events_text)

            # Risk origin and target objective
            risk_origin_name = (
                ro_to.risk_origin.get_name_translated
                if hasattr(ro_to.risk_origin, "get_name_translated")
                else str(ro_to.risk_origin)
            )
            ro_to_text = f"**{_('Risk origin').capitalize()}:** {risk_origin_name}\n**{_('Target objective').capitalize()}:** {ro_to.target_objective}"
            description_parts.append(ro_to_text)

            # Strategic scenario
            strategic_scenario = operational_scenario.attack_path.strategic_scenario
            if strategic_scenario.description:
                description_parts.append(
                    f"**{_('Strategic Scenario').capitalize()}:** {strategic_scenario.name}\n{strategic_scenario.description}"
                )
            elif strategic_scenario.name:
                description_parts.append(
                    f"**{_('Strategic Scenario').capitalize()}:** {strategic_scenario.name}"
                )

            # Attack path
            attack_path = operational_scenario.attack_path
            if attack_path.description:
                description_parts.append(
                    f"**{_('Attack path').capitalize()}:** {attack_path.name}\n{attack_path.description}"
                )
            elif attack_path.name:
                description_parts.append(
                    f"**{_('Attack path').capitalize()}:** {attack_path.name}"
                )

            # Operating modes
            operating_modes = operational_scenario.operating_modes.all()
            if operating_modes.exists():
                operating_modes_list = []
                for om in operating_modes:
                    likelihood_display = om.get_likelihood_display()
                    likelihood_text = (
                        f" [{_('Likelihood').capitalize()}: {likelihood_display['name']}]"
                        if likelihood_display["value"] >= 0
                        else ""
                    )
                    om_text = f"- {om.name}{likelihood_text}"
                    if om.description:
                        om_text += f": {om.description}"
                    operating_modes_list.append(om_text)
                operating_modes_text = (
                    f"**{_('operating modes').capitalize()}:**\n"
                    + "\n".join(operating_modes_list)
                )
                description_parts.append(operating_modes_text)
            elif operational_scenario.operating_modes_description:
                description_parts.append(
                    f"**{_('operating modes').capitalize()}:**\n{operational_scenario.operating_modes_description}"
                )

            return "\n\n".join(description_parts)

        # Update or create risk scenarios for selected operational scenarios
        for operational_scenario in selected_operational_scenarios:
            processed_os_ids.add(operational_scenario.id)

            # Try to find existing risk scenario for this operational scenario
            risk_scenario = None

            # First, try to find by operational_scenario link
            try:
                risk_scenario = risk_assessment.risk_scenarios.get(
                    operational_scenario=operational_scenario
                )
            except RiskScenario.DoesNotExist:
                # Fallback: try to match by name (for backward compatibility with old data)
                # Remove [ARCHIVED] prefix when matching
                clean_name = operational_scenario.name.replace("[ARCHIVED] ", "")
                try:
                    risk_scenario = risk_assessment.risk_scenarios.get(
                        operational_scenario__isnull=True, name=clean_name
                    )
                    # Establish the link for this old risk scenario
                    risk_scenario.operational_scenario = operational_scenario
                except RiskScenario.DoesNotExist:
                    # Also try matching with [ARCHIVED] prefix
                    try:
                        risk_scenario = risk_assessment.risk_scenarios.get(
                            operational_scenario__isnull=True,
                            name=f"[ARCHIVED] {clean_name}",
                        )
                        # Establish the link for this old risk scenario
                        risk_scenario.operational_scenario = operational_scenario
                    except RiskScenario.DoesNotExist:
                        pass

            if risk_scenario:
                # Update existing risk scenario - remove [ARCHIVED] prefix if present
                risk_scenario.name = operational_scenario.name.replace(
                    "[ARCHIVED] ", ""
                )
                risk_scenario.description = build_description(operational_scenario)

                risk_scenario.risk_origin = operational_scenario.ro_to.risk_origin

                # Update inherent or current probability/impact based on feature flag
                if ff_is_enabled("inherent_risk"):
                    risk_scenario.inherent_proba = operational_scenario.likelihood
                    risk_scenario.inherent_impact = operational_scenario.gravity
                else:
                    risk_scenario.current_proba = operational_scenario.likelihood
                    risk_scenario.current_impact = operational_scenario.gravity

                risk_scenario.save()

                # Update relationships
                risk_scenario.assets.set(operational_scenario.get_assets())
                risk_scenario.threats.set(operational_scenario.threats.all())

                # Merge existing controls: keep user-added ones + update from EBIOS RM
                # Get current existing controls
                current_existing_controls = set(
                    risk_scenario.existing_applied_controls.all()
                )
                # Get controls from EBIOS RM
                ebios_controls = set(operational_scenario.get_applied_controls())
                # Merge them (union)
                merged_controls = current_existing_controls | ebios_controls
                risk_scenario.existing_applied_controls.set(merged_controls)

                updated_count += 1

            else:
                # Create new risk scenario (no existing match found)
                risk_scenario = RiskScenario(
                    risk_assessment=risk_assessment,
                    operational_scenario=operational_scenario,
                    name=operational_scenario.name,
                    ref_id=operational_scenario.ref_id
                    if operational_scenario.ref_id
                    else RiskScenario.get_default_ref_id(risk_assessment),
                    description=build_description(operational_scenario),
                    risk_origin=operational_scenario.ro_to.risk_origin,
                )
                if ff_is_enabled("inherent_risk"):
                    risk_scenario.inherent_proba = operational_scenario.likelihood
                    risk_scenario.inherent_impact = operational_scenario.gravity
                else:
                    risk_scenario.current_proba = operational_scenario.likelihood
                    risk_scenario.current_impact = operational_scenario.gravity
                risk_scenario.save()

                risk_scenario.assets.set(operational_scenario.get_assets())
                risk_scenario.threats.set(operational_scenario.threats.all())
                risk_scenario.existing_applied_controls.set(
                    operational_scenario.get_applied_controls()
                )

                created_count += 1

        # Archive risk scenarios that are no longer selected
        for risk_scenario in risk_assessment.risk_scenarios.filter(
            operational_scenario__isnull=False
        ):
            if risk_scenario.operational_scenario_id not in processed_os_ids:
                if not risk_scenario.name.startswith("[ARCHIVED] "):
                    risk_scenario.name = f"[ARCHIVED] {risk_scenario.name}"
                    risk_scenario.save()
                    archived_count += 1

        return Response(
            {
                "success": True,
                "updated": updated_count,
                "created": created_count,
                "archived": archived_count,
                "total_scenarios": risk_assessment.risk_scenarios.count(),
            }
        )

    @action(detail=False, name="Risk assessments per status")
    def per_status(self, request):
        data = assessment_per_status(request.user, RiskAssessment)
        return Response({"results": data})

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get status choices")
    def status(self, request):
        return Response(dict(RiskAssessment.Status.choices))

    @action(detail=False, name="Get quality check")
    def quality_check(self, request):
        """
        Returns the quality check of the risk assessments
        """
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            folder=Folder.get_root_folder(),
            user=request.user,
            object_type=RiskAssessment,
        )
        risk_assessments = RiskAssessment.objects.filter(id__in=viewable_objects)
        res = [
            {"id": a.id, "name": a.name, "quality_check": a.quality_check()}
            for a in risk_assessments
        ]
        return Response({"results": res})

    @action(detail=True, methods=["get"], url_path="quality_check")
    def quality_check_detail(self, request, pk):
        """
        Returns the quality check of the risk_assessment
        """
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            folder=Folder.get_root_folder(),
            user=request.user,
            object_type=RiskAssessment,
        )
        if UUID(pk) in viewable_objects:
            risk_assessment = self.get_object()
            return Response(risk_assessment.quality_check())
        else:
            return Response(status=status.HTTP_403_FORBIDDEN)

    @action(detail=True, name="Get action plan Excel")
    def action_plan_excel(self, request, pk):
        (object_ids_view, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, RiskAssessment
        )
        if UUID(pk) not in object_ids_view:
            return Response(
                {"error": "Permission denied"}, status=status.HTTP_403_FORBIDDEN
            )

        risk_assessment = RiskAssessment.objects.get(id=pk)
        risk_scenarios = risk_assessment.risk_scenarios.all()
        queryset = AppliedControl.objects.filter(
            risk_scenarios__in=risk_scenarios
        ).distinct()

        serializer = RiskAssessmentActionPlanSerializer(
            queryset, many=True, context={"pk": pk}
        )

        wb = Workbook()
        ws = wb.active
        ws.title = "Action Plan"

        headers = [
            "Name",
            "Description",
            "Domain",
            "Category",
            "CSF Function",
            "Priority",
            "Status",
            "ETA",
            "Expiry date",
            "Effort",
            "Impact",
            "Cost",
            "Assigned to",
            "Covered scenarios",
        ]
        ws.append(headers)

        for item in serializer.data:
            row = [
                item.get("name"),
                item.get("description"),
                item.get("folder").get("str"),
                item.get("category"),
                item.get("csf_function"),
                item.get("priority"),
                item.get("status"),
                item.get("eta"),
                item.get("expiry_date"),
                item.get("effort"),
                item.get("impact"),
                item.get("annual_cost"),
                "\n".join([ra.get("str") for ra in item.get("owner")]),
                "\n".join([ra.get("str") for ra in item.get("risk_scenarios")]),
            ]
            ws.append(row)

        for row_idx, row in enumerate(ws.iter_rows(min_row=2), 2):  # Skip header row
            max_lines = 1
            for cell in row:
                if cell.value and isinstance(cell.value, str):
                    line_count = cell.value.count("\n") + 1
                    max_lines = max(max_lines, line_count)
            ws.row_dimensions[row_idx].height = max(15, max_lines * 15)

        for col in ws.columns:
            max_length = 0
            col_letter = get_column_letter(col[0].column)
            for cell in col:
                try:
                    if cell.value:
                        max_length = max(max_length, len(str(cell.value)))
                except Exception as e:
                    logger.error(f"Error processing cell value: {e}")
                    pass
            ws.column_dimensions[col_letter].width = max_length + 2

        last_col_letter = get_column_letter(len(headers))
        for cell in ws[last_col_letter]:
            cell.alignment = Alignment(wrap_text=True)

        response = HttpResponse(
            content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
        )
        response["Content-Disposition"] = (
            f'attachment; filename="action_plan_{pk}.xlsx"'
        )
        wb.save(response)

        return response

    @action(detail=True, name="Get risk assessment CSV")
    def risk_assessment_csv(self, request, pk):
        (object_ids_view, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, RiskAssessment
        )
        if UUID(pk) in object_ids_view:
            risk_assessment = self.get_object()

            response = HttpResponse(content_type="text/csv")

            writer = csv.writer(response, delimiter=";")
            columns = [
                "ref_id",
                "assets",
                "threats",
                "name",
                "description",
                "existing_applied_controls",
                "current_impact",
                "current_proba",
                "current_risk",
                "additional_controls",
                "residual_impact",
                "residual_proba",
                "residual_risk",
                "treatment",
                "strength_of_knowledge",
            ]
            if ff_is_enabled("inherent_risk"):
                # insert inherent_risk just before existing_applied_controls
                columns.insert(
                    columns.index("existing_applied_controls"), "inherent_impact"
                )
                columns.insert(
                    columns.index("existing_applied_controls"), "inherent_proba"
                )
                columns.insert(
                    columns.index("existing_applied_controls"), "inherent_level"
                )
            writer.writerow(columns)

            for scenario in risk_assessment.risk_scenarios.all().order_by("ref_id"):
                additional_controls = ",".join(
                    [m.name for m in scenario.applied_controls.all()]
                )
                existing_controls = ",".join(
                    [m.name for m in scenario.existing_applied_controls.all()]
                )

                threats = ",".join([t.name for t in scenario.threats.all()])
                assets = ",".join([t.name for t in scenario.assets.all()])

                row = [
                    scenario.ref_id,
                    assets,
                    threats,
                    scenario.name,
                    scenario.description,
                    existing_controls,
                    scenario.get_current_impact()["name"],
                    scenario.get_current_proba()["name"],
                    scenario.get_current_risk()["name"],
                    additional_controls,
                    scenario.get_residual_impact()["name"],
                    scenario.get_residual_proba()["name"],
                    scenario.get_residual_risk()["name"],
                    scenario.treatment,
                    RiskScenario.DEFAULT_SOK_OPTIONS[scenario.strength_of_knowledge][
                        "name"
                    ],
                ]
                if ff_is_enabled("inherent_risk"):
                    row.insert(
                        columns.index("inherent_impact"),
                        scenario.get_inherent_impact()["name"],
                    )
                    row.insert(
                        columns.index("inherent_proba"),
                        scenario.get_inherent_proba()["name"],
                    )
                    row.insert(
                        columns.index("inherent_level"),
                        scenario.get_inherent_risk()["name"],
                    )
                writer.writerow(row)

            return response
        else:
            return Response(
                {"error": "Permission denied"}, status=status.HTTP_403_FORBIDDEN
            )

    @action(detail=True, name="Get risk assessment XLSX")
    def risk_assessment_xlsx(self, request, pk):
        (object_ids_view, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, RiskAssessment
        )
        if UUID(pk) not in object_ids_view:
            return Response(
                {"error": "Permission denied"}, status=status.HTTP_403_FORBIDDEN
            )

        risk_assessment = self.get_object()
        entries = []

        columns = [
            "ref_id",
            "assets",
            "threats",
            "name",
            "description",
            "existing_applied_controls",
            "current_impact",
            "current_proba",
            "current_risk",
            "additional_controls",
            "residual_impact",
            "residual_proba",
            "residual_risk",
            "treatment",
            "strength_of_knowledge",
        ]

        if ff_is_enabled("inherent_risk"):
            # insert inherent_risk columns just before existing_applied_controls
            columns.insert(
                columns.index("existing_applied_controls"), "inherent_impact"
            )
            columns.insert(columns.index("existing_applied_controls"), "inherent_proba")
            columns.insert(columns.index("existing_applied_controls"), "inherent_level")

        scenarios = risk_assessment.risk_scenarios.prefetch_related(
            "applied_controls",
            "existing_applied_controls",
            "threats",
            "assets",
        ).order_by("ref_id")

        for scenario in scenarios:
            additional_controls = ", ".join(
                escape_excel_formula(m.name) for m in scenario.applied_controls.all()
            )
            existing_controls = ", ".join(
                escape_excel_formula(m.name)
                for m in scenario.existing_applied_controls.all()
            )
            threats = ", ".join(
                escape_excel_formula(t.name) for t in scenario.threats.all()
            )
            assets = ", ".join(
                escape_excel_formula(t.name) for t in scenario.assets.all()
            )

            entry = {
                "ref_id": escape_excel_formula(scenario.ref_id),
                "assets": assets,
                "threats": threats,
                "name": escape_excel_formula(scenario.name),
                "description": escape_excel_formula(scenario.description),
                "existing_applied_controls": existing_controls,
                "current_impact": scenario.get_current_impact()["name"],
                "current_proba": scenario.get_current_proba()["name"],
                "current_risk": scenario.get_current_risk()["name"],
                "additional_controls": additional_controls,
                "residual_impact": scenario.get_residual_impact()["name"],
                "residual_proba": scenario.get_residual_proba()["name"],
                "residual_risk": scenario.get_residual_risk()["name"],
                "treatment": scenario.treatment,
                "strength_of_knowledge": RiskScenario.DEFAULT_SOK_OPTIONS[
                    scenario.strength_of_knowledge
                ]["name"],
            }

            if ff_is_enabled("inherent_risk"):
                entry["inherent_impact"] = scenario.get_inherent_impact()["name"]
                entry["inherent_proba"] = scenario.get_inherent_proba()["name"]
                entry["inherent_level"] = scenario.get_inherent_risk()["name"]

            entries.append(entry)

        df = pd.DataFrame(entries, columns=columns)

        buffer = io.BytesIO()
        with pd.ExcelWriter(buffer, engine="openpyxl") as writer:
            df.to_excel(writer, index=False, sheet_name="Risk Assessment")
            worksheet = writer.sheets["Risk Assessment"]

            from openpyxl.styles import Alignment

            # Apply text wrapping to description and control columns
            wrap_columns = [
                "name",
                "description",
                "existing_applied_controls",
                "additional_controls",
            ]
            wrap_indices = [
                df.columns.get_loc(col) + 1 for col in wrap_columns if col in df.columns
            ]

            for col_idx in wrap_indices:
                for row_idx in range(2, len(df) + 2):
                    cell = worksheet.cell(row=row_idx, column=col_idx)
                    cell.alignment = Alignment(wrap_text=True)

            # Adjust column widths
            for idx, col in enumerate(df.columns, 1):
                column_letter = worksheet.cell(row=1, column=idx).column_letter
                if col in wrap_columns:
                    worksheet.column_dimensions[column_letter].width = 40
                else:
                    worksheet.column_dimensions[column_letter].width = 20

        buffer.seek(0)
        response = HttpResponse(
            buffer.read(),
            content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
        )
        response["Content-Disposition"] = (
            f'attachment; filename="risk_assessment_{pk}.xlsx"'
        )

        return response

    @action(detail=True, name="Get risk assessment PDF")
    def risk_assessment_pdf(self, request, pk):
        (object_ids_view, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, RiskAssessment
        )
        if UUID(pk) in object_ids_view:
            risk_assessment = self.get_object()
            context = RiskScenario.objects.filter(
                risk_assessment=risk_assessment
            ).order_by("ref_id")
            for scenario in context:
                scenario.strength_of_knowledge = RiskScenario.DEFAULT_SOK_OPTIONS[
                    scenario.strength_of_knowledge
                ]["name"]
            general_settings = GlobalSettings.objects.filter(name="general").first()
            swap_axes = general_settings.value.get("risk_matrix_swap_axes", False)
            flip_vertical = general_settings.value.get(
                "risk_matrix_flip_vertical", False
            )
            matrix_settings = {
                "swap_axes": "_swapaxes" if swap_axes else "",
                "flip_vertical": "_vflip" if flip_vertical else "",
            }
            ff_settings = GlobalSettings.objects.filter(
                name=GlobalSettings.Names.FEATURE_FLAGS
            ).first()
            if ff_settings is None:
                feature_flags = {}
            else:
                feature_flags = ff_settings.value
            data = {
                "context": context,
                "risk_assessment": risk_assessment,
                "ri_clusters": build_scenario_clusters(
                    risk_assessment,
                    include_inherent=feature_flags.get("inherent_risk", False),
                ),
                "risk_matrix": risk_assessment.risk_matrix,
                "settings": matrix_settings,
                "feature_flags": feature_flags,
            }
            html = render_to_string("core/ra_pdf.html", data)
            pdf_file = HTML(string=html).write_pdf()
            response = HttpResponse(pdf_file, content_type="application/pdf")
            return response
        else:
            return Response({"error": "Permission denied"})

    @action(detail=True, name="Get action plan PDF")
    def action_plan_pdf(self, request, pk):
        (object_ids_view, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, RiskAssessment
        )
        if UUID(pk) in object_ids_view:
            context = {
                "to_do": list(),
                "in_progress": list(),
                "on_hold": list(),
                "active": list(),
                "deprecated": list(),
                "--": list(),
            }
            color_map = {
                "to_do": "#FFF8F0",
                "in_progress": "#392F5A",
                "on_hold": "#F4D06F",
                "active": "#9DD9D2",
                "deprecated": "#ff8811",
                "--": "#e5e7eb",
            }
            status = AppliedControl.Status.choices
            risk_assessment_object: RiskAssessment = self.get_object()
            risk_scenarios_objects = risk_assessment_object.risk_scenarios.all()
            applied_controls = (
                AppliedControl.objects.filter(risk_scenarios__in=risk_scenarios_objects)
                .distinct()
                .order_by("eta")
            )
            for applied_control in applied_controls:
                context[applied_control.status].append(
                    applied_control
                ) if applied_control.status else context["--"].append(applied_control)
            data = {
                "status_text": status,
                "color_map": color_map,
                "context": context,
                "risk_assessment": risk_assessment_object,
            }
            html = render_to_string("core/risk_action_plan_pdf.html", data)
            pdf_file = HTML(string=html).write_pdf()
            response = HttpResponse(pdf_file, content_type="application/pdf")
            return response
        else:
            return Response({"error": "Permission denied"})

    @action(
        detail=True,
        name="Duplicate risk assessment",
        methods=["post"],
        serializer_class=RiskAssessmentDuplicateSerializer,
    )
    def duplicate(self, request, pk):
        serializer = RiskAssessmentDuplicateSerializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        data = serializer.validated_data

        (object_ids_view, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, RiskAssessment
        )

        if UUID(pk) in object_ids_view:
            risk_assessment = self.get_object()

            perimeter = data.get("perimeter")
            folder = data.get("folder")

            duplicate_risk_assessment = RiskAssessment.objects.create(
                name=data.get("name"),
                description=data.get("description"),
                perimeter=perimeter,
                folder=folder,
                version=data.get("version"),
                risk_matrix=risk_assessment.risk_matrix,
                ref_id=data.get("ref_id"),
                eta=risk_assessment.eta,
                due_date=risk_assessment.due_date,
                status=risk_assessment.status,
            )

            duplicate_risk_assessment.authors.set(risk_assessment.authors.all())
            duplicate_risk_assessment.reviewers.set(risk_assessment.reviewers.all())

            for scenario in risk_assessment.risk_scenarios.all():
                duplicate_scenario = RiskScenario.objects.create(
                    risk_assessment=duplicate_risk_assessment,
                    name=scenario.name,
                    description=scenario.description,
                    treatment=scenario.treatment,
                    current_proba=scenario.current_proba,
                    current_impact=scenario.current_impact,
                    residual_proba=scenario.residual_proba,
                    residual_impact=scenario.residual_impact,
                    strength_of_knowledge=scenario.strength_of_knowledge,
                    justification=scenario.justification,
                    ref_id=scenario.ref_id,
                )

                duplicate_scenario.qualifications.set(scenario.qualifications.all())

                for field in [
                    "applied_controls",
                    "threats",
                    "assets",
                    "existing_applied_controls",
                ]:
                    duplicate_related_objects(
                        scenario,
                        duplicate_scenario,
                        duplicate_risk_assessment.folder,
                        field,
                    )

                if duplicate_risk_assessment.folder in [risk_assessment.folder] + [
                    folder for folder in risk_assessment.folder.get_sub_folders()
                ]:
                    duplicate_scenario.owner.set(scenario.owner.all())

                duplicate_scenario.save()

            duplicate_risk_assessment.save()
            return Response(
                {
                    "results": RiskAssessmentReadSerializer(
                        duplicate_risk_assessment
                    ).data
                }
            )

    @action(
        detail=True,
        methods=["post"],
        url_path="sync-to-actions",
    )
    def sync_to_applied_controls(self, request, pk):
        dry_run = request.query_params.get("dry_run", True)
        if dry_run == "false":
            dry_run = False
        reset_residual = request.data.get("reset_residual", False)
        risk_assessment = RiskAssessment.objects.get(id=pk)

        if not RoleAssignment.is_access_allowed(
            user=request.user,
            perm=Permission.objects.get(codename="change_riskassessment"),
            folder=Folder.get_folder(risk_assessment),
        ):
            return Response(status=status.HTTP_403_FORBIDDEN)

        changes = risk_assessment.sync_to_applied_controls(
            reset_residual=reset_residual, dry_run=dry_run
        )
        return Response(
            {"changes": RiskScenarioReadSerializer(changes, many=True).data}
        )

    @action(
        detail=True,
        methods=["post"],
        url_path="convert_to_quantitative",
    )
    def convert_to_quantitative(self, request, pk):
        """
        Convert a qualitative risk assessment to a quantitative risk study.

        Expected payload:
        {
            "probability_anchors": [{"index": 0, "value": 0.05}, ...],
            "impact_anchors": [{"index": 0, "central_value": 25000}, ...],
            "loss_threshold": 100000
        }
        """
        from crq.models import (
            QuantitativeRiskStudy,
            QuantitativeRiskScenario,
            QuantitativeRiskHypothesis,
        )

        risk_assessment: RiskAssessment = self.get_object()

        # Check permissions
        if not RoleAssignment.is_access_allowed(
            user=request.user,
            perm=Permission.objects.get(codename="add_quantitativeriskstudy"),
            folder=risk_assessment.folder,
        ):
            return Response(
                {
                    "detail": "You do not have permission to create quantitative risk studies in this domain."
                },
                status=status.HTTP_403_FORBIDDEN,
            )

        # Validate payload
        probability_anchors = request.data.get("probability_anchors", [])
        impact_anchors = request.data.get("impact_anchors", [])
        loss_threshold = request.data.get("loss_threshold")

        if not probability_anchors or not impact_anchors or loss_threshold is None:
            return Response(
                {
                    "detail": "Missing required fields: probability_anchors, impact_anchors, and loss_threshold"
                },
                status=status.HTTP_400_BAD_REQUEST,
            )

        # Validate loss_threshold is a positive number
        try:
            loss_threshold = float(loss_threshold)
            if loss_threshold <= 0:
                return Response(
                    {"detail": "loss_threshold must be greater than 0"},
                    status=status.HTTP_400_BAD_REQUEST,
                )
        except (TypeError, ValueError):
            return Response(
                {"detail": "loss_threshold must be a valid number"},
                status=status.HTTP_400_BAD_REQUEST,
            )

        # Validate and build probability mapping
        probability_map = {}
        try:
            for item in probability_anchors:
                idx = item["index"]
                value = float(item["value"])
                if not (0 <= value <= 1):
                    return Response(
                        {
                            "detail": f"Probability value must be between 0 and 1, got {value} for index {idx}"
                        },
                        status=status.HTTP_400_BAD_REQUEST,
                    )
                probability_map[idx] = value
        except (KeyError, TypeError, ValueError) as e:
            logger.warning(f"Invalid probability anchor format: {e}")
            return Response(
                {
                    "detail": "Invalid probability anchor format. Each anchor must have 'index' and 'value' fields, with value between 0 and 1."
                },
                status=status.HTTP_400_BAD_REQUEST,
            )

        # Validate and build impact mapping
        # Using a narrow fixed ratio: roughly ±30% around central value
        # LB = central_value * 0.7, UB = central_value * 1.43
        # This gives a spread factor of ~2x which minimizes overlap between adjacent levels
        impact_map = {}
        try:
            for item in impact_anchors:
                idx = item["index"]
                central_value = float(item["central_value"])
                if central_value <= 0:
                    return Response(
                        {
                            "detail": f"Impact central value must be positive, got {central_value} for index {idx}"
                        },
                        status=status.HTTP_400_BAD_REQUEST,
                    )
                impact_map[idx] = {
                    "lb": central_value * 0.7,
                    "ub": central_value * 1.43,
                }
        except (KeyError, TypeError, ValueError) as e:
            logger.warning(f"Invalid impact anchor format: {e}")
            return Response(
                {
                    "detail": "Invalid impact anchor format. Each anchor must have 'index' and 'central_value' fields, with central_value greater than 0."
                },
                status=status.HTTP_400_BAD_REQUEST,
            )

        # Generate ref_id for the quantitative study
        # If source has ref_id, append _QUANT, otherwise leave blank
        quant_ref_id = (
            f"{risk_assessment.ref_id}_QUANT" if risk_assessment.ref_id else ""
        )

        # Create the quantitative risk study with [QUANT] prefix
        quantitative_study = QuantitativeRiskStudy.objects.create(
            name=f"[QUANT] {risk_assessment.name}",
            description=f"Converted from qualitative risk assessment: {risk_assessment.name}\n\n{risk_assessment.description or ''}",
            folder=risk_assessment.folder,
            ref_id=quant_ref_id,
            status=risk_assessment.status if risk_assessment.status else "planned",
            loss_threshold=loss_threshold,
            distribution_model="lognormal_ci90",
        )

        # Copy authors and reviewers
        quantitative_study.authors.set(risk_assessment.authors.all())
        quantitative_study.reviewers.set(risk_assessment.reviewers.all())

        scenarios_converted = 0
        scenarios_skipped = 0

        # Convert each risk scenario to a quantitative risk scenario
        for scenario in risk_assessment.risk_scenarios.all():
            # Skip scenarios without threats or assets
            if not scenario.threats.exists() and not scenario.assets.exists():
                logger.info(
                    f"Skipping scenario '{scenario.name}' - no threats or assets associated"
                )
                scenarios_skipped += 1
                continue

            # Check if we have current probability and impact
            current_proba = scenario.current_proba
            current_impact = scenario.current_impact
            has_current = (
                current_proba is not None
                and current_impact is not None
                and current_proba in probability_map
                and current_impact in impact_map
            )

            # Check if we have residual probability and impact
            residual_proba = scenario.residual_proba
            residual_impact = scenario.residual_impact
            has_residual = (
                residual_proba is not None
                and residual_impact is not None
                and residual_proba in probability_map
                and residual_impact in impact_map
            )

            # Skip if we don't have at least current values
            if not has_current and not has_residual:
                logger.info(
                    f"Skipping scenario '{scenario.name}' - no valid probability/impact values"
                )
                scenarios_skipped += 1
                continue

            # Create quantitative risk scenario with same name and ref_id
            quant_scenario = QuantitativeRiskScenario.objects.create(
                quantitative_risk_study=quantitative_study,
                name=scenario.name,
                description=scenario.description or "",
                ref_id=scenario.ref_id or "",
                folder=risk_assessment.folder,
            )

            # Copy threats, assets, and owners
            quant_scenario.threats.set(scenario.threats.all())
            quant_scenario.assets.set(scenario.assets.all())
            quant_scenario.vulnerabilities.set(scenario.vulnerabilities.all())
            quant_scenario.owner.set(scenario.owner.all())

            # Delete the auto-created "baseline" current hypothesis
            # (we'll create our own if needed)
            QuantitativeRiskHypothesis.objects.filter(
                quantitative_risk_scenario=quant_scenario, risk_stage="current"
            ).delete()

            # Create current hypothesis if current values are set
            if has_current:
                probability_value = probability_map[current_proba]
                impact_value = impact_map[current_impact]

                current_hypothesis = QuantitativeRiskHypothesis.objects.create(
                    quantitative_risk_scenario=quant_scenario,
                    name="current",
                    risk_stage="current",
                    ref_id=scenario.ref_id or "",
                    folder=risk_assessment.folder,
                    parameters={
                        "probability": probability_value,
                        "impact": {
                            "distribution": "LOGNORMAL-CI90",
                            "lb": impact_value["lb"],
                            "ub": impact_value["ub"],
                        },
                    },
                )

                # Link to existing applied controls
                if scenario.existing_applied_controls.exists():
                    current_hypothesis.existing_applied_controls.set(
                        scenario.existing_applied_controls.all()
                    )
                current_hypothesis.save()

                # Run simulation for the current hypothesis
                try:
                    current_hypothesis.run_simulation()
                    logger.info(
                        f"Simulation completed for current hypothesis: {scenario.name}"
                    )
                except Exception as e:
                    logger.error(
                        f"Failed to run simulation for scenario {scenario.name}: {str(e)}"
                    )

            # Create residual hypothesis if residual values are set
            if has_residual:
                residual_probability_value = probability_map[residual_proba]
                residual_impact_value = impact_map[residual_impact]

                residual_hypothesis = QuantitativeRiskHypothesis.objects.create(
                    quantitative_risk_scenario=quant_scenario,
                    name="residual",
                    risk_stage="residual",
                    is_selected=True,
                    ref_id=f"{scenario.ref_id}_RES" if scenario.ref_id else "",
                    folder=risk_assessment.folder,
                    parameters={
                        "probability": residual_probability_value,
                        "impact": {
                            "distribution": "LOGNORMAL-CI90",
                            "lb": residual_impact_value["lb"],
                            "ub": residual_impact_value["ub"],
                        },
                    },
                )

                # Link to existing controls and added controls
                if scenario.existing_applied_controls.exists():
                    residual_hypothesis.existing_applied_controls.set(
                        scenario.existing_applied_controls.all()
                    )
                if scenario.applied_controls.exists():
                    residual_hypothesis.added_applied_controls.set(
                        scenario.applied_controls.all()
                    )
                residual_hypothesis.save()

                # Run simulation for the residual hypothesis
                try:
                    residual_hypothesis.run_simulation()
                    logger.info(
                        f"Simulation completed for residual hypothesis: {scenario.name}"
                    )
                except Exception as e:
                    logger.error(
                        f"Failed to run simulation for residual scenario {scenario.name}: {str(e)}"
                    )

            scenarios_converted += 1

        logger.info(
            f"Conversion completed: {scenarios_converted} scenarios converted, {scenarios_skipped} skipped"
        )

        return Response(
            {
                "message": "Conversion successful",
                "quantitative_risk_study_id": str(quantitative_study.id),
                "scenarios_converted": scenarios_converted,
                "scenarios_skipped": scenarios_skipped,
            },
            status=status.HTTP_201_CREATED,
        )


def convert_date_to_timestamp(date):
    """
    Converts a date object (datetime.date) to a Linux timestamp.
    It creates a datetime object for the date at midnight and makes it timezone-aware.
    """
    if date:
        date_as_datetime = datetime.combine(date, datetime.min.time())
        aware_datetime = pytz.UTC.localize(date_as_datetime)
        return int(time.mktime(aware_datetime.timetuple())) * 1000
    return None


class AppliedControlFilterSet(GenericFilterSet):
    folder = df.ModelMultipleChoiceFilter(queryset=Folder.objects.all())
    reference_control = df.ModelMultipleChoiceFilter(
        queryset=ReferenceControl.objects.all()
    )

    todo = df.BooleanFilter(method="filter_todo")
    to_review = df.BooleanFilter(method="filter_to_review")
    compliance_assessments = df.ModelMultipleChoiceFilter(
        method="filter_compliance_assessments",
        queryset=ComplianceAssessment.objects.all(),
    )
    risk_assessments = df.ModelMultipleChoiceFilter(
        method="filter_risk_assessments",
        queryset=RiskAssessment.objects.all(),
    )
    findings_assessments = df.ModelMultipleChoiceFilter(
        method="filter_findings_assessments",
        queryset=FindingsAssessment.objects.all(),
    )
    status = df.MultipleChoiceFilter(
        choices=AppliedControl.Status.choices, lookup_expr="icontains"
    )
    is_assigned = df.BooleanFilter(method="filter_is_assigned")
    # Nullable choice filters that support filtering for unset values using "--"
    category = NullableChoiceFilter(choices=AppliedControl.CATEGORY)
    csf_function = NullableChoiceFilter(choices=AppliedControl.CSF_FUNCTION)
    priority = NullableChoiceFilter(choices=AppliedControl.PRIORITY)
    effort = NullableChoiceFilter(choices=AppliedControl.EFFORT)
    control_impact = NullableChoiceFilter(choices=AppliedControl.IMPACT)

    def filter_findings_assessments(self, queryset, name, value):
        if value:
            findings_assessments = FindingsAssessment.objects.filter(
                id__in=[x.id for x in value]
            )
            if len(findings_assessments) == 0:
                return queryset
            findings = chain.from_iterable(
                [fa.findings.all() for fa in findings_assessments]
            )
            return queryset.filter(findings__in=findings).distinct()
        return queryset

    def filter_risk_assessments(self, queryset, name, value):
        if value:
            risk_assessments = RiskAssessment.objects.filter(
                id__in=[x.id for x in value]
            )
            if len(risk_assessments) == 0:
                return queryset
            risk_scenarios = chain.from_iterable(
                [ra.risk_scenarios.all() for ra in risk_assessments]
            )
            return queryset.filter(risk_scenarios__in=risk_scenarios).distinct()
        return queryset

    def filter_compliance_assessments(self, queryset, name, value):
        if not value:
            return queryset

        # Fetch compliance assessments with related requirements to prevent N+1 queries.
        # We need the requirement's implementation_groups for the filtering logic.
        compliance_assessments = ComplianceAssessment.objects.filter(
            id__in=[x.id for x in value]
        ).prefetch_related("requirement_assessments__requirement")

        if not compliance_assessments.exists():
            return queryset

        valid_ra_ids = set()

        for ca in compliance_assessments:
            selected_groups = ca.selected_implementation_groups
            current_ras = ca.requirement_assessments.all()

            # If no groups are selected in the audit, current behavior applies (show all)
            if not selected_groups:
                valid_ra_ids.update(ra.id for ra in current_ras)
                continue

            # Convert selected groups to a set for O(1) lookup
            selected_set = set(selected_groups)

            for ra in current_ras:
                req_groups = ra.requirement.implementation_groups or []

                # Check for intersection: Keep if ANY requirement group matches ANY selected group
                if not selected_set.isdisjoint(req_groups):
                    valid_ra_ids.add(ra.id)

        return queryset.filter(requirement_assessments__in=valid_ra_ids).distinct()

    def filter_todo(self, queryset, name, value):
        if value:
            return (
                queryset.filter(eta__lte=date.today() + timedelta(days=30))
                .exclude(status="active")
                .order_by("eta")
            )

        return queryset

    def filter_to_review(self, queryset, name, value):
        if value:
            return queryset.filter(
                expiry_date__lte=date.today() + timedelta(days=30)
            ).order_by("expiry_date")
        return queryset

    def filter_is_assigned(self, queryset, name, value):
        if value:
            return queryset.filter(owner__isnull=False).distinct()
        else:
            return queryset.filter(owner__isnull=True)

    class Meta:
        model = AppliedControl
        fields = {
            "name": ["exact"],
            "category": ["exact"],
            "csf_function": ["exact"],
            "priority": ["exact"],
            "effort": ["exact"],
            "control_impact": ["exact"],
            # category, csf_function, priority, effort, control_impact use NullableChoiceFilter (defined above)
            "filtering_labels": ["exact"],
            "risk_scenarios": ["exact"],
            "risk_scenarios_e": ["exact"],
            "requirement_assessments": ["exact"],
            "evidences": ["exact"],
            "objectives": ["exact"],
            "assets": ["exact"],
            "stakeholders": ["exact"],
            "progress_field": ["exact"],
            "security_exceptions": ["exact"],
            "owner": ["exact"],
            "findings": ["exact"],
            "eta": ["exact", "lte", "gte", "lt", "gt", "month", "year"],
            "ref_id": ["exact"],
            "processings": ["exact"],
            "genericcollection": ["exact"],
        }


class AppliedControlViewSet(ExportMixin, BaseModelViewSet):
    """
    API endpoint that allows applied controls to be viewed or edited.
    """

    model = AppliedControl
    filterset_class = AppliedControlFilterSet
    search_fields = ["name", "description", "ref_id"]

    @staticmethod
    def _extract_cost_field(control, *path):
        """Helper to safely extract nested cost fields."""
        value = control.cost
        if not value:
            return ""
        for key in path:
            value = value.get(key, "")
            if not value:
                return ""
        return value

    export_config = {
        "fields": {
            "internal_id": {"source": "id", "label": "internal_id"},
            "ref_id": {"source": "ref_id", "label": "ref_id", "escape": True},
            "name": {"source": "name", "label": "name", "escape": True},
            "description": {
                "source": "description",
                "label": "description",
                "escape": True,
            },
            "reference_control_name": {
                "source": "reference_control.name",
                "label": "reference_control_name",
                "escape": True,
            },
            "reference_control_ref_id": {
                "source": "reference_control.ref_id",
                "label": "reference_control_ref_id",
                "escape": True,
            },
            "category": {"source": "category", "label": "category"},
            "csf_function": {"source": "csf_function", "label": "csf_function"},
            "folder": {"source": "folder.name", "label": "folder"},
            "status": {"source": "status", "label": "status"},
            "start_date": {"source": "start_date", "label": "start_date"},
            "eta": {"source": "eta", "label": "eta"},
            "expiry_date": {"source": "expiry_date", "label": "expiry_date"},
            "priority": {"source": "priority", "label": "priority"},
            "effort": {"source": "get_effort_display", "label": "effort"},
            "impact": {"source": "get_control_impact_display", "label": "impact"},
            "cost_currency": {
                "source": "cost",
                "label": "cost_currency",
                "format": lambda cost: cost.get("currency", "") if cost else "",
            },
            "cost_build_fixed": {
                "source": "cost",
                "label": "cost_build_fixed",
                "format": lambda cost: (
                    cost.get("build", {}).get("fixed_cost", "") if cost else ""
                ),
            },
            "cost_build_people_days": {
                "source": "cost",
                "label": "cost_build_people_days",
                "format": lambda cost: (
                    cost.get("build", {}).get("people_days", "") if cost else ""
                ),
            },
            "cost_run_fixed": {
                "source": "cost",
                "label": "cost_run_fixed",
                "format": lambda cost: (
                    cost.get("run", {}).get("fixed_cost", "") if cost else ""
                ),
            },
            "cost_run_people_days": {
                "source": "cost",
                "label": "cost_run_people_days",
                "format": lambda cost: (
                    cost.get("run", {}).get("people_days", "") if cost else ""
                ),
            },
            "cost_amortization_period": {
                "source": "cost",
                "label": "cost_amortization_period",
                "format": lambda cost: (
                    cost.get("amortization_period", "") if cost else ""
                ),
            },
            "owner": {
                "source": "owner",
                "label": "owner",
                "format": lambda qs: (
                    ",".join(str(o) for o in qs.all()) if qs.exists() else ""
                ),
            },
            "labels": {
                "source": "filtering_labels",
                "label": "labels",
                "format": lambda qs: ",".join(lbl.label for lbl in qs.all()),
            },
        },
        "filename": "audit_export",
        "select_related": ["reference_control", "folder"],
        "prefetch_related": ["owner", "filtering_labels"],
    }

    def get_queryset(self):
        """Optimize queries by prefetching related objects used in the table view and serializer"""
        qs = (
            super()
            .get_queryset()
            .select_related(
                "folder",
                "folder__parent_folder",  # For get_folder_full_path() optimization
            )
        )
        if self.action == "autocomplete":
            return qs
        return qs.select_related("reference_control").prefetch_related(
            "owner",
            "filtering_labels__folder",  # FieldsRelatedField includes folder
            "findings",  # Used for findings_count
            "evidences",  # Serialized as FieldsRelatedField
            "objectives",  # ManyToManyField to OrganisationObjective
            "assets",  # ManyToManyField used in table
            "security_exceptions",  # Serialized as FieldsRelatedField
        )

    @action(detail=False, name="Lightweight autocomplete search")
    def autocomplete(self, request):
        """Minimal endpoint for autocomplete selects."""
        from core.serializers import AppliedControlAutocompleteSerializer

        qs = self.filter_queryset(self.get_queryset())
        page = self.paginate_queryset(qs)
        objects = page if page is not None else qs
        serializer = AppliedControlAutocompleteSerializer(objects, many=True)
        data = serializer.data
        field_models = self._get_fieldsrelated_map(serializer)
        if field_models:
            allowed_ids = self._get_accessible_ids_map(set(field_models.values()))
            data = self._filter_related_fields(data, field_models, allowed_ids)
        if page is not None:
            return self.get_paginated_response(data)
        return Response(data)

    def perform_create(self, serializer):
        create_remote_object = serializer.validated_data.pop(
            "create_remote_object", False
        )
        integration_config = serializer.validated_data.pop("integration_config", None)
        serializer.validated_data.pop("remote_object_id", None)  # Remove if present

        # Create the local object first
        super().perform_create(serializer)
        instance = serializer.instance

        if create_remote_object and integration_config:
            try:
                logger.info(
                    "Creating remote object for Applied Control",
                    applied_control_id=instance.id,
                    integration_config_id=integration_config.id,
                )
                sync_object_to_integrations.schedule(
                    args=(
                        ContentType.objects.get_for_model(self.model),
                        serializer.instance.id,
                        [integration_config.id],
                        ["name", "description", "status", "priority"],
                    ),
                    delay=1,  # Small delay to ensure transaction is committed
                )

            except Exception:
                logger.error(
                    "Error creating remote object for Applied Control",
                    applied_control_id=instance.id,
                    exc_info=True,
                )

    def perform_update(self, serializer):
        integration_config = serializer.validated_data.pop("integration_config", None)
        remote_object_id = serializer.validated_data.pop("remote_object_id", None)
        serializer.validated_data.pop("create_remote_object", None)  # Remove if present

        super().perform_update(serializer)
        if not integration_config or not remote_object_id:
            return
        try:
            if remote_object_id:
                logger.info(
                    "Attaching applied control to external object",
                    applied_control_id=serializer.instance.id,
                    remote_id=remote_object_id,
                )
                sync_mapping = SyncMapping.objects.create(
                    configuration=integration_config,
                    content_type=ContentType.objects.get_for_model(self.model),
                    local_object_id=serializer.instance.id,
                    remote_id=remote_object_id,
                    sync_status=SyncMapping.SyncStatus.PENDING,
                )
                sync_object_to_integrations.schedule(
                    args=(
                        sync_mapping.content_type,
                        serializer.instance.id,
                        [integration_config.id],
                        ["status"],
                    ),
                    delay=1,  # Small delay to ensure transaction is committed
                )

        except Exception:
            logger.error("Error creating SyncMapping", exc_info=True)

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get status choices")
    def status(self, request):
        return Response(dict(AppliedControl.Status.choices))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get category choices")
    def category(self, request):
        return Response(add_unset_option(dict(AppliedControl.CATEGORY)))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get csf_function choices")
    def csf_function(self, request):
        return Response(add_unset_option(dict(AppliedControl.CSF_FUNCTION)))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get priority choices")
    def priority(self, request):
        return Response(add_unset_option(dict(AppliedControl.PRIORITY)))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get effort choices")
    def effort(self, request):
        return Response(add_unset_option(dict(AppliedControl.EFFORT)))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get impact choices")
    def control_impact(self, request):
        return Response(add_unset_option(dict(AppliedControl.IMPACT)))

    @action(detail=False, name="Get all applied controls owners")
    def owner(self, request):
        return Response(
            ActorReadSerializer(
                Actor.objects.filter(applied_controls__isnull=False).distinct(),
                many=True,
            ).data
        )

    @action(detail=False, name="Get updatable measures")
    def updatables(self, request):
        (_, object_ids_change, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, AppliedControl
        )

        return Response({"results": object_ids_change})

    @action(
        detail=False, name="Something"
    )  # Write a good name for the "name" keyword argument
    def per_status(self, request):
        data = applied_control_per_status(request.user)
        return Response({"results": data})

    @action(detail=False, name="Get the ordered todo applied controls")
    def todo(self, request):
        (object_ids_view, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, AppliedControl
        )

        measures = sorted(
            AppliedControl.objects.filter(id__in=object_ids_view)
            .filter(eta__lte=date.today() + timedelta(days=30))
            .exclude(status="active")
            .order_by("eta"),
            key=lambda mtg: mtg.get_ranking_score(),
            reverse=True,
        )

        ranking_scores = {str(mtg.id): mtg.get_ranking_score() for mtg in measures}

        measures = [AppliedControlReadSerializer(mtg).data for mtg in measures]

        # How to add ranking_score directly in the serializer ?

        for i in range(len(measures)):
            measures[i]["ranking_score"] = ranking_scores[measures[i]["id"]]

        """
        The serializer of AppliedControl isn't applied automatically for this function
        """

        return Response({"results": measures})

    @action(detail=False, name="Get the secuity measures to review")
    def to_review(self, request):
        measures = measures_to_review(request.user)

        measures = [AppliedControlReadSerializer(mtg).data for mtg in measures]

        """
        The serializer of AppliedControl isn't applied automatically for this function
        """

        return Response({"results": measures})

    @action(detail=False, methods=["get"])
    def get_controls_info(self, request):
        folder_id = request.query_params.get("folder", None)
        scoped_folder = (
            Folder.objects.get(id=folder_id) if folder_id else Folder.get_root_folder()
        )

        (view_ac_ids, _, _) = RoleAssignment.get_accessible_object_ids(
            scoped_folder, request.user, AppliedControl
        )
        (view_ca_ids, _, _) = RoleAssignment.get_accessible_object_ids(
            scoped_folder, request.user, ComplianceAssessment
        )
        (view_ra_ids, _, _) = RoleAssignment.get_accessible_object_ids(
            scoped_folder, request.user, RiskAssessment
        )

        ac_qs = AppliedControl.objects.filter(id__in=view_ac_ids).only("id", "name")
        ca_qs = ComplianceAssessment.objects.filter(id__in=view_ca_ids).only(
            "id", "name"
        )
        ra_qs = RiskAssessment.objects.filter(id__in=view_ra_ids).only("id", "name")

        ac_ids = list(ac_qs.values_list("id", flat=True))
        ca_ids = set(ca_qs.values_list("id", flat=True))
        ra_ids = set(ra_qs.values_list("id", flat=True))

        audit_rows = (
            RequirementAssessment.objects.filter(
                compliance_assessment_id__in=ca_ids, applied_controls__id__in=ac_ids
            )
            .values("compliance_assessment_id", "applied_controls")
            .annotate(coverage=Count("id"))
        )

        risk_rows = (
            RiskScenario.objects.filter(
                risk_assessment_id__in=ra_ids, applied_controls__id__in=ac_ids
            )
            .values("risk_assessment_id", "applied_controls")
            .annotate(coverage=Count("id"))
        )

        links = []
        used_audit_ids = set()
        used_ra_ids = set()

        ac_counter = {ac_id: 0 for ac_id in ac_ids}

        for row in audit_rows:
            ca_id = row["compliance_assessment_id"]
            ac_id = row["applied_controls"]
            cov = row["coverage"]
            links.append({"source": ca_id, "target": ac_id, "coverage": cov})
            used_audit_ids.add(ca_id)
            ac_counter[ac_id] = ac_counter.get(ac_id, 0) + cov

        for row in risk_rows:
            ra_id = row["risk_assessment_id"]
            ac_id = row["applied_controls"]
            cov = row["coverage"]
            links.append({"source": ra_id, "target": ac_id, "coverage": cov})
            used_ra_ids.add(ra_id)
            ac_counter[ac_id] = ac_counter.get(ac_id, 0) + cov

        nodes = []

        for ac in ac_qs:
            nodes.append(
                {
                    "id": ac.id,
                    "label": ac.name,
                    "shape": "hexagon",
                    "counter": ac_counter.get(ac.id, 0),
                    "color": "#47e845",
                }
            )

        for audit in ca_qs.filter(id__in=used_audit_ids):
            nodes.append(
                {
                    "id": audit.id,
                    "label": audit.name,
                    "shape": "circle",
                    "color": "#5D4595",
                }
            )

        for ra in ra_qs.filter(id__in=used_ra_ids):
            nodes.append(
                {"id": ra.id, "label": ra.name, "shape": "square", "color": "#E6499F"}
            )

        return Response({"nodes": nodes, "links": links})

    @action(detail=False, name="Get priority chart data")
    def priority_chart_data(self, request):
        qs = AppliedControl.objects.exclude(status="active")

        data = {
            "--": [],
            "to_do": [],
            "in_progress": [],
            "on_hold": [],
            "deprecated": [],
        }
        angle_offsets = {"4": 0, "3": 90, "1": 180, "2": 270}
        status_offset = {
            "--": 4,
            "to_do": 12,
            "in_progress": 20,
            "on_hold": 28,
            "deprecated": 36,
        }

        not_displayed_cnt = 0

        p_dict = qs.aggregate(
            p1=Count("priority", filter=Q(priority=1)),
            p2=Count("priority", filter=Q(priority=2)),
            p3=Count("priority", filter=Q(priority=3)),
            p4=Count("priority", filter=Q(priority=4)),
        )
        for ac in qs:
            if ac.priority:
                if ac.eta:
                    days_countdown = min(100, ac.days_until_eta)
                    # how many days until the ETA
                else:
                    days_countdown = 100
                impact_factor = 5 + ac.links_count

                # angle = angle_offsets[str(ac.priority)]+ (next(offsets) % 80) + random.randint(1,4)
                angle = (
                    angle_offsets[str(ac.priority)]
                    + status_offset[ac.status]
                    + random.randint(1, 40)
                )
                # angle = angle_offsets[str(ac.priority)] + next(offsets)

                vector = [
                    days_countdown,
                    angle,
                    impact_factor,
                    f"[{ac.priority}] {str(ac)}",
                    ac.status,
                    ac.id,
                ]
                if ac.status:
                    data[ac.status].append(vector)
                else:
                    data["unclassified"].append(vector)
            else:
                print("priority unset - add it to triage lot")
                not_displayed_cnt += 1

        data["not_displayed"] = not_displayed_cnt
        data["priority_cnt"] = p_dict

        return Response(data)

    @action(detail=False, methods=["get"])
    def get_gantt_data(self, request):
        def format_date(input):
            return datetime.strftime(input, "%Y-%m-%d")

        entries = []
        (viewable_controls_ids, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, AppliedControl
        )

        applied_controls = AppliedControl.objects.filter(
            id__in=viewable_controls_ids
        ).select_related("folder")

        for ac in applied_controls:
            if ac.eta:
                endDate = format_date(ac.eta)
                startDate = format_date(ac.start_date) if ac.start_date else endDate
                if ac.start_date:
                    startDate = format_date(ac.start_date)
                else:
                    startDate = format_date(ac.eta - timedelta(days=1))
                entries.append(
                    {
                        "id": ac.id,
                        "start": startDate,
                        "end": endDate,
                        "name": ac.name,
                        "progress": ac.progress_field,
                        "description": ac.description
                        if ac.description
                        else "(no description)",
                        "domain": ac.folder.name,
                    }
                )
        return Response(entries)

    @action(detail=False, methods=["get"])
    def impact_effort(self, request):
        # TODO consider the case of passing the domain as a filter
        (viewable_controls_ids, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, AppliedControl
        )
        viewable_controls_set = set(viewable_controls_ids)

        output = [
            [[], [], [], [], []],
            [[], [], [], [], []],
            [[], [], [], [], []],
            [[], [], [], [], []],
            [[], [], [], [], []],
        ]

        assessed_controls = AppliedControl.objects.filter(
            id__in=viewable_controls_set,
            control_impact__isnull=False,
            effort__isnull=False,
        ).values_list(
            "id",
            "name",
            "control_impact",
            "effort",
        )

        # MAP_EFFORT for converting effort strings to integers
        MAP_EFFORT = {None: -1, "XS": 1, "S": 2, "M": 3, "L": 4, "XL": 5}

        for control_id, name, impact, effort_str in assessed_controls:
            # Convert effort string to integer
            effort_int = MAP_EFFORT.get(effort_str, -1)

            # Convert to 0-based indices
            impact_idx = 5 - impact
            effort_idx = effort_int - 1

            # Add to matrix if both values are valid (1-5)
            if 0 <= impact_idx < 5 and 0 <= effort_idx < 5:
                output[impact_idx][effort_idx].append(
                    {
                        "id": control_id,
                        "impact": impact,
                        "effort": effort_int,
                        "name": name,
                    }
                )

        # Get viewable but not assessed controls
        all_assessed_ids = set(control_id for control_id, _, _, _ in assessed_controls)
        viewable_not_assessed = viewable_controls_set - all_assessed_ids

        print("Matrix populated with assessed controls")
        print(f"Viewable but not assessed: {len(viewable_not_assessed)} controls")
        return Response(output)

    @action(detail=False, methods=["get"])
    def get_timeline_info(self, request):
        entries = []
        COLORS_PALETTE = [
            "#F72585",
            "#7209B7",
            "#3A0CA3",
            "#4361EE",
            "#4CC9F0",
            "#A698DC",
        ]
        colorMap = {}
        (viewable_controls_ids, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, AppliedControl
        )

        applied_controls = AppliedControl.objects.filter(
            id__in=viewable_controls_ids
        ).select_related("folder")

        for ac in applied_controls:
            if ac.eta:
                endDate = convert_date_to_timestamp(ac.eta)
                startDate = (
                    convert_date_to_timestamp(ac.start_date)
                    if ac.start_date
                    else endDate
                )
                entries.append(
                    {
                        "startDate": startDate,
                        "endDate": endDate,
                        "name": ac.name,
                        "description": ac.description
                        if ac.description
                        else "(no description)",
                        "domain": ac.folder.name,
                    }
                )
        color_cycle = cycle(COLORS_PALETTE)
        for domain in Folder.objects.all():
            colorMap[domain.name] = next(color_cycle)
        return Response({"entries": entries, "colorMap": colorMap})

    @action(
        detail=True,
        name="Duplicate applied control",
        methods=["post"],
        serializer_class=AppliedControlDuplicateSerializer,
    )
    def duplicate(self, request, pk):
        serializer = AppliedControlDuplicateSerializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        data = serializer.validated_data

        (object_ids_view, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, AppliedControl
        )
        if UUID(pk) not in object_ids_view:
            return Response(
                {"results": "applied control not found"},
                status=status.HTTP_404_NOT_FOUND,
            )

        applied_control = self.get_object()
        new_folder = data["folder"]
        duplicate_applied_control = AppliedControl.objects.create(
            reference_control=applied_control.reference_control,
            name=data["name"],
            description=data["description"],
            folder=new_folder,
            ref_id=applied_control.ref_id,
            category=applied_control.category,
            csf_function=applied_control.csf_function,
            priority=applied_control.priority,
            status=applied_control.status,
            start_date=applied_control.start_date,
            eta=applied_control.eta,
            expiry_date=applied_control.expiry_date,
            link=applied_control.link,
            effort=applied_control.effort,
            cost=applied_control.cost,
            progress_field=applied_control.progress_field,
        )
        duplicate_applied_control.owner.set(applied_control.owner.all())
        duplicate_applied_control.filtering_labels.set(
            applied_control.filtering_labels.all()
        )
        if data["duplicate_evidences"]:
            duplicate_related_objects(
                applied_control, duplicate_applied_control, new_folder, "evidences"
            )
            duplicate_applied_control.save()

        return Response(
            {"results": AppliedControlReadSerializer(duplicate_applied_control).data}
        )

    @action(detail=False, methods=["get"])
    def ids(self, request):
        my_map = dict()

        (viewable_items, _, _) = RoleAssignment.get_accessible_object_ids(
            folder=Folder.get_root_folder(),
            user=request.user,
            object_type=AppliedControl,
        )
        for item in AppliedControl.objects.filter(id__in=viewable_items):
            if my_map.get(item.folder.name) is None:
                my_map[item.folder.name] = {}
            my_map[item.folder.name].update({item.name: item.id})

        return Response(my_map)

    @action(detail=False, name="Generate data for applied controls impact graph")
    def impact_graph(self, request):
        (viewable_controls_ids, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, AppliedControl
        )
        csf_functions_map = dict()
        categories = [{"name": "--"}]
        for i, option in enumerate(ReferenceControl.CSF_FUNCTION, 1):
            csf_functions_map[option[0]] = i
            categories.append({"name": option[1]})
        categories.append({"name": "requirements"})  # 7
        categories.append({"name": "scenarios"})  # 9
        categories.append({"name": "audits"})  # 8
        categories.append({"name": "risk assessments"})

        nodes = list()
        links = list()
        indexes = dict()
        idx_cnt = 0
        for ac in AppliedControl.objects.filter(id__in=viewable_controls_ids):
            ac_key = f"ac-{ac.id}"
            nodes.append(
                {
                    "name": ac.name,
                    "value": ac.name,
                    "category": csf_functions_map.get(ac.csf_function, 0),
                }
            )
            indexes[ac_key] = idx_cnt
            idx_cnt += 1
            # attached requirement_assessments
            for req in RequirementAssessment.objects.filter(applied_controls__id=ac.id):
                req_key = f"req-{req.id}"
                if req_key not in indexes:
                    # Add RequirementAssessment node only if it doesn't exist
                    nodes.append(
                        {
                            "name": req.requirement.ref_id
                            or req.requirement.safe_display_str,
                            "value": req.requirement.description
                            or req.requirement.safe_display_str,
                            "category": 7,
                            "symbol": "triangle",
                        }
                    )
                    indexes[req_key] = idx_cnt
                    idx_cnt += 1

                audit = req.compliance_assessment
                audit_key = f"audit-{audit.id}"
                if audit_key not in indexes:
                    nodes.append(
                        {
                            "name": audit.name,
                            "value": audit.framework.name,
                            "category": 9,
                            "symbol": "rect",
                        }
                    )
                    indexes[audit_key] = idx_cnt
                    idx_cnt += 1
                links.append({"source": indexes[audit_key], "target": indexes[req_key]})
                links.append({"source": indexes[ac_key], "target": indexes[req_key]})

            for sc in RiskScenario.objects.filter(applied_controls__id=ac.id):
                sc_key = f"sc-{sc.id}"
                if sc_key not in indexes:
                    nodes.append(
                        {
                            "name": sc.ref_id,
                            "value": sc.name,
                            "category": 8,
                            "symbol": "diamond",
                        }
                    )
                    indexes[sc_key] = idx_cnt
                    idx_cnt += 1

                ra = sc.risk_assessment
                ra_key = f"ra-{ra.id}"
                if ra_key not in indexes:
                    nodes.append(
                        {
                            "name": ra.name,
                            "value": ra.name,
                            "category": 10,
                            "symbol": "rect",
                        }
                    )
                    indexes[ra_key] = idx_cnt
                    idx_cnt += 1
                links.append({"source": indexes[ra_key], "target": indexes[sc_key]})
                links.append({"source": indexes[ac_key], "target": indexes[sc_key]})

        return Response({"nodes": nodes, "categories": categories, "links": links})

    @action(detail=False, name="Get applied controls sunburst data")
    def sunburst_data(self, request):
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, AppliedControl
        )
        queryset = AppliedControl.objects.filter(id__in=viewable_objects)

        # Build hierarchical structure: csf_function -> category -> priority -> status
        hierarchy = {}

        for control in queryset.select_related("folder"):
            # Level 1: CSF Function
            csf_function = (
                dict(AppliedControl.CSF_FUNCTION).get(
                    control.csf_function, "No CSF Function"
                )
                if control.csf_function
                else "No CSF Function"
            )
            if csf_function not in hierarchy:
                hierarchy[csf_function] = {}

            # Level 2: Category
            category = (
                dict(AppliedControl.CATEGORY).get(control.category, "No Category")
                if control.category
                else "No Category"
            )
            if category not in hierarchy[csf_function]:
                hierarchy[csf_function][category] = {}

            # Level 3: Priority
            priority = (
                dict(AppliedControl.PRIORITY).get(control.priority, "No Priority")
                if control.priority
                else "No Priority"
            )
            if priority not in hierarchy[csf_function][category]:
                hierarchy[csf_function][category][priority] = {}

            # Level 4: Status
            status = (
                dict(AppliedControl.Status.choices).get(control.status, "No Status")
                if control.status
                else "No Status"
            )
            if status not in hierarchy[csf_function][category][priority]:
                hierarchy[csf_function][category][priority][status] = 0
            hierarchy[csf_function][category][priority][status] += 1

        # CSF Function color mapping (matching NightingaleChart palette)
        csf_color_map = {
            "(undefined)": "#505372",
            "Govern": "#FAE482",
            "Identify": "#85C4EA",
            "Protect": "#B29BBA",
            "Detect": "#FAB647",
            "Respond": "#E47677",
            "Recover": "#8ACB93",
            "No CSF Function": "#505372",
        }

        # Convert to sunburst format
        def build_sunburst_data(data_dict, name="Root", level=0, parent_color=None):
            if isinstance(data_dict, int):
                result = {"name": name, "value": data_dict}
                if parent_color:
                    result["itemStyle"] = {"color": parent_color}
                return result

            children = []
            total_value = 0

            # Determine color for this level
            current_color = parent_color
            if level == 1 and name in csf_color_map:
                current_color = csf_color_map[name]

            for key, value in data_dict.items():
                child = build_sunburst_data(value, key, level + 1, current_color)
                children.append(child)
                total_value += child.get("value", 0)

            result = {"name": name, "value": total_value}
            if children:
                result["children"] = children

            # Apply color at all levels if we have a color
            if current_color and level >= 1:
                result["itemStyle"] = {"color": current_color}

            return result

        sunburst_data = []
        for csf_function, function_data in hierarchy.items():
            function_node = build_sunburst_data(function_data, csf_function, level=1)
            sunburst_data.append(function_node)

        return Response({"results": sunburst_data})


class ActionPlanList(generics.ListAPIView):
    filterset_fields = {
        "folder": ["exact"],
        "status": ["exact"],
        "category": ["exact"],
        "csf_function": ["exact"],
        "priority": ["exact"],
        "reference_control": ["exact"],
        "effort": ["exact"],
        "control_impact": ["exact"],
        "filtering_labels": ["exact"],
        "risk_scenarios": ["exact"],
        "risk_scenarios_e": ["exact"],
        "requirement_assessments": ["exact"],
        "evidences": ["exact"],
        "objectives": ["exact"],
        "assets": ["exact"],
        "stakeholders": ["exact"],
        "progress_field": ["exact"],
        "security_exceptions": ["exact"],
        "owner": ["exact"],
        "findings": ["exact"],
        "eta": ["exact", "lte", "gte", "lt", "gt"],
    }
    search_fields = ["name", "description", "ref_id"]

    filter_backends = [
        DjangoFilterBackend,
        filters.SearchFilter,
        filters.OrderingFilter,
    ]
    ordering_fields = "__all__"
    ordering = ["eta"]

    def get_serializer_context(self):
        context = super().get_serializer_context()
        context.update({"pk": self.kwargs["pk"]})
        return context


class ActionPlanBudgetOverview:
    """Mixin that computes budget aggregation over an applied controls queryset."""

    @staticmethod
    def _safe_float(value, default=0):
        """Coerce a JSON-sourced value to float, returning default on failure."""
        try:
            return float(value)
        except (TypeError, ValueError):
            return default

    @staticmethod
    def _compute_annual_cost(ctrl, daily_rate):
        """Compute annual cost without per-control GlobalSettings lookup."""
        if not ctrl.cost:
            return 0
        _f = ActionPlanBudgetOverview._safe_float
        build_cost = ctrl.cost.get("build", {})
        run_cost = ctrl.cost.get("run", {})
        amortization_period = _f(ctrl.cost.get("amortization_period", 1), 1) or 1
        annual_cost = 0
        build_fixed = _f(build_cost.get("fixed_cost", 0))
        build_people = _f(build_cost.get("people_days", 0))
        if build_fixed > 0:
            annual_cost += build_fixed / amortization_period
        if build_people > 0:
            annual_cost += (build_people * daily_rate) / amortization_period
        run_fixed = _f(run_cost.get("fixed_cost", 0))
        run_people = _f(run_cost.get("people_days", 0))
        if run_fixed > 0:
            annual_cost += run_fixed
        if run_people > 0:
            annual_cost += run_people * daily_rate
        return annual_cost

    @staticmethod
    def compute_budget_overview(queryset):
        from core.utils import get_global_currency, format_currency
        from global_settings.models import GlobalSettings

        # Single DB hit for daily_rate instead of N hits via ctrl.annual_cost
        _f = ActionPlanBudgetOverview._safe_float
        general_settings = GlobalSettings.objects.filter(name="general").first()
        daily_rate = _f(
            general_settings.value.get("daily_rate", 500) if general_settings else 500,
            500,
        )

        currency = get_global_currency()
        fmt = lambda v: format_currency(v, currency)

        controls = list(queryset)
        count = len(controls)
        by_status: dict = {}
        by_priority: dict = {}
        by_category: dict = {}
        total_annual_cost = 0.0
        count_with_cost = 0

        for ctrl in controls:
            cost = ActionPlanBudgetOverview._compute_annual_cost(ctrl, daily_rate)
            if cost > 0:
                count_with_cost += 1
            total_annual_cost += cost

            # by status — use raw value as key, display value for frontend
            s = ctrl.status or "_unset"
            status_label = ctrl.get_status_display() if ctrl.status else "not_set"
            bucket = by_status.setdefault(
                s, {"status": status_label, "count": 0, "total": 0.0}
            )
            bucket["count"] += 1
            bucket["total"] += cost

            # by priority
            p = ctrl.priority if ctrl.priority is not None else "_unset"
            priority_label = (
                ctrl.get_priority_display() if ctrl.priority is not None else "not_set"
            )
            bucket = by_priority.setdefault(
                p,
                {"priority": priority_label, "count": 0, "total": 0.0},
            )
            bucket["count"] += 1
            bucket["total"] += cost

            # by category
            c = ctrl.category or "_unset"
            category_label = ctrl.get_category_display() if ctrl.category else "not_set"
            bucket = by_category.setdefault(
                c,
                {"category": category_label, "count": 0, "total": 0.0},
            )
            bucket["count"] += 1
            bucket["total"] += cost

        # Pre-format totals with proper currency position
        for bucket in by_status.values():
            bucket["total_display"] = fmt(bucket["total"])
        for bucket in by_priority.values():
            bucket["total_display"] = fmt(bucket["total"])
        for bucket in by_category.values():
            bucket["total_display"] = fmt(bucket["total"])

        return {
            "count": count,
            "count_with_cost": count_with_cost,
            "total_annual_cost": round(total_annual_cost, 2),
            "total_annual_cost_display": fmt(round(total_annual_cost, 2)),
            "currency": currency,
            "by_status": [v for v in by_status.values() if v["count"] > 0],
            "by_priority": [v for v in by_priority.values() if v["count"] > 0],
            "by_category": [v for v in by_category.values() if v["count"] > 0],
        }


class UserRolesOnFolderList(generics.ListAPIView):
    filterset_fields = {}
    search_fields = ["email"]
    serializer_class = UserRolesOnFolderSerializer

    filter_backends = [
        DjangoFilterBackend,
        filters.SearchFilter,
        filters.OrderingFilter,
    ]
    ordering_fields = "__all__"
    ordering = ["email"]

    _user_roles_map = None  # cached variable

    def get_queryset(self):
        folder = get_object_or_404(Folder, id=self.kwargs["pk"])

        # authorize
        (viewable_ids, _updatable_ids, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), self.request.user, Folder
        )
        if folder.id not in viewable_ids:
            raise PermissionDenied()

        # visibility - get user IDs the current user can see via IAM filtering
        (visible_user_ids, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), self.request.user, User
        )
        visible_ids = set(visible_user_ids) | {
            self.request.user.id
        }  # always include self

        # roles per user (no role filtering)
        raw_map = folder.get_user_roles()  # {user_id: [Role, ...]}

        # keep users that are visible AND have at least one role in raw_map
        self._user_roles_map = {
            uid: roles
            for uid, roles in raw_map.items()
            if uid in visible_ids and roles  # roles non-empty in raw_map
        }

        return User.objects.filter(id__in=self._user_roles_map.keys())

    def get_serializer_context(self):
        ctx = super().get_serializer_context()
        ctx.update(
            {
                "pk": self.kwargs["pk"],
                "user_roles_map": self._user_roles_map or {},
            }
        )
        return ctx

    def list(self, request, *args, **kwargs):
        if not RoleAssignment.is_access_allowed(
            user=self.request.user,
            perm=Permission.objects.get(codename="change_folder"),
            folder=get_object_or_404(Folder, id=self.kwargs["pk"]),
        ):
            raise PermissionDenied()

        return super().list(request, *args, **kwargs)


class ComplianceAssessmentActionPlanList(ActionPlanList):
    serializer_class = ComplianceAssessmentActionPlanSerializer

    def get_queryset(self):
        """RBAC not automatic as we don't inherit from BaseModelViewSet -> enforce it explicitly"""
        compliance_id = self.kwargs["pk"]

        if not RoleAssignment.is_object_readable(
            self.request.user,
            ComplianceAssessment,
            compliance_id,
        ):
            raise PermissionDenied()

        assessment = ComplianceAssessment.objects.get(id=compliance_id)
        requirement_assessments = assessment.get_requirement_assessments(
            include_non_assessable=True
        )

        qs = AppliedControl.objects.filter(
            requirement_assessments__in=requirement_assessments
        ).distinct()

        viewable_controls, _, _ = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(),
            self.request.user,
            AppliedControl,
        )
        return qs.filter(id__in=viewable_controls)


class ComplianceAssessmentEvidenceList(generics.ListAPIView):
    serializer_class = ComplianceAssessmentEvidenceSerializer
    filterset_fields = {
        "folder": ["exact"],
        "status": ["exact"],
        "owner": ["exact"],
        "name": ["icontains"],
        "expiry_date": ["exact", "lte", "gte"],
        "created_at": ["exact", "lte", "gte"],
        "updated_at": ["exact", "lte", "gte"],
    }
    search_fields = ["name", "description"]
    ordering_fields = ["name", "status", "updated_at", "expiry_date"]
    ordering = ["name"]

    filter_backends = [
        DjangoFilterBackend,
        filters.SearchFilter,
        filters.OrderingFilter,
    ]

    def get_serializer_context(self):
        context = super().get_serializer_context()
        context.update({"pk": self.kwargs["pk"]})
        return context

    def get_queryset(self):
        """RBAC not automatic as we don't inherit from BaseModelViewSet -> enforce it explicitly"""
        compliance_id = self.kwargs["pk"]

        if not RoleAssignment.is_object_readable(
            self.request.user,
            ComplianceAssessment,
            compliance_id,
        ):
            raise PermissionDenied()

        compliance_assessment = ComplianceAssessment.objects.get(id=compliance_id)

        # Get all requirement assessments for this compliance assessment
        requirement_assessments = RequirementAssessment.objects.filter(
            compliance_assessment=compliance_assessment
        ).prefetch_related("evidences", "applied_controls__evidences")

        # Get visible evidences to filter result
        viewable_evidences, _, _ = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), self.request.user, Evidence
        )

        # Collect evidence IDs from both direct and indirect relationships
        evidence_ids = set()
        for req_assessment in requirement_assessments:
            for evidence in req_assessment.evidences.all():
                if evidence.id in viewable_evidences:
                    evidence_ids.add(evidence.id)
            for applied_control in req_assessment.applied_controls.all():
                for evidence in applied_control.evidences.all():
                    if evidence.id in viewable_evidences:
                        evidence_ids.add(evidence.id)

        return Evidence.objects.filter(id__in=evidence_ids).distinct()


class RiskAssessmentActionPlanList(ActionPlanList):
    serializer_class = RiskAssessmentActionPlanSerializer

    def get_queryset(self):
        """RBAC not automatic as we don't inherit from BaseModelViewSet -> enforce it explicitly"""
        risk_id = self.kwargs["pk"]

        if not RoleAssignment.is_object_readable(
            self.request.user,
            RiskAssessment,
            risk_id,
        ):
            raise PermissionDenied()

        assessment = RiskAssessment.objects.get(id=risk_id)
        risk_scenarios = assessment.risk_scenarios.all()

        qs = AppliedControl.objects.filter(
            Q(risk_scenarios__in=risk_scenarios)
            | Q(risk_scenarios_e__in=risk_scenarios)
        ).distinct()

        viewable_controls, _, _ = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(),
            self.request.user,
            AppliedControl,
        )
        return qs.filter(id__in=viewable_controls)


class ComplianceAssessmentActionPlanBudgetOverview(
    ActionPlanBudgetOverview, ComplianceAssessmentActionPlanList
):
    def get(self, request, *args, **kwargs):
        qs = self.filter_queryset(self.get_queryset())
        return Response(self.compute_budget_overview(qs))


class RiskAssessmentActionPlanBudgetOverview(
    ActionPlanBudgetOverview, RiskAssessmentActionPlanList
):
    def get(self, request, *args, **kwargs):
        qs = self.filter_queryset(self.get_queryset())
        return Response(self.compute_budget_overview(qs))


class PolicyViewSet(AppliedControlViewSet):
    model = Policy
    filterset_fields = [
        "folder",
        "csf_function",
        "status",
        "reference_control",
        "effort",
        "risk_scenarios",
        "requirement_assessments",
        "evidences",
        "genericcollection",
    ]
    search_fields = ["name", "description", "ref_id"]

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get csf_function choices")
    def csf_function(self, request):
        return Response(dict(AppliedControl.CSF_FUNCTION))


class RiskScenarioFilter(GenericFilterSet):
    risk_assessment = df.ModelMultipleChoiceFilter(
        queryset=RiskAssessment.objects.all()
    )
    # Aliased filters for user-friendly query params
    folder = df.UUIDFilter(
        field_name="risk_assessment__perimeter__folder", label="Folder ID"
    )
    perimeter = df.UUIDFilter(
        field_name="risk_assessment__perimeter", label="Perimeter ID"
    )
    within_tolerance = df.ChoiceFilter(
        choices=[("YES", "YES"), ("NO", "NO"), ("--", "--")],
        method="filter_within_tolerance",
    )
    applied_controls = df.ModelMultipleChoiceFilter(
        method="filter_applied_controls",
        queryset=AppliedControl.objects.all(),
    )
    exclude = df.UUIDFilter(
        method="filter_exclude",
        label="Exclude scenario",
    )

    def filter_within_tolerance(self, queryset, name, value):
        if value == "YES":
            return queryset.filter(
                risk_assessment__risk_tolerance__gte=0,
                current_level__lte=models.F("risk_assessment__risk_tolerance"),
            )
        elif value == "NO":
            return queryset.filter(
                risk_assessment__risk_tolerance__gte=0,
                current_level__gt=models.F("risk_assessment__risk_tolerance"),
            )
        elif value == "--":
            return queryset.filter(risk_assessment__risk_tolerance__lt=0)
        return queryset

    def filter_applied_controls(self, queryset, name, value):
        """Filter by both extra controls (applied_controls) and existing controls (existing_applied_controls)"""
        if value:
            return queryset.filter(
                Q(applied_controls__in=value) | Q(existing_applied_controls__in=value)
            ).distinct()
        return queryset

    def filter_exclude(self, queryset, name, value):
        """Exclude a specific scenario from the queryset"""
        if value:
            return queryset.exclude(id=value)
        return queryset

    class Meta:
        model = RiskScenario
        # Only include actual model fields here
        fields = {
            "name": ["exact"],
            "current_impact": ["exact"],
            "current_proba": ["exact"],
            "current_level": ["exact"],
            "residual_impact": ["exact"],
            "residual_proba": ["exact"],
            "residual_level": ["exact"],
            "treatment": ["exact"],
            "threats": ["exact"],
            "assets": ["exact"],
            "existing_applied_controls": ["exact"],
            "security_exceptions": ["exact"],
            "owner": ["exact"],
            "qualifications": ["exact"],
            "filtering_labels": ["exact"],
        }


class RiskScenarioViewSet(ExportMixin, BaseModelViewSet):
    """
    API endpoint that allows risk scenarios to be viewed or edited.
    """

    model = RiskScenario
    filterset_class = RiskScenarioFilter
    ordering = ["ref_id"]
    search_fields = ["name", "description", "ref_id"]

    export_config = {
        "fields": {
            "internal_id": {"source": "id", "label": "internal_id"},
            "ref_id": {"source": "ref_id", "label": "ref_id", "escape": True},
            "name": {"source": "name", "label": "name", "escape": True},
            "description": {
                "source": "description",
                "label": "description",
                "escape": True,
            },
            "risk_assessment": {
                "source": "risk_assessment.name",
                "label": "risk_assessment",
                "escape": True,
            },
            "treatment": {"source": "get_treatment_display", "label": "treatment"},
            "inherent_probability": {
                "source": "get_inherent_proba",
                "label": "inherent_probability",
                "format": lambda v: v.get("name", "--"),
            },
            "inherent_impact": {
                "source": "get_inherent_impact",
                "label": "inherent_impact",
                "format": lambda v: v.get("name", "--"),
            },
            "inherent_level": {
                "source": "get_inherent_risk",
                "label": "inherent_level",
                "format": lambda v: v.get("name", "--"),
            },
            "current_probability": {
                "source": "get_current_proba",
                "label": "current_probability",
                "format": lambda v: v.get("name", "--"),
            },
            "current_impact": {
                "source": "get_current_impact",
                "label": "current_impact",
                "format": lambda v: v.get("name", "--"),
            },
            "current_level": {
                "source": "get_current_risk",
                "label": "current_level",
                "format": lambda v: v.get("name", "--"),
            },
            "residual_probability": {
                "source": "get_residual_proba",
                "label": "residual_probability",
                "format": lambda v: v.get("name", "--"),
            },
            "residual_impact": {
                "source": "get_residual_impact",
                "label": "residual_impact",
                "format": lambda v: v.get("name", "--"),
            },
            "residual_level": {
                "source": "get_residual_risk",
                "label": "residual_level",
                "format": lambda v: v.get("name", "--"),
            },
            "owners": {
                "source": "owner",
                "label": "owners",
                "format": lambda qs: ",".join(
                    escape_excel_formula(str(o)) for o in qs.all()
                ),
            },
            "threats": {
                "source": "threats",
                "label": "threats",
                "format": lambda qs: ",".join(
                    escape_excel_formula(t.name) for t in qs.all()
                ),
            },
            "assets": {
                "source": "assets",
                "label": "assets",
                "format": lambda qs: ",".join(
                    escape_excel_formula(a.name) for a in qs.all()
                ),
            },
            "vulnerabilities": {
                "source": "vulnerabilities",
                "label": "vulnerabilities",
                "format": lambda qs: ",".join(
                    escape_excel_formula(v.name) for v in qs.all()
                ),
            },
            "applied_controls": {
                "source": "applied_controls",
                "label": "applied_controls",
                "format": lambda qs: ",".join(
                    escape_excel_formula(c.name) for c in qs.all()
                ),
            },
            "existing_applied_controls": {
                "source": "existing_applied_controls",
                "label": "existing_applied_controls",
                "format": lambda qs: ",".join(
                    escape_excel_formula(c.name) for c in qs.all()
                ),
            },
            "qualifications": {
                "source": "qualifications",
                "label": "qualifications",
                "format": lambda qs: ",".join(
                    escape_excel_formula(q.name) for q in qs.all()
                ),
            },
        },
        "filename": "risk_scenarios_export",
        "select_related": ["risk_assessment"],
        "prefetch_related": [
            "owner",
            "threats",
            "assets",
            "vulnerabilities",
            "applied_controls",
            "existing_applied_controls",
            "qualifications",
        ],
    }

    def get_queryset(self):
        queryset = super().get_queryset()
        return queryset.select_related(
            "risk_assessment",
            "risk_assessment__risk_matrix",
            "risk_assessment__perimeter",
            "risk_assessment__perimeter__folder",
        ).prefetch_related(
            "threats",
            "assets",
            "applied_controls",
            "existing_applied_controls",
            "owner",
            "security_exceptions",
        )

    def _perform_write(self, serializer):
        if not serializer.validated_data.get(
            "ref_id"
        ) and serializer.validated_data.get("risk_assessment"):
            risk_assessment = serializer.validated_data["risk_assessment"]
            ref_id = RiskScenario.get_default_ref_id(risk_assessment)
            serializer.validated_data["ref_id"] = ref_id
        serializer.save()

    def perform_create(self, serializer):
        return self._perform_write(serializer)

    def perform_update(self, serializer):
        return self._perform_write(serializer)

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get treatment choices")
    def treatment(self, request):
        return Response(dict(RiskScenario.TREATMENT_OPTIONS))

    @action(detail=False, name="Get qualifications count")
    def qualifications_count(self, request):
        folder_id = request.query_params.get("folder", None)
        return Response(
            {"results": qualifications_count_per_name(request.user, folder_id)}
        )

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=True, name="Get probability choices")
    def probability(self, request, pk):
        undefined = {-1: "--"}
        _choices = {
            i: name
            for i, name in enumerate(
                x["name"] for x in self.get_object().get_matrix()["probability"]
            )
        }
        choices = undefined | _choices
        return Response(choices)

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=True, name="Get impact choices")
    def impact(self, request, pk):
        undefined = dict([(-1, "--")])
        _choices = dict(
            zip(
                list(range(0, 64)),
                [x["name"] for x in self.get_object().get_matrix()["impact"]],
            )
        )
        choices = undefined | _choices
        return Response(choices)

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=True, name="Get strength of knowledge choices")
    def strength_of_knowledge(self, request, pk):
        undefined = {-1: RiskScenario.DEFAULT_SOK_OPTIONS[-1]}
        _sok_choices = self.get_object().get_matrix().get("strength_of_knowledge")
        if _sok_choices is not None:
            sok_choices = dict(
                enumerate(
                    {
                        "name": x["name"],
                        "description": x.get("description"),
                        "symbol": x.get("symbol"),
                    }
                    for x in _sok_choices
                )
            )
        else:
            sok_choices = RiskScenario.DEFAULT_SOK_OPTIONS
        choices = undefined | sok_choices
        return Response(choices)

    @action(detail=False, name="Get risk count per level")
    def count_per_level(self, request):
        folder_id = request.query_params.get("folder", None)
        return Response(
            {
                "results": risks_count_per_level(
                    request.user,
                    None,
                    folder_id,
                    include_inherent=ff_is_enabled("inherent_risk"),
                )
            }
        )

    @action(detail=False, name="Get risk scenarios count per status")
    def per_status(self, request):
        return Response({"results": risk_per_status(request.user)})

    @action(
        detail=False, methods=["get"], permission_classes=[permissions.IsAuthenticated]
    )
    def default_ref_id(self, request):
        risk_assessment_id = request.query_params.get("risk_assessment")
        if not risk_assessment_id:
            return Response(
                {"error": "Missing 'risk_assessment' parameter."}, status=400
            )
        try:
            risk_assessment = RiskAssessment.objects.get(pk=risk_assessment_id)

            # Use the class method to compute the default ref_id
            default_ref_id = RiskScenario.get_default_ref_id(risk_assessment)
            return Response({"results": default_ref_id})
        except Exception as e:
            logger.error("Error in default_ref_id: %s", str(e))
            return Response(
                {"error": "Error in default_ref_id has occurred."}, status=400
            )

    @action(
        detail=True,
        methods=["post"],
        url_path="sync-to-actions",
    )
    def sync_to_applied_controls(self, request, pk):
        dry_run = request.query_params.get("dry_run", True)
        if dry_run == "false":
            dry_run = False
        reset_residual = request.data.get("reset_residual", False)
        risk_scenario = RiskScenario.objects.get(id=pk)

        if not RoleAssignment.is_access_allowed(
            user=request.user,
            perm=Permission.objects.get(codename="change_riskscenario"),
            folder=Folder.get_folder(risk_scenario),
        ):
            return Response(status=status.HTTP_403_FORBIDDEN)

        changes = risk_scenario.sync_to_applied_controls(
            reset_residual=reset_residual, dry_run=dry_run
        )
        return Response(
            {"changes": AppliedControlReadSerializer(changes, many=True).data}
        )


class RiskAcceptanceFilterSet(GenericFilterSet):
    folder = df.ModelMultipleChoiceFilter(queryset=Folder.objects.all())
    approver = df.ModelMultipleChoiceFilter(queryset=User.objects.all())

    to_review = df.BooleanFilter(method="filter_to_review")

    def filter_to_review(self, queryset, name, value):
        if value:
            return (
                queryset.filter(expiry_date__lte=date.today() + timedelta(days=30))
                .filter(state__in=["submitted", "accepted"])
                .order_by("expiry_date")
            )

        return queryset

    class Meta:
        model = RiskAcceptance
        fields = {
            "state": ["exact"],
            "risk_scenarios": ["exact"],
            "expiry_date": ["exact", "lte", "gte", "lt", "gt", "month", "year"],
        }


class RiskAcceptanceViewSet(BaseModelViewSet):
    """
    API endpoint that allows risk acceptance to be viewed or edited.
    """

    permission_overrides = {
        "accept": "approve_riskacceptance",
        "reject": "approve_riskacceptance",
        "revoke": "approve_riskacceptance",
    }

    model = RiskAcceptance
    serializer_class = RiskAcceptanceWriteSerializer
    filterset_class = RiskAcceptanceFilterSet
    search_fields = ["name", "description", "justification"]

    def update(self, request, *args, **kwargs):
        initial_data = self.get_object()
        updated_data = request.data
        if (
            updated_data.get("justification") != initial_data.justification
            and request.user != initial_data.approver
        ):
            _data = {
                "non_field_errors": "The justification can only be edited by the approver"
            }
            return Response(data=_data, status=status.HTTP_400_BAD_REQUEST)
        else:
            return super().update(request, *args, **kwargs)

    @action(detail=False, name="Get acceptances to review")
    def to_review(self, request):
        acceptances = acceptances_to_review(request.user)

        acceptances = [
            RiskAcceptanceReadSerializer(acceptance).data for acceptance in acceptances
        ]

        """
        The serializer of AppliedControl isn't applied automatically for this function
        """

        return Response({"results": acceptances})

    @action(detail=True, methods=["post"], name="Submit risk acceptance")
    def submit(self, request, pk):
        if self.get_object().approver:
            self.get_object().set_state("submitted")
            return Response({"results": "state updated to submitted"})
        else:
            return Response(
                {"error": "Missing 'approver' field"}, status=status.HTTP_403_FORBIDDEN
            )

    # This set back risk acceptance to "Created"
    @action(detail=True, methods=["post"], name="Draft risk acceptance")
    def draft(self, request, pk):
        self.get_object().set_state("created")
        return Response({"results": "state updated back to created"})

    @action(detail=True, methods=["post"], name="Accept risk acceptance")
    def accept(self, request, pk):
        if request.user != self.get_object().approver:
            logger.error(
                "Only the approver can accept the risk acceptance",
                user=request.user,
                approver=self.get_object().approver,
            )
            raise PermissionDenied(
                {"error": "Only the approver can accept the risk acceptance"}
            )
        self.get_object().set_state("accepted")
        return Response({"results": "state updated to accepted"})

    @action(detail=True, methods=["post"], name="Reject risk acceptance")
    def reject(self, request, pk):
        if request.user != self.get_object().approver:
            logger.error(
                "Only the approver can reject the risk acceptance",
                user=request.user,
                approver=self.get_object().approver,
            )
            raise PermissionDenied(
                {"error": "Only the approver can reject the risk acceptance"}
            )
        self.get_object().set_state("rejected")
        return Response({"results": "state updated to rejected"})

    @action(detail=True, methods=["post"], name="Revoke risk acceptance")
    def revoke(self, request, pk):
        if request.user != self.get_object().approver:
            logger.error(
                "Only the approver can revoke the risk acceptance",
                user=request.user,
                approver=self.get_object().approver,
            )
            raise PermissionDenied(
                {"error": "Only the approver can revoke the risk acceptance"}
            )
        self.get_object().set_state("revoked")
        return Response({"results": "state updated to revoked"})

    @action(detail=False, methods=["get"], name="Get waiting risk acceptances")
    def waiting(self, request):
        acceptance_count = RiskAcceptance.objects.filter(
            approver=request.user, state="submitted"
        ).count()
        return Response({"count": acceptance_count})

    def perform_update(self, serializer):
        risk_acceptance = serializer.validated_data

        if risk_acceptance.get("approver"):
            for scenario in risk_acceptance.get("risk_scenarios"):
                if not RoleAssignment.is_access_allowed(
                    risk_acceptance.get("approver"),
                    Permission.objects.get(codename="approve_riskacceptance"),
                    scenario.risk_assessment.perimeter.folder,
                ):
                    raise ValidationError(
                        "The approver is not allowed to approve this risk acceptance"
                    )
        risk_acceptance = serializer.save()
        dispatch_webhook_event(risk_acceptance, "updated", serializer)

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get state choices")
    def state(self, request):
        return Response(dict(RiskAcceptance.ACCEPTANCE_STATE))


class UserFilter(GenericFilterSet):
    is_approver = df.BooleanFilter(method="filter_approver", label="Approver")
    is_applied_control_owner = df.BooleanFilter(
        method="filter_applied_control_owner", label="Applied control owner"
    )
    exclude_current = df.BooleanFilter(
        method="filter_exclude_current", label="Exclude current user"
    )

    def filter_approver(self, queryset, name, value):
        """we don't know yet which folders will be used, so filter on any folder"""
        approvers_id = []
        for candidate in User.objects.all():
            if "approve_riskacceptance" in candidate.permissions:
                approvers_id.append(candidate.id)
        if value:
            return queryset.filter(id__in=approvers_id)
        return queryset.exclude(id__in=approvers_id)

    def filter_applied_control_owner(self, queryset, name, value):
        return queryset.filter(applied_controls__isnull=not value)

    def filter_exclude_current(self, queryset, name, value):
        """Exclude the current user from the queryset"""
        if value and self.request and self.request.user:
            return queryset.exclude(id=self.request.user.id)
        return queryset

    class Meta:
        model = User
        fields = [
            "email",
            "first_name",
            "last_name",
            "is_active",
            "keep_local_login",
            "is_approver",
            "is_third_party",
            "expiry_date",
            "user_groups",
            "exclude_current",
            "representative__entity",
        ]


class ValidationFlowFilterSet(GenericFilterSet):
    folder = df.ModelMultipleChoiceFilter(queryset=Folder.objects.all())
    requester = df.ModelMultipleChoiceFilter(queryset=User.objects.all())
    approver = df.ModelMultipleChoiceFilter(queryset=User.objects.all())

    linked_models = df.CharFilter(method="filter_linked_models", label="Linked models")

    def filter_linked_models(self, queryset, name, value):
        """
        Filter validation flows by linked model types.
        Usage: ?linked_models=risk_assessments,evidences
        """
        if not value:
            return queryset

        model_types = [m.strip() for m in value.split(",")]
        filtered_qs = queryset

        for model_type in model_types:
            # Use the field name with __isnull=False to check if objects are linked
            filter_kwargs = {f"{model_type}__isnull": False}
            filtered_qs = filtered_qs.filter(**filter_kwargs).distinct()

        return filtered_qs

    class Meta:
        model = ValidationFlow
        fields = [
            "status",
            "filtering_labels",
            "compliance_assessments",
            "risk_assessments",
            "crq_studies",
            "ebios_studies",
            "entity_assessments",
            "findings_assessments",
            "evidences",
            "security_exceptions",
            "policies",
        ]


class ValidationFlowViewSet(BaseModelViewSet):
    """
    API endpoint that allows validation flows to be viewed or edited.
    """

    model = ValidationFlow
    serializer_class = ValidationFlowWriteSerializer
    filterset_class = ValidationFlowFilterSet
    search_fields = ["ref_id", "request_notes"]

    def get_queryset(self):
        queryset = super().get_queryset()

        def related_model(field_name):
            return ValidationFlow._meta.get_field(field_name).remote_field.model

        events_qs = FlowEvent.objects.select_related("event_actor").order_by(
            "-created_at"
        )
        compliance_qs = ComplianceAssessment.objects.select_related("perimeter__folder")
        risk_qs = RiskAssessment.objects.select_related("perimeter__folder")
        bia_model = related_model("business_impact_analysis")
        bia_qs = bia_model.objects.select_related("perimeter__folder")
        crq_model = related_model("crq_studies")
        ebios_model = related_model("ebios_studies")
        entity_assessment_model = related_model("entity_assessments")
        entity_assessment_qs = entity_assessment_model.objects.select_related(
            "perimeter__folder"
        )
        findings_assessment_model = related_model("findings_assessments")
        findings_assessment_qs = findings_assessment_model.objects.select_related(
            "perimeter__folder"
        )
        evidence_model = related_model("evidences")
        security_exception_model = related_model("security_exceptions")
        policy_model = related_model("policies")
        processing_model = related_model("processings")
        accreditation_model = related_model("accreditations")
        contract_model = related_model("contracts")

        queryset = queryset.select_related("requester", "approver").prefetch_related(
            Prefetch("events", queryset=events_qs),
            Prefetch("compliance_assessments", queryset=compliance_qs),
            Prefetch("risk_assessments", queryset=risk_qs),
            Prefetch("business_impact_analysis", queryset=bia_qs),
            Prefetch("crq_studies", queryset=crq_model.objects.all()),
            Prefetch("ebios_studies", queryset=ebios_model.objects.all()),
            Prefetch("entity_assessments", queryset=entity_assessment_qs),
            Prefetch("findings_assessments", queryset=findings_assessment_qs),
            Prefetch(
                "evidences",
                queryset=evidence_model.objects.select_related("folder"),
            ),
            Prefetch(
                "security_exceptions",
                queryset=security_exception_model.objects.select_related("folder"),
            ),
            Prefetch(
                "policies",
                queryset=policy_model.objects.select_related("folder"),
            ),
            Prefetch(
                "processings",
                queryset=processing_model.objects.select_related("folder"),
            ),
            Prefetch(
                "accreditations",
                queryset=accreditation_model.objects.select_related("folder"),
            ),
            Prefetch(
                "contracts",
                queryset=contract_model.objects.select_related("folder"),
            ),
        )

        m2m_through_fields = {
            "has_compliance_assessments": ValidationFlow.compliance_assessments.through,
            "has_risk_assessments": ValidationFlow.risk_assessments.through,
            "has_business_impact_analysis": ValidationFlow.business_impact_analysis.through,
            "has_crq_studies": ValidationFlow.crq_studies.through,
            "has_ebios_studies": ValidationFlow.ebios_studies.through,
            "has_entity_assessments": ValidationFlow.entity_assessments.through,
            "has_findings_assessments": ValidationFlow.findings_assessments.through,
            "has_evidences": ValidationFlow.evidences.through,
            "has_security_exceptions": ValidationFlow.security_exceptions.through,
            "has_policies": ValidationFlow.policies.through,
            "has_processings": ValidationFlow.processings.through,
            "has_accreditations": ValidationFlow.accreditations.through,
            "has_contracts": ValidationFlow.contracts.through,
        }
        annotations = {
            alias: Exists(through.objects.filter(validationflow_id=OuterRef("pk")))
            for alias, through in m2m_through_fields.items()
        }

        return queryset.annotate(**annotations)

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get status choices")
    def status(self, request):
        return Response(dict(ValidationFlow.Status.choices))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get linked models choices")
    def linked_models(self, request):
        """Return available model types that can be linked to validation flows"""
        model_types = {
            "compliance_assessments": "Compliance Assessments",
            "risk_assessments": "Risk Assessments",
            "business_impact_analysis": "Business Impact Analysis",
            "crq_studies": "Quantitative Risk Studies",
            "ebios_studies": "EBIOS RM Studies",
            "entity_assessments": "Entity Assessments",
            "findings_assessments": "Findings Assessments",
            "evidences": "Evidences",
            "security_exceptions": "Security Exceptions",
            "policies": "Policies",
            "processings": "Processings",
            "accreditations": "Accreditations",
            "contracts": "Contracts",
        }
        return Response(model_types)

    @action(
        detail=False, methods=["get"], permission_classes=[permissions.IsAuthenticated]
    )
    def default_ref_id(self, request):
        try:
            # Use the class method to compute the default ref_id (globally unique)
            default_ref_id = ValidationFlow.get_default_ref_id()
            return Response({"results": default_ref_id})
        except Exception as e:
            logger.error("Error in default_ref_id: %s", str(e))
            return Response(
                {"error": "Error in default_ref_id has occurred."}, status=400
            )


class ActorViewSet(BaseModelViewSet):
    http_method_names = ["get", "head", "options"]

    model = Actor
    search_fields = []
    ordering = [
        "type_rank",
        "display_name",
        "id",
    ]

    def get_queryset(self):
        queryset = super().get_queryset()
        queryset = queryset.select_related("user", "team", "entity")

        queryset = queryset.annotate(
            # Define the order: User (1), Team (2), Entity (3)
            type_rank=Case(
                When(user__isnull=False, then=Value(1)),
                When(team__isnull=False, then=Value(2)),
                When(entity__isnull=False, then=Value(3)),
                default=Value(4),  # Fallback
                output_field=IntegerField(),
            ),
            # Combine the different name fields into one column
            display_name=Coalesce(
                "user__email", "team__name", "entity__name", output_field=CharField()
            ),
        )

        allow_entities = (
            GlobalSettings.objects.filter(name=GlobalSettings.Names.GENERAL)
            .values_list("value__allow_assignments_to_entities", flat=True)
            .first()
        )
        if not allow_entities:
            queryset = queryset.filter(entity__isnull=True)

        third_parties = Actor.objects.filter(user__is_third_party=True)
        queryset = queryset.exclude(id__in=third_parties)

        return queryset.order_by("type_rank", "display_name")


class TeamViewSet(BaseModelViewSet):
    model = Team

    def get_queryset(self):
        return (
            super()
            .get_queryset()
            .select_related("folder", "leader")
            .prefetch_related("members", "deputies")
        )


class UserViewSet(BaseModelViewSet):
    """
    API endpoint that allows users to be viewed or edited
    """

    model = User
    ordering = ["-is_active", "-is_superuser", "email", "id"]
    filterset_class = UserFilter
    search_fields = ["email", "first_name", "last_name"]

    def get_queryset(self):
        # Use base IAM filtering
        # but ensure current user is always included
        queryset = super().get_queryset() | User.objects.filter(pk=self.request.user.pk)

        # Add prefetch for user_groups visibility
        viewable_user_group_ids = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), self.request.user, UserGroup
        )[0]
        return queryset.distinct().prefetch_related(
            Prefetch(
                "user_groups",
                queryset=UserGroup.objects.filter(id__in=viewable_user_group_ids).only(
                    "id", "builtin"
                ),
            )
        )

    def update(self, request: Request, *args, **kwargs) -> Response:
        user = self.get_object()
        if user.is_admin():
            number_of_admin_users = User.get_admin_users().count()
            admin_group = UserGroup.objects.get(name="BI-UG-ADM")
            if number_of_admin_users == 1:
                new_user_groups = set(request.data["user_groups"])
                if str(admin_group.pk) not in new_user_groups:
                    return Response(
                        {"error": "attemptToRemoveOnlyAdminUserGroup"},
                        status=status.HTTP_403_FORBIDDEN,
                    )

        return super().update(request, *args, **kwargs)

    def destroy(self, request, *args, **kwargs):
        user = self.get_object()
        if user.is_admin():
            number_of_admin_users = User.get_admin_users().count()
            if number_of_admin_users == 1:
                return Response(
                    {"error": "attemptToDeleteOnlyAdminAccountError"},
                    status=status.HTTP_403_FORBIDDEN,
                )

        return super().destroy(request, *args, **kwargs)


class UserGroupOrderingFilter(filters.OrderingFilter):
    """
    Custom ordering filter:
    - Performs in-memory (Python) sorting only for `localization_dict`.
    - The sort key is a tuple: (folder full_path OR folder.name, object.name).
    - Supports `-localization_dict` for descending order.
    - For all other fields, it falls back to standard SQL ordering.
    """

    def filter_queryset(self, request, queryset, view):
        ordering = self.get_ordering(request, queryset, view)
        if not ordering:
            return queryset

        # Special case: in-memory sorting for `localization_dict`
        if len(ordering) == 1 and ordering[0].lstrip("-") == "localization_dict":
            desc = ordering[0].startswith("-")

            # Optimize DB access: fetch the related folder in one query
            queryset = queryset.select_related("folder")

            # Materialize queryset into a list to sort in Python
            data = list(queryset)

            def full_path_or_name(folder):
                """
                Build a string key from the folder:
                - Prefer the full path (names of all parent folders + current).
                - Fall back to the folder's name if no path is available.
                """
                if folder is None:
                    return ""

                path_list = getattr(folder, "get_folder_full_path", None)
                if callable(path_list):
                    items = folder.get_folder_full_path(include_root=False)
                    names = [getattr(f, "name", "") or "" for f in items]
                    if names:
                        return "/".join(names)

                # Fallback: just the folder name
                return getattr(folder, "name", "") or ""

            def key_func(obj):
                # Get the folder from the object
                folder = getattr(obj, "folder", None)
                # If you want to be more robust, you could use:
                # from yourapp.models import Folder
                # folder = Folder.get_folder(obj)

                path_key = full_path_or_name(folder).casefold()
                name_key = (getattr(obj, "name", "") or "").casefold()
                return (path_key, name_key)

            # Perform stable sort, reverse if `-localization_dict`
            data.sort(key=key_func, reverse=desc)
            return data

        # Default case: fall back to SQL ordering
        return super().filter_queryset(request, queryset, view)


class UserGroupViewSet(BaseModelViewSet):
    """
    API endpoint that allows user groups to be viewed or edited
    """

    model = UserGroup
    ordering = ["builtin", "name"]
    ordering_fields = ["localization_dict"]
    filterset_fields = ["folder"]
    search_fields = [
        "folder__name"
    ]  # temporary hack, filters only by folder name, not role name
    filter_backends = [
        DjangoFilterBackend,
        UserGroupOrderingFilter,
        filters.SearchFilter,
    ]

    def destroy(self, request, *args, **kwargs):
        user_group = self.get_object()
        if user_group.builtin:
            return Response(
                {"error": "attemptToDeleteBuiltinUserGroup"},
                status=status.HTTP_403_FORBIDDEN,
            )
        return super().destroy(request, *args, **kwargs)


class RoleAssignmentViewSet(BaseModelViewSet):
    """
    API endpoint that allows role assignments to be viewed or edited.
    """

    model = RoleAssignment
    ordering = ["builtin", "folder"]
    filterset_fields = ["folder"]


class FolderFilter(GenericFilterSet):
    parent_folder = df.ModelMultipleChoiceFilter(queryset=Folder.objects.all())

    owned = df.BooleanFilter(method="get_owned_folders", label="owned")
    content_type = df.MultipleChoiceFilter(
        choices=Folder.ContentType, lookup_expr="icontains"
    )

    def get_owned_folders(self, queryset, name, value):
        owned_folders_id = []
        for folder in Folder.objects.all():
            if folder.owner.all().first():
                owned_folders_id.append(folder.id)
        if value:
            return queryset.filter(id__in=owned_folders_id)
        return queryset.exclude(id__in=owned_folders_id)

    class Meta:
        model = Folder
        fields = [
            "name",
            "content_type",
            "owner",
            "owned",
            "filtering_labels",
        ]


class FolderViewSet(BaseModelViewSet):
    """
    API endpoint that allows folders to be viewed or edited.
    """

    model = Folder
    filterset_class = FolderFilter
    search_fields = ["name"]
    batch_size = 100  # Configurable batch size for processing domain import

    def perform_create(self, serializer):
        """
        Create the default user groups after domain creation
        """
        folder = serializer.save()
        Folder.create_default_ug_and_ra(folder)

    def list(self, request, *args, **kwargs):
        return super().list(request, *args, **kwargs)

    def retrieve(self, request, *args, **kwargs):
        return super().retrieve(request, *args, **kwargs)

    @action(detail=False, methods=["get"])
    def org_tree(self, request):
        """
        Returns the tree of domains and perimeters
        """
        include_perimeters = request.query_params.get(
            "include_perimeters", "True"
        ).lower() in ["true", "1", "yes"]
        include_enclaves = request.query_params.get(
            "include_enclaves", "False"
        ).lower() in ["true", "1", "yes"]

        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            folder=Folder.get_root_folder(),
            user=request.user,
            object_type=Folder,
        )

        # Add ancestors so viewable folders aren't orphaned
        needed_folders = set(viewable_objects)

        for folder_id in viewable_objects:
            current = Folder.objects.get(pk=folder_id)
            while current and current.parent_folder_id:
                needed_folders.add(current.parent_folder_id)
                current = Folder.objects.get(pk=current.parent_folder_id)

        folders_list = []
        for folder in (
            Folder.objects.exclude(content_type="GL")
            .filter(id__in=needed_folders, parent_folder=Folder.get_root_folder())
            .distinct()
        ):
            # Skip enclaves at top level if not included
            if (
                not include_enclaves
                and folder.content_type == Folder.ContentType.ENCLAVE
            ):
                continue
            entry = {
                "name": folder.name,
                "uuid": folder.id,
                "viewable": folder.id in viewable_objects,
                "content_type": folder.content_type,
            }
            # Add enclave-specific styling
            if folder.content_type == Folder.ContentType.ENCLAVE:
                entry.update(
                    {
                        "symbol": "triangle",
                        "symbolSize": 12,
                        "itemStyle": {"color": "#6366f1"},
                    }
                )
            folder_content = get_folder_content(
                folder,
                include_perimeters=include_perimeters,
                include_enclaves=include_enclaves,
                viewable_objects=viewable_objects,
                needed_folders=needed_folders,
            )
            if len(folder_content) > 0:
                entry.update({"children": folder_content})
            folders_list.append(entry)

        root_folder = Folder.get_root_folder()
        return Response(
            {
                "name": root_folder.name,
                "uuid": str(root_folder.id),
                "content_type": root_folder.content_type,
                "children": folders_list,
            }
        )

    @action(detail=False, methods=["get"])
    def ids(self, request):
        my_map = dict()

        (viewable_items, _, _) = RoleAssignment.get_accessible_object_ids(
            folder=Folder.get_root_folder(),
            user=request.user,
            object_type=Folder,
        )
        for item in Folder.objects.filter(id__in=viewable_items):
            my_map[item.name] = item.id
        return Response(my_map)

    def _get_quality_checks_for_folders(self, folders, user):
        """
        Helper method to aggregate quality checks for a queryset of folders.
        Enforces RBAC for both folders and assessments.
        """
        # Get viewable assessment IDs for proper RBAC
        (viewable_ca_ids, _, _) = RoleAssignment.get_accessible_object_ids(
            folder=Folder.get_root_folder(), user=user, object_type=ComplianceAssessment
        )
        (viewable_ra_ids, _, _) = RoleAssignment.get_accessible_object_ids(
            folder=Folder.get_root_folder(), user=user, object_type=RiskAssessment
        )

        res = {
            str(f.id): {
                "folder": FolderReadSerializer(f).data,
                "compliance_assessments": {"objects": {}},
                "risk_assessments": {"objects": {}},
            }
            for f in folders
        }
        for ca in ComplianceAssessment.objects.filter(
            folder__in=folders, id__in=viewable_ca_ids
        ):
            res[str(ca.folder.id)]["compliance_assessments"]["objects"][str(ca.id)] = {
                "object": ComplianceAssessmentReadSerializer(ca).data,
                "quality_check": ca.quality_check(),
            }
        for ra in RiskAssessment.objects.filter(
            folder__in=folders, id__in=viewable_ra_ids
        ):
            res[str(ra.folder.id)]["risk_assessments"]["objects"][str(ra.id)] = {
                "object": RiskAssessmentReadSerializer(ra).data,
                "quality_check": ra.quality_check(),
            }
        return res

    @action(detail=False, methods=["get"])
    def quality_check(self, request):
        """
        Returns the quality check of assessments grouped by folder.
        """
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            folder=Folder.get_root_folder(), user=request.user, object_type=Folder
        )
        folders = Folder.objects.filter(id__in=viewable_objects).exclude(
            content_type=Folder.ContentType.ROOT
        )
        return Response(
            {"results": self._get_quality_checks_for_folders(folders, request.user)}
        )

    @action(detail=True, methods=["get"], url_path="quality_check")
    def quality_check_detail(self, request, pk):
        """
        Returns the quality check of assessments for a specific folder.
        """
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            folder=Folder.get_root_folder(), user=request.user, object_type=Folder
        )
        if UUID(pk) not in viewable_objects:
            return Response(status=status.HTTP_403_FORBIDDEN)

        folder = self.get_object()
        result = self._get_quality_checks_for_folders(
            Folder.objects.filter(pk=folder.pk), request.user
        )
        return Response(result.get(str(folder.id), {}))

    @action(detail=False, methods=["get"])
    def my_assignments(self, request):
        include_teams = (
            request.query_params.get("include_teams", "false").lower() == "true"
        )

        if include_teams:
            actors = Actor.get_all_for_user(request.user)
        else:
            actors = [request.user.actor]

        risk_assessments = RiskAssessment.objects.filter(
            Q(authors__in=actors) | Q(reviewers__in=actors)
        ).distinct()

        audits = (
            ComplianceAssessment.objects.filter(
                Q(authors__in=actors) | Q(reviewers__in=actors)
            )
            .order_by(F("eta").asc(nulls_last=True))
            .distinct()
        )

        sum = 0
        avg_progress = 0
        audits_count = audits.count()
        if audits_count > 0:
            for audit in audits:
                sum += audit.progress
            avg_progress = int(sum / audits.count())

        controls = (
            AppliedControl.objects.filter(owner__in=actors)
            .order_by(F("eta").asc(nulls_last=True))
            .distinct()
        )
        non_active_controls = controls.exclude(status="active")
        risk_scenarios = RiskScenario.objects.filter(owner__in=actors).distinct()
        controls_progress = 0
        evidences_progress = 0
        tot_ac = controls.count()
        if tot_ac > 0:
            alive_ac = controls.filter(status="active").count()
            controls_progress = int((alive_ac / tot_ac) * 100)

            with_evidences = 0
            for ctl in controls:
                with_evidences += 1 if ctl.has_evidences() else 0

            evidences_progress = int((with_evidences / tot_ac) * 100)

        RA_serializer = RiskAssessmentReadSerializer(risk_assessments[:10], many=True)
        CA_serializer = ComplianceAssessmentReadSerializer(audits[:6], many=True)
        AC_serializer = AppliedControlReadSerializer(
            non_active_controls[:10], many=True
        )
        RS_serializer = RiskScenarioReadSerializer(risk_scenarios[:10], many=True)

        # Return the actor IDs used for filtering so frontend can use them consistently
        actor_ids = [str(actor.id) for actor in actors]

        return Response(
            {
                "risk_assessments": RA_serializer.data,
                "audits": CA_serializer.data,
                "controls": AC_serializer.data,
                "risk_scenarios": RS_serializer.data,
                "metrics": {
                    "progress": {
                        "audits": avg_progress,
                        "controls": controls_progress,
                        "evidences": evidences_progress,
                    }
                },
                "actor_ids": actor_ids,
            }
        )

    @action(detail=True, methods=["get"])
    def export(self, request, pk):
        """Export the domain as a zip file containing a JSON dump of all objects and their relationships"""
        include_attachments = True
        instance = self.get_object()

        logger.info(
            "Starting domain export",
            domain_id=instance.id,
            domain_name=instance.name,
            include_attachments=include_attachments,
            user=request.user.username,
        )

        objects = get_domain_export_objects(instance)

        for model in objects.keys():
            if not RoleAssignment.is_access_allowed(
                user=request.user,
                perm=Permission.objects.get(codename=f"view_{model}"),
                folder=instance,
            ):
                logger.error(
                    "User does not have permission to export object",
                    user=request.user,
                    model=model,
                )
                raise PermissionDenied(
                    {"error": "userDoesNotHavePermissionToExportDomain"}
                )

        logger.debug(
            "Retrieved domain objects for export",
            object_types=list(objects.keys()),
            total_objects=sum(len(queryset) for queryset in objects.values()),
            objects_per_model={
                model: len(queryset) for model, queryset in objects.items()
            },
        )

        # Create in-memory zip file
        zip_buffer = io.BytesIO()

        with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zipf:
            if include_attachments:
                revisions = objects.get(
                    "evidencerevision", EvidenceRevision.objects.none()
                ).filter(attachment__isnull=False)
                logger.info(
                    "Processing evidence attachments",
                    total_revisions=revisions.count(),
                    domain_id=instance.id,
                )

                for revision in revisions:
                    if revision.attachment and default_storage.exists(
                        revision.attachment.name
                    ):
                        # Read file directly into memory
                        with default_storage.open(revision.attachment.name) as file:
                            file_content = file.read()
                            # Write the file content directly to the zip
                            zipf.writestr(
                                os.path.join(
                                    "attachments",
                                    "evidence-revisions",
                                    f"{revision.evidence_id}_v{revision.version}_"
                                    f"{os.path.basename(revision.attachment.name)}",
                                ),
                                file_content,
                            )

            # Add the JSON dump to the zip file
            dumpfile_name = (
                f"ciso-assistant-{slugify(instance.name)}-domain-{timezone.now()}"
            )
            dump_data = ExportSerializer.dump_data(scope=[*objects.values()])

            logger.debug(
                "Adding JSON dump to zip",
                json_size=len(json.dumps(dump_data).encode("utf-8")),
                filename=f"{dumpfile_name}.json",
            )

            zipf.writestr("data.json", json.dumps(dump_data).encode("utf-8"))

        # Reset buffer position to the start
        zip_buffer.seek(0)
        final_size = len(zip_buffer.getvalue())

        # Create the response with the in-memory zip file
        response = HttpResponse(zip_buffer.getvalue(), content_type="application/zip")
        response["Content-Disposition"] = f'attachment; filename="{dumpfile_name}.zip"'

        logger.info(
            "Domain export completed successfully",
            domain_id=instance.id,
            domain_name=instance.name,
            zip_size=final_size,
            filename=f"{dumpfile_name}.zip",
        )

        return response

    @action(
        detail=False,
        methods=["post"],
        url_path="import",
        parser_classes=(FileUploadParser,),
    )
    def import_domain(self, request):
        """Handle file upload and initiate import process."""
        load_missing_libraries = (
            request.query_params.get("load_missing_libraries", "false").lower()
            == "true"
        )
        try:
            if not RoleAssignment.is_access_allowed(
                user=request.user,
                perm=Permission.objects.get(codename="add_folder"),
                folder=Folder.get_root_folder(),
            ):
                raise PermissionDenied()
            domain_name = request.headers.get(
                "X-CISOAssistantDomainName", str(uuid.uuid4())
            )
            parsed_data = self._process_uploaded_file(request.data["file"])
            result = self._import_objects(
                parsed_data, domain_name, load_missing_libraries, user=request.user
            )
            return Response(result, status=status.HTTP_200_OK)

        except PermissionDenied:
            logger.error(
                "User does not have permission to import domain",
                user=request.user,
                exc_info=True,
            )
            return Response(
                {"error": "userDoesNotHavePermissionToImportDomain"},
                status=status.HTTP_403_FORBIDDEN,
            )

        except KeyError:
            logger.error("No file provided in the request", exc_info=True)
            return Response(
                {"errors": ["No file provided"]}, status=status.HTTP_400_BAD_REQUEST
            )

        except json.JSONDecodeError:
            logger.error("Invalid JSON format in uploaded file", exc_info=True)
            return Response(
                {"errors": ["Invalid JSON format"]}, status=status.HTTP_400_BAD_REQUEST
            )

    @action(
        detail=False,
        methods=["post"],
        url_path="import-dummy",
    )
    def import_dummy_domain(self, request):
        domain_name = "DEMO"
        try:
            PROJECT_DIR = Path(__file__).resolve().parent.parent
            dummy_fixture_path = PROJECT_DIR / "fixtures" / "dummy-domain.bak"
            if not dummy_fixture_path.exists():
                logger.error("Dummy domain fixture not found", path=dummy_fixture_path)
                return Response(
                    {"error": "dummyDomainFixtureNotFound"},
                    status=status.HTTP_500_INTERNAL_SERVER_ERROR,
                )
            parsed_data = self._process_uploaded_file(dummy_fixture_path)
            result = self._import_objects(
                parsed_data, domain_name, load_missing_libraries=True, user=request.user
            )
            logger.info("Dummy domain imported successfully", domain_name=domain_name)
            return Response(result, status=status.HTTP_200_OK)

        except json.JSONDecodeError:
            logger.error("Invalid JSON format in dummy fixture file")
            return Response(
                {"errors": ["Invalid JSON format"]},
                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
            )
        except Exception:
            logger.error("Error importing dummy domain")
            return Response(
                {"error": "failedToImportDummyDomain"},
                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
            )

    def _process_uploaded_file(self, dump_file: str | Path) -> Any:
        """Process the uploaded file and return parsed data."""
        if not zipfile.is_zipfile(dump_file):
            logger.error("Invalid ZIP file format")
            raise ValidationError({"file": "invalidZipFileFormat"})

        with zipfile.ZipFile(dump_file, mode="r") as zipf:
            if "data.json" not in zipf.namelist():
                logger.error(
                    "No data.json file found in uploaded file", files=zipf.namelist()
                )
                raise ValidationError({"file": "noDataJsonFileFound"})
            infolist = zipf.infolist()
            directories = list(set([Path(f.filename).parent.name for f in infolist]))
            decompressed_data = zipf.read("data.json")
            # Decode bytes to string if necessary
            if isinstance(decompressed_data, bytes):
                decompressed_data = decompressed_data.decode("utf-8")
            try:
                json_dump = json.loads(decompressed_data)
                import_version = json_dump["meta"]["media_version"]
                schema_version = json_dump["meta"].get("schema_version")
            except json.JSONDecodeError:
                logger.error("Invalid JSON format in uploaded file", exc_info=True)
                raise
            if "objects" not in json_dump:
                raise ValidationError("badly formatted json")

            # Check backup and local version

            try:
                schema_version_int = int(schema_version)
            except (ValueError, TypeError) as e:
                logger.error(
                    "Invalid schema version format",
                    schema_version=schema_version,
                    exc_info=e,
                )
                raise ValidationError({"error": "invalidSchemaVersionFormat"})
            compare_schema_versions(schema_version_int, import_version)

            if "attachments" in directories:
                attachments = {
                    f for f in infolist if Path(f.filename).parent.name == "attachments"
                }
                logger.info(
                    "Attachments found in uploaded file",
                    attachments_count=len(attachments),
                )
                for attachment in attachments:
                    try:
                        content = zipf.read(attachment)
                        current_name = Path(attachment.filename).name
                        new_name = default_storage.save(
                            current_name, io.BytesIO(content)
                        )
                        if new_name != current_name:
                            for x in json_dump["objects"]:
                                if (
                                    x["model"] == "core.evidence"
                                    and x["fields"]["attachment"] == current_name
                                ):
                                    x["fields"]["attachment"] = new_name

                    except Exception:
                        logger.error("Error extracting attachment", exc_info=True)

        return json_dump

    def _get_models_map(self, objects):
        """Build a map of model names to model classes."""
        model_names = {obj["model"] for obj in objects}
        return {name: apps.get_model(name) for name in model_names}

    def _resolve_dependencies(self, all_models):
        """Resolve model dependencies and detect cycles."""
        logger.debug("Resolving model dependencies", all_models=all_models)

        graph = build_dependency_graph(all_models)

        logger.debug("Dependency graph", graph=graph)

        try:
            return topological_sort(graph)
        except ValueError as e:
            logger.error("Cyclic dependency detected", error=str(e))
            raise ValidationError({"error": "Cyclic dependency detected"})

    def _import_terminologies(
        self,
        names: str | List[str] | None,
        field_path: Terminology.FieldPath,
    ) -> QuerySet[Terminology] | Terminology | None:
        """
        Ensure that requested terminologies exist and are visible. If not, create them. If they exist but are not visible, make them visible.

        Args:
            names: name of the Terminology (str) to import, or list of names for multiple Terminologies.
            field_path: which Terminology.FieldPath to assign

        Returns:
            - For single input: Terminology object
            - For list input: QuerySet of Terminology objects
            - For None or empty list: None
        """
        if not names:
            return None

        # Convert single value to list
        single_value = False
        if isinstance(names, str):  # Foreign key case
            names = [names]
            single_value = True

        # Fetch existing
        existing = Terminology.objects.filter(name__in=names, field_path=field_path)
        existing_names = set(existing.values_list("name", flat=True))

        # Create missing
        missing_names = set(names) - existing_names
        if missing_names:
            Terminology.objects.bulk_create(
                [
                    Terminology(name=name, field_path=field_path, is_visible=True)
                    for name in missing_names
                ],
                ignore_conflicts=True,
            )

        # Ensure visibility
        Terminology.objects.filter(
            name__in=names, field_path=field_path, is_visible=False
        ).update(is_visible=True)

        result_qs = Terminology.objects.filter(name__in=names, field_path=field_path)

        if single_value:
            return result_qs.first()
        return result_qs

    def _import_objects(
        self,
        parsed_data: dict,
        domain_name: str,
        load_missing_libraries: bool,
        user: User,
    ) -> dict[str, str]:
        """
        Import and validate objects using appropriate serializers.
        Handles both validation and creation in separate phases within a transaction.

        Args:
            parsed_data: The data parsed from the uploaded JSON dump.
            domain_name: The name choosen by the user for the new domain being created.
            load_missing_libraries: Whether to automatically load missing libraries from stored ones.
            user: The user performing the import, for permission checks.

        Returns:
            A dict with success message if no error was encountered.
        """
        validation_errors = []
        required_libraries = []
        missing_libraries = []
        link_dump_database_ids = {}

        # First check if objects exist
        objects = parsed_data.get("objects")
        if not objects:
            logger.error("No objects found in the dump")
            raise ValidationError({"error": "No objects found in the dump"})

        try:
            # Validate models and check for domain
            models_map = self._get_models_map(objects)
            if Folder in models_map.values():
                logger.error("Dump contains a domain")
                raise ValidationError({"error": "Dump contains a domain"})

            # check that user has permission to create all objects to import
            error_dict = {}
            for model in filter(
                lambda x: x not in [RequirementAssessment], models_map.values()
            ):
                if not RoleAssignment.is_access_allowed(
                    user=user,
                    perm=Permission.objects.get(
                        codename=f"add_{model._meta.model_name}"
                    ),
                    folder=Folder.get_root_folder(),
                ):
                    error_dict[model._meta.model_name] = "permission_denied"
            if error_dict:
                logger.error(
                    "User does not have permission to import objects",
                    error_dict=error_dict,
                )
                raise PermissionDenied()

            # Validation phase (outside transaction since it doesn't modify database)
            creation_order = self._resolve_dependencies(list(models_map.values()))

            logger.debug("Resolved creation order", creation_order=creation_order)
            logger.debug("Starting objects validation", objects_count=len(objects))

            # Validate all objects first
            for model in creation_order:
                self._validate_model_objects(
                    model=model,
                    objects=objects,
                    validation_errors=validation_errors,
                    required_libraries=required_libraries,
                )

            logger.debug("required_libraries", required_libraries=required_libraries)

            # If validation errors exist, raise them immediately
            if validation_errors:
                logger.error(
                    "Failed to validate objects", validation_errors=validation_errors
                )
                raise ValidationError({"validation_errors": validation_errors})

            # Creation phase - wrap in transaction
            with transaction.atomic():
                # Create base folder and store its ID
                base_folder = Folder.objects.create(
                    name=domain_name,
                    content_type=Folder.ContentType.DOMAIN,
                    create_iam_groups=True,
                )
                link_dump_database_ids["base_folder"] = base_folder
                Folder.create_default_ug_and_ra(base_folder)

                # Check for missing libraries after folder creation
                for library in required_libraries:
                    if not LoadedLibrary.objects.filter(
                        urn=library["urn"], version=library["version"]
                    ).exists():
                        if (
                            StoredLibrary.objects.filter(
                                urn=library["urn"], version__gte=library["version"]
                            ).exists()
                            and load_missing_libraries
                        ):
                            StoredLibrary.objects.get(
                                urn=library["urn"], version__gte=library["version"]
                            ).load()
                        else:
                            missing_libraries.append(library)

                logger.debug("missing_libraries", missing_libraries=missing_libraries)

                # If missing libraries exist, raise specific error
                if missing_libraries:
                    logger.warning(f"Missing libraries: {missing_libraries}")
                    raise ValidationError({"missing_libraries": missing_libraries})

                logger.info(
                    "Starting objects creation",
                    objects_count=len(objects),
                    creation_order=creation_order,
                )

                # Create all objects within the transaction
                for model in creation_order:
                    self._create_model_objects(
                        model=model,
                        objects=objects,
                        link_dump_database_ids=link_dump_database_ids,
                    )

            return {"message": "Import successful"}

        except ValidationError as e:
            logger.error(f"error: {e}")
            raise
        except Exception as e:
            # Handle unexpected errors with a generic message
            logger.exception(f"Failed to import objects: {str(e)}")
            raise ValidationError({"non_field_errors": "errorOccuredDuringImport"})

    def _validate_model_objects(
        self, model, objects, validation_errors, required_libraries
    ):
        """Validate all objects for a model before creation."""
        model_name = f"{model._meta.app_label}.{model._meta.model_name}"
        model_objects = [obj for obj in objects if obj["model"] == model_name]

        if not model_objects:
            return

        # Process validation in batches
        for i in range(0, len(model_objects), self.batch_size):
            batch = model_objects[i : i + self.batch_size]
            self._validate_batch(
                model=model,
                batch=batch,
                validation_errors=validation_errors,
                required_libraries=required_libraries,
            )

    def _validate_batch(self, model, batch, validation_errors, required_libraries):
        """Validate a batch of objects."""
        model_name = f"{model._meta.app_label}.{model._meta.model_name}"

        for obj in batch:
            obj_id = obj.get("id")
            fields = obj.get("fields", {}).copy()

            try:
                # Handle library objects
                if model == LoadedLibrary:
                    required_libraries.append(
                        {"urn": fields["urn"], "version": fields["version"]}
                    )
                    logger.info(
                        "Adding library to required libraries", urn=fields["urn"]
                    )
                    continue
                if fields.get("library"):
                    continue

                # Validate using serializer
                serializer_class = import_export_serializer_class(model)
                serializer = serializer_class(data=fields)

                if not serializer.is_valid():
                    validation_errors.append(
                        {
                            "model": model_name,
                            "id": obj_id,
                            "errors": serializer.errors,
                        }
                    )

            except Exception as e:
                logger.error(
                    f"Error validating object {obj_id} in {model_name}: {str(e)}",
                    exc_info=True,
                )
                validation_errors.append(
                    {
                        "model": model_name,
                        "id": obj_id,
                        "errors": [str(e)],
                    }
                )

    def _create_model_objects(
        self,
        model: models.Model,
        objects: List[dict],
        link_dump_database_ids: dict[str, Any],
    ) -> None:
        """
        Create all objects for a model after validation. Creation happens in batch of size `self.batch_size`.

        Args:
            model: Current model for which objects are being created.
            objects: List of all objects from the dump, used to filter only those relevant for the current model.
            link_dump_database_ids: Mapping of dump ID to real database ID/Object/URN of created objects, used to resolve object relationships.
        """
        logger.debug("Creating objects for model", model=model)

        # rebuild model name and keep objects only of this model for creation
        model_name = f"{model._meta.app_label}.{model._meta.model_name}"
        model_objects = [obj for obj in objects if obj["model"] == model_name]

        logger.debug("Model objects", model=model, count=len(model_objects))

        if not model_objects:
            return

        # Handle self-referencing dependencies
        self_ref_field = get_self_referencing_field(model)
        if self_ref_field:
            try:
                model_objects = sort_objects_by_self_reference(
                    model_objects, self_ref_field
                )
            except ValueError as e:
                logger.error(f"Cyclic dependency detected in {model_name}: {str(e)}")
                raise ValidationError(
                    {"error": f"Cyclic dependency detected in {model_name}"}
                )

        # Process creation in batches
        for i in range(0, len(model_objects), self.batch_size):
            batch = model_objects[i : i + self.batch_size]
            self._create_batch(
                model=model,
                batch=batch,
                link_dump_database_ids=link_dump_database_ids,
            )

    def _create_batch(
        self,
        model: models.Model,
        batch: List[dict],
        link_dump_database_ids: dict[str, Any],
    ) -> None:
        """
        Create a batch of objects with proper relationship handling in 4 main steps:
        1. Handle special cases (e.g. library objects, folder reference)
        2. Process model-specific relationships, separating m2m relationships to be handled in the fourth step, after the object is created (because it requires the database ID of the created object to be set)
        3. Create the object
        4. Handle many-to-many relationships

        Args:
            model: Current model for which objects are being created.
            batch: Sublist of all objects from the dump to create in this batch.
            link_dump_database_ids: Mapping of dump ID to real database ID/Object/URN of created objects, used to resolve object relationships.
        """
        # Create all objects in the batch within a single transaction
        with transaction.atomic():
            try:
                objects_creation_data = []

                for obj in batch:
                    obj_id = obj.get("id")
                    fields = obj.get("fields", {}).copy()

                    # Handle library objects
                    if fields.get("library") or model == LoadedLibrary:
                        logger.info(f"Skipping creation of library object {obj_id}")
                        link_dump_database_ids[obj_id] = fields.get("urn")
                        continue

                    # Handle folder reference
                    if fields.get("folder"):
                        fields["folder"] = link_dump_database_ids.get("base_folder")

                    # Process model-specific relationships
                    # Handling of m2m happens in a second step (see call to self._set_many_to_many_relations),
                    many_to_many_map_ids = {}
                    fields = self._process_model_relationships(
                        model=model,
                        fields=fields,
                        link_dump_database_ids=link_dump_database_ids,
                        many_to_many_map_ids=many_to_many_map_ids,
                    )

                    # Run clean to validate unique constraints
                    try:
                        model(**fields).clean()
                    except ValidationError as e:
                        for field, error in e.error_dict.items():
                            fields[field] = f"{fields[field]} {uuid.uuid4()}"

                    # Create the object
                    logger.debug("Creating object", fields=fields)
                    objects_creation_data.append(
                        {
                            "id": obj_id,
                            "fields": fields,
                            "many_to_many_map_ids": many_to_many_map_ids,
                        }
                    )

                # Use bulk_create for models without custom save() or for
                # RequirementAssessment (whose side effects are handled
                # explicitly below). Fall back to individual create() for
                # other models with custom save() to preserve their side
                # effects without double-writing.
                is_requirement_assessment = model is RequirementAssessment
                has_save_override = model.save is not models.Model.save

                if has_save_override and not is_requirement_assessment:
                    created_objects = []
                    for object_creation_data in objects_creation_data:
                        obj_created = model.objects.create(
                            **object_creation_data["fields"]
                        )
                        created_objects.append(obj_created)
                else:
                    objects_to_create = [
                        model(**ocd["fields"]) for ocd in objects_creation_data
                    ]
                    created_objects = model.objects.bulk_create(objects_to_create)

                if is_requirement_assessment:
                    seen_cas = set()
                    for ra in created_objects:
                        ca_id = ra.compliance_assessment_id
                        if ca_id not in seen_cas:
                            seen_cas.add(ca_id)
                            ra.trigger_compliance_assessment_update_hooks()

                for obj_created, object_creation_data in zip(
                    created_objects, objects_creation_data
                ):
                    if obj_created.id is None:
                        obj_created.refresh_from_db()

                    obj_id = object_creation_data["id"]
                    link_dump_database_ids[obj_id] = obj_created.id

                    many_to_many_map_ids = object_creation_data["many_to_many_map_ids"]
                    self._set_many_to_many_relations(
                        model=model,
                        obj=obj_created,
                        many_to_many_map_ids=many_to_many_map_ids,
                    )

            except Exception as e:
                logger.error(
                    "Error creating object batch",
                    model=model._meta.model_name,
                    exc_info=True,
                )
                # This will trigger a rollback of the entire batch
                raise ValidationError(
                    f"Error creating {model._meta.model_name}: {str(e)}"
                )

    def _process_model_relationships(
        self,
        model: models.Model,
        fields: dict[str, Any],
        link_dump_database_ids: dict[str, Any],
        many_to_many_map_ids: dict[str, QuerySet | List[UUID | str] | None],
    ):
        """
        Process model-specific relationships:
         - M2M relationships are removed from `fields` and stored in a separate map to be processed later (see `_set_many_to_many_relations`).
         - Other relationships are directly converted to their database IDs or instances.

        Args:
            model: Current model for which the object is being created
            fields: The fields of the object being created.
            link_dump_database_ids: Mapping of dump ID to real database ID/Object/URN of created objects, used to resolve object relationships.
            many_to_many_map_ids: A map to store the IDs, names or QuerySet of related objects for m2m relationships.

        Returns:
            The updated fields with relationships processed (except for m2m which are removed).
        """

        def get_mapped_ids(
            ids: List[str], link_dump_database_ids: Dict[str, str]
        ) -> List[str]:
            return [
                link_dump_database_ids[id] for id in ids if id in link_dump_database_ids
            ]

        model_name = model._meta.model_name
        _fields = fields.copy()

        logger.debug(
            "Processing model relationships", model=model_name, _fields=_fields
        )

        match model_name:
            case "asset":
                many_to_many_map_ids["parent_ids"] = get_mapped_ids(
                    _fields.pop("parent_assets", []), link_dump_database_ids
                )

            case "riskassessment":
                _fields["perimeter"] = Perimeter.objects.filter(
                    id=link_dump_database_ids.get(_fields["perimeter"])
                ).first()  # filter.first to handle when no perimeter on the riskassessment (since it's optional)
                _fields["risk_matrix"] = RiskMatrix.objects.get(
                    urn=_fields.get("risk_matrix")
                )
                _fields["ebios_rm_study"] = (
                    EbiosRMStudy.objects.get(
                        id=link_dump_database_ids.get(_fields["ebios_rm_study"])
                    )
                    if _fields.get("ebios_rm_study")
                    else None
                )

            case "complianceassessment":
                _fields["perimeter"] = Perimeter.objects.get(
                    id=link_dump_database_ids.get(_fields["perimeter"])
                )
                _fields["framework"] = Framework.objects.get(urn=_fields["framework"])

            case "appliedcontrol":
                many_to_many_map_ids["evidence_ids"] = get_mapped_ids(
                    _fields.pop("evidences", []), link_dump_database_ids
                )
                many_to_many_map_ids["objective_ids"] = get_mapped_ids(
                    _fields.pop("objectives", []), link_dump_database_ids
                )
                ref_control_id = link_dump_database_ids.get(
                    _fields["reference_control"]
                )
                _fields["reference_control"] = ReferenceControl.objects.filter(
                    urn=ref_control_id
                ).first()

            case "evidence":
                many_to_many_map_ids["owner_ids"] = get_mapped_ids(
                    _fields.pop("owner", []), link_dump_database_ids
                )

            case "evidencerevision":
                _fields.pop("size", None)
                _fields.pop("attachment_hash", None)
                _fields["evidence"] = Evidence.objects.get(
                    id=link_dump_database_ids.get(_fields["evidence"])
                )

            case "requirementassessment":
                logger.debug("Looking for requirement", urn=_fields.get("requirement"))
                _fields["requirement"] = RequirementNode.objects.get(
                    urn=_fields.get("requirement")
                )
                _fields["compliance_assessment"] = ComplianceAssessment.objects.get(
                    id=link_dump_database_ids.get(_fields["compliance_assessment"])
                )
                many_to_many_map_ids.update(
                    {
                        "applied_controls": get_mapped_ids(
                            _fields.pop("applied_controls", []), link_dump_database_ids
                        ),
                        "evidence_ids": get_mapped_ids(
                            _fields.pop("evidences", []), link_dump_database_ids
                        ),
                    }
                )

            case "vulnerability":
                many_to_many_map_ids["applied_controls"] = get_mapped_ids(
                    _fields.pop("applied_controls", []), link_dump_database_ids
                )

            case "riskscenario":
                _fields["risk_assessment"] = RiskAssessment.objects.get(
                    id=link_dump_database_ids.get(_fields["risk_assessment"])
                )
                _fields["risk_origin"] = self._import_terminologies(
                    _fields.get("risk_origin"), Terminology.FieldPath.ROTO_RISK_ORIGIN
                )
                # TODO: feels complicated, may be simplified
                # Process all related _fields at once
                related__fields = [
                    "threats",
                    "vulnerabilities",
                    "assets",
                    "applied_controls",
                    "existing_applied_controls",
                    "qualifications",
                ]
                for field in related__fields:
                    map_key = (
                        f"{field.rstrip('s')}_ids"
                        if not field.endswith("controls")
                        else f"{field}_ids"
                    )
                    if field == "qualifications":
                        many_to_many_map_ids[map_key] = self._import_terminologies(
                            _fields.pop(field, []), Terminology.FieldPath.QUALIFICATIONS
                        )
                    else:
                        many_to_many_map_ids[map_key] = get_mapped_ids(
                            _fields.pop(field, []), link_dump_database_ids
                        )

            case "entity":
                _fields.pop("owned_folders", None)
                many_to_many_map_ids["relationship_ids"] = self._import_terminologies(
                    _fields.pop("relationship", []),
                    Terminology.FieldPath.ENTITY_RELATIONSHIP,
                )  # relationship_ids are in fact names of the relationships

            case "ebiosrmstudy":
                _fields.update(
                    {
                        "risk_matrix": RiskMatrix.objects.get(
                            urn=_fields.get("risk_matrix")
                        ),
                        "reference_entity": Entity.objects.get(
                            id=link_dump_database_ids.get(_fields["reference_entity"])
                        ),
                    }
                )
                many_to_many_map_ids.update(
                    {
                        "asset_ids": get_mapped_ids(
                            _fields.pop("assets", []), link_dump_database_ids
                        ),
                        "compliance_assessment_ids": get_mapped_ids(
                            _fields.pop("compliance_assessments", []),
                            link_dump_database_ids,
                        ),
                    }
                )

            case "fearedevent":
                _fields["ebios_rm_study"] = EbiosRMStudy.objects.get(
                    id=link_dump_database_ids.get(_fields["ebios_rm_study"])
                )
                many_to_many_map_ids.update(
                    {
                        "qualification_ids": self._import_terminologies(
                            _fields.pop("qualifications", []),
                            Terminology.FieldPath.QUALIFICATIONS,
                        ),
                        "asset_ids": get_mapped_ids(
                            _fields.pop("assets", []), link_dump_database_ids
                        ),
                    }
                )

            case "roto":
                _fields["ebios_rm_study"] = EbiosRMStudy.objects.get(
                    id=link_dump_database_ids.get(_fields["ebios_rm_study"])
                )
                many_to_many_map_ids["feared_event_ids"] = get_mapped_ids(
                    _fields.pop("feared_events", []), link_dump_database_ids
                )
                _fields["risk_origin"] = self._import_terminologies(
                    _fields[
                        "risk_origin"
                    ],  # risk origin must exist, it's a mandatory field on RoTo
                    Terminology.FieldPath.ROTO_RISK_ORIGIN,
                )

            case "stakeholder":
                _fields.update(
                    {
                        "ebios_rm_study": EbiosRMStudy.objects.get(
                            id=link_dump_database_ids.get(_fields["ebios_rm_study"])
                        ),
                        "entity": Entity.objects.get(
                            id=link_dump_database_ids.get(_fields["entity"])
                        ),
                    }
                )
                _fields["category"] = self._import_terminologies(
                    _fields["category"],
                    Terminology.FieldPath.ENTITY_RELATIONSHIP,  # category must exist, it's a mandatory field on Stakeholder
                )
                many_to_many_map_ids["applied_controls"] = get_mapped_ids(
                    _fields.pop("applied_controls", []), link_dump_database_ids
                )

            case "strategicscenario":
                _fields.update(
                    {
                        "ebios_rm_study": EbiosRMStudy.objects.get(
                            id=link_dump_database_ids.get(_fields["ebios_rm_study"])
                        ),
                        "ro_to_couple": RoTo.objects.get(
                            id=link_dump_database_ids.get(_fields["ro_to_couple"])
                        ),
                    }
                )

            case "attackpath":
                _fields.update(
                    {
                        "ebios_rm_study": EbiosRMStudy.objects.get(
                            id=link_dump_database_ids.get(_fields["ebios_rm_study"])
                        ),
                        "strategic_scenario": StrategicScenario.objects.get(
                            id=link_dump_database_ids.get(_fields["strategic_scenario"])
                        ),
                    }
                )
                many_to_many_map_ids["stakeholder_ids"] = get_mapped_ids(
                    _fields.pop("stakeholders", []), link_dump_database_ids
                )

            case "operationalscenario":
                _fields.update(
                    {
                        "ebios_rm_study": EbiosRMStudy.objects.get(
                            id=link_dump_database_ids.get(_fields["ebios_rm_study"])
                        ),
                        "attack_path": AttackPath.objects.get(
                            id=link_dump_database_ids.get(_fields["attack_path"])
                        ),
                    }
                )
                many_to_many_map_ids["threat_ids"] = get_mapped_ids(
                    _fields.pop("threats", []), link_dump_database_ids
                )

        return _fields

    def _set_many_to_many_relations(
        self,
        model: models.Model,
        obj: models.Model,
        many_to_many_map_ids: dict[str, QuerySet | List[UUID | str] | None],
    ) -> None:
        """
        Set many-to-many relationships after object creation.

        Args:
            model: Current model for which the object was created.
            obj: The created object for which to set the m2m relationships.
            many_to_many_map_ids: A map containing the IDs, names or QuerySet of related objects for m2m relationships.

        """
        model_name = model._meta.model_name

        match model_name:
            case "asset":
                if parent_ids := many_to_many_map_ids.get("parent_ids"):
                    logger.debug(
                        "Setting parent assets", asset=obj, parent_ids=parent_ids
                    )
                    obj.parent_assets.set(Asset.objects.filter(id__in=parent_ids))

            case "appliedcontrol":
                if evidence_ids := many_to_many_map_ids.get("evidence_ids"):
                    obj.evidences.set(Evidence.objects.filter(id__in=evidence_ids))

                if objectives_ids := many_to_many_map_ids.get("objective_ids"):
                    obj.objectives.set(
                        OrganisationObjective.objects.filter(id__in=objectives_ids)
                    )

            case "requirementassessment":
                if applied_control_ids := many_to_many_map_ids.get("applied_controls"):
                    obj.applied_controls.set(
                        AppliedControl.objects.filter(id__in=applied_control_ids)
                    )
                if evidence_ids := many_to_many_map_ids.get("evidence_ids"):
                    obj.evidences.set(Evidence.objects.filter(id__in=evidence_ids))

            case "vulnerability":
                if applied_control_ids := many_to_many_map_ids.get("applied_controls"):
                    obj.applied_controls.set(
                        AppliedControl.objects.filter(id__in=applied_control_ids)
                    )

            case "riskscenario":
                if threat_ids := many_to_many_map_ids.get("threat_ids"):
                    uuids, urns = self._split_uuids_urns(threat_ids)
                    obj.threats.set(
                        Threat.objects.filter(Q(id__in=uuids) | Q(urn__in=urns))
                    )
                if qualification_ids := many_to_many_map_ids.get("qualification_ids"):
                    obj.qualifications.set(qualification_ids)

                for field, model_class in {
                    "vulnerability_ids": (Vulnerability, "vulnerabilities"),
                    "asset_ids": (Asset, "assets"),
                    "applied_control_ids": (AppliedControl, "applied_controls"),
                    "existing_applied_control_ids": (
                        AppliedControl,
                        "existing_applied_controls",
                    ),
                }.items():
                    if ids := many_to_many_map_ids.get(field):
                        getattr(obj, model_class[1]).set(
                            model_class[0].objects.filter(id__in=ids)
                        )

            case "ebiosrmstudy":
                if asset_ids := many_to_many_map_ids.get("asset_ids"):
                    obj.assets.set(Asset.objects.filter(id__in=asset_ids))
                if compliance_assessment_ids := many_to_many_map_ids.get(
                    "compliance_assessment_ids"
                ):
                    obj.compliance_assessments.set(
                        ComplianceAssessment.objects.filter(
                            id__in=compliance_assessment_ids
                        )
                    )

            case "fearedevent":
                if qualification_ids := many_to_many_map_ids.get("qualification_ids"):
                    obj.qualifications.set(qualification_ids)

                if asset_ids := many_to_many_map_ids.get("asset_ids"):
                    obj.assets.set(Asset.objects.filter(id__in=asset_ids))

            case "roto":
                if feared_event_ids := many_to_many_map_ids.get("feared_event_ids"):
                    obj.feared_events.set(
                        FearedEvent.objects.filter(id__in=feared_event_ids)
                    )

            case "stakeholder":
                if applied_control_ids := many_to_many_map_ids.get("applied_controls"):
                    obj.applied_controls.set(
                        AppliedControl.objects.filter(id__in=applied_control_ids)
                    )

            case "attackpath":
                if stakeholder_ids := many_to_many_map_ids.get("stakeholder_ids"):
                    obj.stakeholders.set(
                        Stakeholder.objects.filter(id__in=stakeholder_ids)
                    )

            case "operationalscenario":
                if threat_ids := many_to_many_map_ids.get("threat_ids"):
                    uuids, urns = self._split_uuids_urns(threat_ids)
                    obj.threats.set(
                        Threat.objects.filter(Q(id__in=uuids) | Q(urn__in=urns))
                    )

            case "entity":
                if relationship_ids := many_to_many_map_ids.get("relationship_ids"):
                    obj.relationship.set(relationship_ids)

    def _split_uuids_urns(self, ids: List[str]) -> Tuple[List[str], List[str]]:
        """Split a list of strings into UUIDs and URNs."""
        uuids = []
        urns = []
        for item in ids:
            try:
                uuid = UUID(str(item))
                uuids.append(uuid)
            except ValueError:
                urns.append(item)
        return uuids, urns

    @action(detail=False, methods=["get"])
    def get_accessible_objects(self, request):
        """
        Return the list of folders, perimeters, frameworks and risk matrices
        the current user can access, based on their role assignments.
        """
        (viewable_folders_ids, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, Folder
        )
        (viewable_perimeters_ids, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, Perimeter
        )
        (viewable_frameworks_ids, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, Framework
        )
        (viewable_risk_matrices_ids, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, RiskMatrix
        )
        res = {
            "folders": [
                {"name": str(f), "id": f.id}
                for f in Folder.objects.filter(id__in=viewable_folders_ids).order_by(
                    Lower("name")
                )
            ],
            "perimeters": [
                {"name": str(p), "id": p.id}
                for p in Perimeter.objects.filter(
                    id__in=viewable_perimeters_ids
                ).order_by(Lower("name"))
            ],
            "frameworks": [
                {"name": f.name, "id": f.id}
                for f in Framework.objects.filter(
                    id__in=viewable_frameworks_ids
                ).order_by(Lower("name"))
            ],
            "risk_matrices": [
                {"name": rm.name, "id": rm.id}
                for rm in RiskMatrix.objects.filter(
                    id__in=viewable_risk_matrices_ids
                ).order_by(Lower("name"))
            ],
        }
        return Response(res)


class UserPreferencesView(APIView):
    permission_classes = [permissions.IsAuthenticated]

    def get(self, request) -> Response:
        prefs = request.user.get_preferences()
        return Response(prefs, status=status.HTTP_200_OK)

    def patch(self, request) -> Response:
        new_language = request.data.get("lang")
        if new_language is None or new_language not in (
            lang[0] for lang in settings.LANGUAGES
        ):
            logger.error(
                f"Error in UserPreferencesView: new_language={new_language} available languages={[lang[0] for lang in settings.LANGUAGES]}"
            )
            return Response(
                {"error": "This language doesn't exist."},
                status=status.HTTP_400_BAD_REQUEST,
            )
        prefs = request.user.get_preferences()
        prefs["lang"] = new_language
        request.user.preferences = prefs
        request.user.save(update_fields=["preferences"])
        return Response({}, status=status.HTTP_200_OK)


@cache_page(60 * SHORT_CACHE_TTL)
@vary_on_cookie
@api_view(["GET"])
@permission_classes([permissions.IsAuthenticated])
def get_counters_view(request):
    """
    API endpoint that returns the counters
    """
    return Response({"results": get_counters(request.user)})


@api_view(["GET"])
@permission_classes([permissions.IsAuthenticated])
def get_metrics_view(request):
    """
    API endpoint that returns the counters
    """
    folder_id = request.query_params.get("folder", None)
    return Response({"results": get_metrics(request.user, folder_id)})


@api_view(["GET"])
@permission_classes([permissions.IsAuthenticated])
def get_audits_metrics_view(request):
    """
    API endpoint that returns the expensive audit metrics (progress avg + audits stats).
    Split from get_metrics to allow independent streaming.
    """
    folder_id = request.query_params.get("folder", None)
    return Response({"results": get_audits_metrics(request.user, folder_id)})


@api_view(["GET"])
@permission_classes([permissions.IsAuthenticated])
def get_combined_assessments_status_view(request):
    """
    API endpoint that returns combined assessment counts per status
    for RiskAssessment, ComplianceAssessment, and FindingsAssessment
    """
    return Response({"results": combined_assessments_per_status(request.user)})


@api_view(["GET"])
@permission_classes([permissions.IsAuthenticated])
def get_governance_calendar_data_view(request):
    """
    API endpoint that returns governance activity calendar data
    Aggregates TaskNode due dates, AppliedControl ETAs, and RiskAcceptance expiry dates
    """
    year = request.query_params.get("year", None)
    if year:
        year = int(year)
    return Response({"results": get_governance_calendar_data(request.user, year)})


# TODO: Add all the proper docstrings for the following list of functions


@cache_page(60 * SHORT_CACHE_TTL)
@vary_on_cookie
@api_view(["GET"])
@permission_classes([permissions.IsAuthenticated])
def get_agg_data(request):
    viewable_risk_assessments = RoleAssignment.get_accessible_object_ids(
        Folder.get_root_folder(), request.user, RiskAssessment
    )[0]
    data = risk_status(
        request.user, RiskAssessment.objects.filter(id__in=viewable_risk_assessments)
    )

    return Response({"results": data})


def serialize_nested(data: Any) -> dict:
    if isinstance(data, (list, tuple)):
        return [serialize_nested(i) for i in data]
    elif isinstance(data, dict):
        return {key: serialize_nested(value) for key, value in data.items()}
    elif isinstance(data, set):
        return {serialize_nested(i) for i in data}
    elif isinstance(data, ReturnDict):
        return dict(data)
    elif isinstance(data, models.query.QuerySet):
        return serialize_nested(list(data))
    elif isinstance(data, RiskAssessment):
        return RiskAssessmentReadSerializer(data).data
    elif isinstance(data, RiskScenario):
        return RiskScenarioReadSerializer(data).data
    elif isinstance(data, uuid.UUID):
        return str(data)
    elif isinstance(data, Promise):
        str_attr = {attr for attr in dir(str) if not attr.startswith("_")}
        proxy_attr = {attr for attr in dir(data) if not attr.startswith("_")}
        if len(str_attr - proxy_attr) == 0:
            return str(data)
    return data


@api_view(["GET"])
@permission_classes([permissions.IsAuthenticated])
def get_composer_data(request):
    risk_assessments = request.GET.get("risk_assessment")
    if risk_assessments is None:
        return Response(
            {"error": "This endpoint requires the 'risk_assessment' query parameter"},
            status=400,
        )

    risk_assessments = risk_assessments.split(",")
    if not all(
        re.fullmatch(
            # UUID REGEX
            r"([0-9a-f]{8}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{12})",
            risk_assessment,
        )
        for risk_assessment in risk_assessments
    ):
        return Response({"error": "Invalid UUID list"}, status=400)

    data = compile_risk_assessment_for_composer(request.user, risk_assessments)
    for _data in data["risk_assessment_objects"]:
        quality_check = serialize_nested(_data["risk_assessment"].quality_check())
        _data["risk_assessment"] = RiskAssessmentReadSerializer(
            _data["risk_assessment"]
        ).data
        _data["risk_assessment"]["quality_check"] = quality_check

    data = serialize_nested(data)
    return Response({"result": data})


# Compliance Assessment


class FrameworkFilter(GenericFilterSet):
    folder = df.ModelMultipleChoiceFilter(queryset=Folder.objects.all())

    baseline = df.ModelChoiceFilter(
        queryset=ComplianceAssessment.objects.all(),
        method="filter_framework",
        label="Baseline",
    )

    def filter_framework(self, queryset, name, value):
        if not value:
            return queryset
        source_framework = value.framework
        target_framework_ids = list(
            RequirementMappingSet.objects.filter(
                source_framework=source_framework
            ).values_list("target_framework__id", flat=True)
        )
        target_framework_ids.append(source_framework.id)
        return queryset.filter(id__in=target_framework_ids)

    class Meta:
        model = Framework
        fields = ["provider"]


class FrameworkViewSet(BaseModelViewSet):
    """
    API endpoint that allows frameworks to be viewed or edited.
    """

    model = Framework
    filterset_class = FrameworkFilter
    search_fields = ["name", "description"]

    def get_queryset(self):
        qs = super().get_queryset().prefetch_related("requirement_nodes")

        # Annotate if the framework is dynamic (any question uses implementation groups)
        qs = qs.annotate(
            is_dynamic=Exists(
                RequirementNode.objects.filter(
                    framework=OuterRef("pk"),
                    questions__icontains="select_implementation_groups",
                )
            )
        )

        return qs

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @method_decorator(vary_on_cookie)
    @action(detail=False, methods=["get"])
    def names(self, request):
        uuid_list = request.query_params.getlist("id[]", [])
        queryset = Framework.objects.filter(id__in=uuid_list)

        return Response(
            {
                str(framework.id): framework.get_name_translated()
                for framework in queryset
            }
        )

    @action(detail=True, methods=["get"])
    def tree(self, request, pk):
        _framework = Framework.objects.get(id=pk)
        return Response(
            get_sorted_requirement_nodes(
                RequirementNode.objects.filter(framework=_framework).all(),
                None,
                _framework.max_score,
            )
        )

    @action(detail=False, name="Get used frameworks")
    def used(self, request):
        viewable_framework = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, Framework
        )[0]
        viewable_assessments = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, ComplianceAssessment
        )[0]
        _used_frameworks = (
            Framework.objects.filter(complianceassessment__isnull=False)
            .filter(id__in=viewable_framework)
            .filter(complianceassessment__id__in=viewable_assessments)
            .distinct()
        )
        used_frameworks = _used_frameworks.values("id", "name")
        for i in range(len(used_frameworks)):
            used_frameworks[i]["compliance_assessments_count"] = (
                ComplianceAssessment.objects.filter(framework=_used_frameworks[i].id)
                .filter(id__in=viewable_assessments)
                .count()
            )
        return Response({"results": used_frameworks})

    @action(
        detail=True, methods=["get"], name="Get framework coverage data from mappings"
    )
    def mapping_stats(self, request, pk):
        from core.mappings.engine import engine

        framework_urn = Framework.objects.filter(id=pk).values_list("urn")[0][0]
        res = engine.paths_and_coverages(framework_urn)
        return Response({"response": res})

    @action(detail=True, methods=["get"], name="Get target frameworks from mappings")
    def mappings(self, request, pk):
        framework = self.get_object()
        available_target_frameworks_objects = [framework]
        mappings = RequirementMappingSet.objects.filter(source_framework=framework)
        for mapping in mappings:
            available_target_frameworks_objects.append(mapping.target_framework)
        available_target_frameworks = FrameworkReadSerializer(
            available_target_frameworks_objects, many=True
        ).data
        return Response({"results": available_target_frameworks})

    @action(detail=False, name="Get provider choices")
    def provider(self, request):
        providers = set(
            Framework.objects.filter(provider__isnull=False).values_list(
                "provider", flat=True
            )
        )
        return Response({p: p for p in providers})

    @action(detail=True, methods=["get"], name="Framework as an Excel template")
    def excel_template(self, request, pk):
        fwk = Framework.objects.get(id=pk)
        req_nodes = RequirementNode.objects.filter(framework=fwk).order_by("urn")
        has_questions = any(rn.questions for rn in req_nodes)
        entries = []

        for rn in req_nodes:
            entry = {
                "urn": rn.urn,
                "assessable": rn.assessable,
                "ref_id": rn.ref_id,
                "name": rn.get_name_translated,
                "description": rn.get_description_translated,
                "compliance_result": "",
                "requirement_progress": "",
                "score": "",
                "observations": "",
            }

            if has_questions:
                if rn.questions:
                    lines = []
                    for q_urn, question in rn.questions.items():
                        q_text = question.get("text", "")
                        if not q_text:
                            continue
                        q_type = question.get("type")
                        if q_type == "multiple_choice":
                            choice_values = [
                                c.get("value", "") for c in question.get("choices", [])
                            ]
                            lines.append(
                                f"{q_text} (multiple) >> [{' / '.join(choice_values)}]"
                            )
                        elif q_type in ("text", "date"):
                            lines.append(f"{q_text} >> [free text]")
                        else:
                            choice_values = [
                                c.get("value", "") for c in question.get("choices", [])
                            ]
                            lines.append(f"{q_text} >> [{' / '.join(choice_values)}]")
                    entry["answers"] = "\n\n".join(lines)
                else:
                    entry["answers"] = ""

            entries.append(entry)

        # Create DataFrame from entries
        df = pd.DataFrame(entries)

        # Create BytesIO object
        buffer = io.BytesIO()

        # Create ExcelWriter with openpyxl engine
        with pd.ExcelWriter(buffer, engine="openpyxl") as writer:
            df.to_excel(writer, index=False)

            # Get the worksheet
            worksheet = writer.sheets["Sheet1"]

            wrap_columns = ["name", "description", "observations", "answers"]

            wrap_indices = [
                df.columns.get_loc(col) + 1 for col in wrap_columns if col in df.columns
            ]

            from openpyxl.styles import Alignment

            for col_idx in wrap_indices:
                for row_idx in range(2, len(df) + 2):
                    cell = worksheet.cell(row=row_idx, column=col_idx)
                    cell.alignment = Alignment(wrap_text=True)

            for idx, col in enumerate(df.columns):
                column_width = 40
                if col in wrap_columns:
                    column_width = 60
                worksheet.column_dimensions[
                    worksheet.cell(row=1, column=idx + 1).column_letter
                ].width = column_width

        # Get the value of the buffer and return as response
        buffer.seek(0)

        response = HttpResponse(
            buffer.getvalue(),
            content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
        )
        response["Content-Disposition"] = (
            f'attachment; filename="{fwk.name}_template.xlsx"'
        )

        return response


class RequirementNodeViewSet(BaseModelViewSet):
    """
    API endpoint that allows requirement groups to be viewed or edited.
    """

    model = RequirementNode
    filterset_fields = ["framework", "urn"]
    search_fields = ["name", "description"]

    def list(self, request, *args, **kwargs):
        return super().list(request, *args, **kwargs)


class RequirementViewSet(BaseModelViewSet):
    """
    API endpoint that allows requirements to be viewed or edited.
    """

    model = RequirementNode
    filterset_fields = ["framework", "urn"]
    search_fields = ["name"]

    def list(self, request, *args, **kwargs):
        return super().list(request, *args, **kwargs)

    @action(detail=True, methods=["get"], name="Inspect specific requirements")
    def inspect_requirement(self, request, pk):
        requirement = RequirementNode.objects.get(id=pk)
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            folder=Folder.get_root_folder(),
            user=request.user,
            object_type=RequirementAssessment,
        )
        requirement_assessments = RequirementAssessment.objects.filter(
            requirement=requirement, id__in=viewable_objects
        ).prefetch_related("folder", "compliance_assessment__perimeter")
        serialized_requirement_assessments = RequirementAssessmentReadSerializer(
            requirement_assessments, many=True
        ).data

        # Group by Domain and Perimeter
        grouped_data = requirement_assessments.values(
            "folder__name", "compliance_assessment__perimeter__name"
        ).annotate(
            compliant_count=Count(
                "id", filter=Q(result=RequirementAssessment.Result.COMPLIANT)
            ),
            total_count=Count("id"),
            assessed_count=Count(
                "id", filter=~Q(status=RequirementAssessment.Status.TODO)
            ),
            assessment_completion_rate=ExpressionWrapper(
                Count("id", filter=~Q(status=RequirementAssessment.Status.TODO))
                * 100.0
                / Count("id"),
                output_field=FloatField(),
            ),
        )

        # Collect all data by domain for calculations
        domain_data_collector = defaultdict(
            lambda: {
                "compliant_count": 0,
                "total_count": 0,
                "assessed_count": 0,
                "perimeter_data": [],
            }
        )

        for item in grouped_data:
            domain_name = item["folder__name"]
            domain_data_collector[domain_name]["compliant_count"] += item[
                "compliant_count"
            ]
            domain_data_collector[domain_name]["total_count"] += item["total_count"]
            domain_data_collector[domain_name]["assessed_count"] += item[
                "assessed_count"
            ]
            domain_data_collector[domain_name]["perimeter_data"].append(item)

        # Organize data by domain and perimeter using the collected data
        domain_dict = defaultdict(
            lambda: {
                "name": "",
                "compliance_result": {
                    "compliant_count": 0,
                    "total_count": 0,
                    "compliance_percentage": 0,
                },
                "assessment_progress": {
                    "assessed_count": 0,
                    "total_count": 0,
                    "assessment_completion_rate": 0,
                },
                "perimeters": [],
            }
        )

        # Structure the final response
        for domain_name, collector_data in domain_data_collector.items():
            domain_dict[domain_name]["name"] = domain_name

            # Calculate domain-level metrics
            domain_total_count = collector_data["total_count"]
            domain_compliant_count = collector_data["compliant_count"]
            domain_assessed_count = collector_data["assessed_count"]

            # Avoid division by zero
            domain_compliance_percentage = 0
            if domain_total_count > 0:
                domain_compliance_percentage = int(
                    (domain_compliant_count / domain_total_count) * 100
                )

            domain_assessment_completion_rate = 0
            if domain_total_count > 0:
                domain_assessment_completion_rate = int(
                    (domain_assessed_count / domain_total_count) * 100
                )

            # Set domain metrics
            domain_dict[domain_name]["compliance_result"] = {
                "compliant_count": domain_compliant_count,
                "total_count": domain_total_count,
                "compliance_percentage": domain_compliance_percentage,
            }

            domain_dict[domain_name]["assessment_progress"] = {
                "assessed_count": domain_assessed_count,
                "total_count": domain_total_count,
                "assessment_completion_rate": domain_assessment_completion_rate,
            }

            # Process perimeter data
            for item in collector_data["perimeter_data"]:
                perimeter_name = item["compliance_assessment__perimeter__name"]

                perimeter_entry = {
                    "name": perimeter_name,
                    "compliance_assessments": [],
                }

                # Add compliance assessments to the perimeter entry
                compliance_assessments = (
                    requirement_assessments.filter(
                        folder__name=domain_name,
                        compliance_assessment__perimeter__name=perimeter_name,
                    )
                    .select_related("compliance_assessment")
                    .values(
                        "compliance_assessment__id",
                        "compliance_assessment__name",
                        "compliance_assessment__version",
                        "compliance_assessment__show_documentation_score",
                        "compliance_assessment__max_score",
                    )
                    .distinct()
                )

                for ca in compliance_assessments:
                    perimeter_entry["compliance_assessments"].append(
                        {
                            "id": ca["compliance_assessment__id"],
                            "name": ca["compliance_assessment__name"],
                            "version": ca["compliance_assessment__version"],
                            "show_documentation_score": ca[
                                "compliance_assessment__show_documentation_score"
                            ],
                            "max_score": ca["compliance_assessment__max_score"],
                        }
                    )

                domain_dict[domain_name]["perimeters"].append(perimeter_entry)

        # Convert defaultdict to list for the response
        data_by_domain = [
            domain_data for domain_data in domain_dict.values() if domain_data["name"]
        ]

        return Response(
            {
                "requirement_assessments": serialized_requirement_assessments,
                "metrics": data_by_domain,
            }
        )


class EvidenceViewSet(BaseModelViewSet):
    """
    API endpoint that allows evidences to be viewed or edited.
    """

    model = Evidence
    filterset_fields = [
        "folder",
        "applied_controls",
        "requirement_assessments",
        "name",
        "timeline_entries",
        "filtering_labels",
        "findings",
        "findings_assessments",
        "genericcollection",
        "owner",
        "status",
        "expiry_date",
        "contracts",
        "processings",
    ]

    @action(detail=False, name="Get all evidences owners")
    def owner(self, request):
        return Response(
            ActorReadSerializer(
                Actor.objects.filter(evidences__isnull=False).distinct(),
                many=True,
            ).data
        )

    @action(methods=["get"], detail=True)
    def attachment(self, request, pk):
        (
            object_ids_view,
            _,
            _,
        ) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, Evidence
        )
        response = Response(status=status.HTTP_403_FORBIDDEN)
        if UUID(pk) in object_ids_view:
            evidence = self.get_object()
            if (
                not evidence.last_revision.attachment
                or not evidence.last_revision.attachment.storage.exists(
                    evidence.last_revision.attachment.name
                )
            ):
                return Response(status=status.HTTP_404_NOT_FOUND)
            if request.method == "GET":
                content_type = mimetypes.guess_type(evidence.last_revision.filename())[
                    0
                ]
                response = HttpResponse(
                    evidence.last_revision.attachment,
                    content_type=content_type,
                    headers={
                        "Content-Disposition": f"attachment; filename={evidence.last_revision.filename()}"
                    },
                    status=status.HTTP_200_OK,
                )
        return response

    @action(detail=False, name="Get status choices")
    def status(self, request):
        return Response(dict(Evidence.Status.choices))


class EvidenceRevisionViewSet(BaseModelViewSet):
    """
    API endpoint that allows evidence revisions to be viewed or edited.
    """

    model = EvidenceRevision
    filterset_fields = ["evidence"]
    ordering = ["-version"]

    @action(methods=["get"], detail=True)
    def attachment(self, request, pk):
        (
            object_ids_view,
            _,
            _,
        ) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, EvidenceRevision
        )
        response = Response(status=status.HTTP_403_FORBIDDEN)
        if UUID(pk) in object_ids_view:
            evidence = self.get_object()
            if not evidence.attachment or not evidence.attachment.storage.exists(
                evidence.attachment.name
            ):
                return Response(status=status.HTTP_404_NOT_FOUND)
            if request.method == "GET":
                content_type = mimetypes.guess_type(evidence.filename())[0]
                response = HttpResponse(
                    evidence.attachment,
                    content_type=content_type,
                    headers={
                        "Content-Disposition": f"attachment; filename={evidence.filename()}"
                    },
                    status=status.HTTP_200_OK,
                )
        return response

    @action(methods=["post"], detail=True)
    def delete_attachment(self, request, pk):
        (
            _,
            _,
            object_ids_delete,
        ) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, EvidenceRevision
        )
        response = Response(status=status.HTTP_403_FORBIDDEN)
        if UUID(pk) in object_ids_delete:
            evidence = self.get_object()
            if evidence.attachment:
                evidence.attachment.delete()
                response = Response(status=status.HTTP_200_OK)
        return response


class UploadAttachmentView(APIView):
    parser_classes = (FileUploadParser,)
    serializer_class = AttachmentUploadSerializer

    def post(self, request, *args, **kwargs):
        pk = kwargs["pk"]
        revision = None
        evidence = None

        # RBAC: scope to objects the user has change permission on (upload is a write)
        accessible_evidence_ids = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, Evidence
        )[1]

        try:
            revision = EvidenceRevision.objects.get(
                pk=pk, evidence__id__in=accessible_evidence_ids
            )
            evidence = revision.evidence
        except EvidenceRevision.DoesNotExist:
            try:
                evidence = Evidence.objects.get(pk=pk, id__in=accessible_evidence_ids)
            except Evidence.DoesNotExist:
                return Response(
                    {"detail": "Not found."}, status=status.HTTP_404_NOT_FOUND
                )

        if revision is None:
            revision = evidence.revisions.order_by(
                "-version"
            ).first() or EvidenceRevision.objects.create(evidence=evidence)

        attachment = request.FILES.get("file")
        if attachment and attachment.name != "undefined":
            if not revision.attachment or revision.attachment != attachment:
                old_attachment = revision.attachment
                revision.attachment = attachment
                try:
                    revision.full_clean()
                except ValidationError as e:
                    revision.attachment = old_attachment
                    messages = []
                    if hasattr(e, "message_dict"):
                        for field_messages in e.message_dict.values():
                            messages.extend(field_messages)
                    elif hasattr(e, "messages"):
                        messages = e.messages
                    else:
                        messages = [str(e.message)]
                    return Response(
                        {"detail": " ".join(messages)},
                        status=status.HTTP_400_BAD_REQUEST,
                    )
                if old_attachment:
                    old_attachment.delete()
                revision.save()

        return Response(status=status.HTTP_200_OK)


class QuickStartView(APIView):
    serializer_class = QuickStartSerializer

    def post(self, request):
        serializer = self.serializer_class(
            data=request.data, context={"request": request}
        )
        if not serializer.is_valid():
            raise ValidationError(serializer.errors)
        try:
            objects = serializer.save()
        except Exception as e:
            logger.error(f"Error in QuickStartView: {e}")
            raise
        else:
            return Response(objects, status=status.HTTP_201_CREATED)


class PresetJourneyViewSet(BaseModelViewSet):
    model = PresetJourney
    filterset_fields = ["folder"]
    search_fields = ["name", "description"]

    @action(detail=True, methods=["get"])
    def dashboard(self, request, pk=None):
        journey = self.get_object()
        from core.serializers import (
            PresetJourneyReadSerializer,
            PresetJourneyStepReadSerializer,
        )

        steps = PresetJourneyStepReadSerializer(journey.steps.all(), many=True).data
        stats = self._compute_stats(journey)
        journey_data = PresetJourneyReadSerializer(journey).data
        return Response(
            {
                "journey": journey_data,
                "steps": steps,
                "stats": stats,
            }
        )

    @action(detail=True, methods=["post"])
    def rename(self, request, pk=None):
        journey = self.get_object()
        name = request.data.get("name", "").strip()
        if not name:
            return Response(
                {"detail": "Name is required."}, status=status.HTTP_400_BAD_REQUEST
            )
        journey.name = name
        journey.save(update_fields=["name"])
        from core.serializers import PresetJourneyReadSerializer

        return Response(PresetJourneyReadSerializer(journey).data)

    @action(detail=True, methods=["post"])
    def upgrade(self, request, pk=None):
        journey = self.get_object()
        stored_lib = (
            StoredLibrary.objects.filter(urn=journey.urn).order_by("-version").first()
        )
        if not stored_lib or stored_lib.version <= journey.version:
            return Response({"detail": "Already up to date."})
        from library.preset_executor import PresetExecutor

        apply_feature_flags = RoleAssignment.is_access_allowed(
            user=request.user,
            perm=Permission.objects.get(codename="change_globalsettings"),
            folder=Folder.get_root_folder(),
        )

        executor = PresetExecutor(stored_lib, request.user, request)
        executor.upgrade_journey(journey, apply_feature_flags=apply_feature_flags)
        from core.serializers import PresetJourneyReadSerializer

        return Response(PresetJourneyReadSerializer(journey).data)

    def _compute_stats(self, journey):
        folder = journey.folder
        steps = journey.steps.all()
        total_steps = steps.count()
        completed_steps = steps.filter(
            status__in=[PresetJourneyStep.Status.DONE, PresetJourneyStep.Status.SKIPPED]
        ).count()

        stats = {
            "total_steps": total_steps,
            "completed_steps": completed_steps,
            "progress_percent": (
                round(completed_steps / total_steps * 100) if total_steps > 0 else 0
            ),
            "assets": Asset.objects.filter(folder=folder).count(),
            "risk_scenarios": RiskScenario.objects.filter(
                risk_assessment__folder=folder
            ).count(),
            "applied_controls": AppliedControl.objects.filter(folder=folder).count(),
        }

        # Compliance progress per compliance assessment in object_refs
        compliance_stats = {}
        readable_ca_qs = ComplianceAssessment.objects.all()
        for ref_name, obj_id in (journey.object_refs or {}).items():
            try:
                ca = readable_ca_qs.get(id=obj_id, folder=folder)
                total_ra = ca.requirement_assessments.count()
                assessed_ra = ca.requirement_assessments.exclude(status="to_do").count()
                compliance_stats[ref_name] = {
                    "name": ca.name,
                    "total": total_ra,
                    "assessed": assessed_ra,
                    "percent": (
                        round(assessed_ra / total_ra * 100) if total_ra > 0 else 0
                    ),
                }
            except (ComplianceAssessment.DoesNotExist, ValueError):
                continue
        stats["compliance"] = compliance_stats
        return stats


class PresetJourneyStepViewSet(BaseModelViewSet):
    model = PresetJourneyStep
    filterset_fields = ["journey", "status"]
    search_fields = ["title", "description"]


class OrganisationObjectiveViewSet(BaseModelViewSet):
    model = OrganisationObjective

    filterset_fields = [
        "folder",
        "status",
        "health",
        "issues",
        "assigned_to",
        "is_active",
    ]
    search_fields = ["name", "description"]

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get status choices")
    def status(self, request):
        return Response(dict(OrganisationObjective.Status.choices))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get health choices")
    def health(self, request):
        return Response(dict(OrganisationObjective.Health.choices))

    @action(detail=False, name="Get is_active choices")
    def is_active(self, request):
        return Response({"true": str(_("Yes")), "false": str(_("No"))})

    @action(
        detail=True,
        name="Duplicate organisation objective",
        methods=["post"],
        serializer_class=OrganisationObjectiveDuplicateSerializer,
    )
    def duplicate(self, request, pk):
        serializer = OrganisationObjectiveDuplicateSerializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        data = serializer.validated_data

        (object_ids_view, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, OrganisationObjective
        )
        if UUID(pk) not in object_ids_view:
            return Response(
                {"results": "organisation objective not found"},
                status=status.HTTP_404_NOT_FOUND,
            )

        new_folder = data["folder"]
        objective = self.get_object()
        duplicate_objective = OrganisationObjective.objects.create(
            name=data["name"],
            description=data["description"],
            folder=new_folder,
            ref_id=objective.ref_id,
            observation=objective.observation,
            status=objective.status,
            health=objective.health,
            is_active=objective.is_active,
            start_date=objective.start_date,
            eta=objective.eta,
            due_date=objective.due_date,
            closing_date=objective.closing_date,
        )
        duplicate_objective.issues.set(objective.issues.all())
        duplicate_objective.assets.set(objective.assets.all())
        duplicate_objective.tasks.set(objective.tasks.all())
        duplicate_objective.assigned_to.set(objective.assigned_to.all())
        duplicate_objective.metrics.set(objective.metrics.all())

        return Response(
            {"results": OrganisationObjectiveReadSerializer(duplicate_objective).data}
        )


class OrganisationIssueViewSet(BaseModelViewSet):
    model = OrganisationIssue

    filterset_fields = ["folder", "category", "origin", "status"]
    search_fields = ["name", "description"]

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get category choices")
    def category(self, request):
        return Response(dict(OrganisationIssue.Category.choices))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get origin choices")
    def origin(self, request):
        return Response(dict(OrganisationIssue.Origin.choices))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get status choices")
    def status(self, request):
        return Response(dict(OrganisationIssue.Status.choices))


class CampaignViewSet(BaseModelViewSet):
    model = Campaign

    filterset_fields = ["folder", "frameworks", "perimeters", "status"]
    search_fields = ["name", "description"]

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get status choices")
    def status(self, request):
        return Response(dict(Campaign.Status.choices))

    @action(detail=True, name="Get campaign metrics")
    def metrics(self, request, pk):
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, Campaign
        )
        if UUID(pk) not in viewable_objects:
            return Response(
                {"error": "Permission denied"}, status=status.HTTP_403_FORBIDDEN
            )
        campaign = self.get_object()
        return Response(campaign.metrics())

    def perform_create(self, serializer):
        super().perform_create(serializer)
        campaign = serializer.instance
        frameworks = serializer.instance.frameworks.all()
        for perimeter in campaign.perimeters.all():
            for framework in frameworks:
                framework_implementation_groups = None
                if campaign.selected_implementation_groups:
                    framework_implementation_groups = [
                        group["value"]
                        for group in campaign.selected_implementation_groups
                        if group["framework"] == str(framework.id)
                    ]
                compliance_assessment = ComplianceAssessment.objects.create(
                    name=f"{campaign.name} - {perimeter.name} - {framework.name}",
                    campaign=campaign,
                    perimeter=perimeter,
                    framework=framework,
                    folder=perimeter.folder,
                    selected_implementation_groups=framework_implementation_groups
                    if framework_implementation_groups
                    else None,
                )
                compliance_assessment.create_requirement_assessments()


class ComplianceAssessmentViewSet(BaseModelViewSet):
    """
    API endpoint that allows compliance assessments to be viewed or edited.
    """

    model = ComplianceAssessment
    filterset_fields = [
        "name",
        "ref_id",
        "folder",
        "framework",
        "perimeter",
        "campaign",
        "status",
        "ebios_rm_studies",
        "assets",
        "evidences",
        "authors",
        "reviewers",
        "genericcollection",
        "extended_result_enabled",
    ]
    search_fields = ["name", "description", "ref_id", "framework__name"]

    def get_serializer_class(self, **kwargs):
        action = kwargs.get("action", self.action)
        if action == "list":
            from core.serializers import ComplianceAssessmentListSerializer

            return ComplianceAssessmentListSerializer
        return super().get_serializer_class(**kwargs)

    def get_queryset(self):
        """Optimize queries for table view and serializer, with conditional annotations for sorting"""
        qs = (
            super()
            .get_queryset()
            .select_related(
                "folder",
                "folder__parent_folder",  # For get_folder_full_path() optimization
                "framework",  # Displayed in table
                "perimeter",  # Displayed in table
            )
        )

        if self.action == "list":
            # List view: lightweight prefetch for progress with implementation groups
            qs = qs.prefetch_related(
                Prefetch(
                    "requirement_assessments",
                    queryset=RequirementAssessment.objects.filter(
                        requirement__assessable=True
                    ).select_related("requirement"),
                ),
            )
        else:
            # Detail/other views: full prefetches for the read serializer
            qs = qs.select_related(
                "framework__library",  # For framework.has_update property
                "perimeter__folder",  # FieldsRelatedField(["id", "folder"]) optimization
                "campaign",  # Serialized as FieldsRelatedField
            ).prefetch_related(
                "assets",  # ManyToManyField serialized as FieldsRelatedField
                "evidences",  # ManyToManyField serialized as FieldsRelatedField
                "authors",  # ManyToManyField from Assessment parent class
                "reviewers",  # ManyToManyField from Assessment parent class
                "requirement_assessments",
                Prefetch(
                    "validationflow_set",
                    queryset=ValidationFlow.objects.select_related("approver"),
                ),
            )

        qs = qs.annotate(
            total_requirements=Count(
                "requirement_assessments",
                filter=Q(requirement_assessments__requirement__assessable=True),
                distinct=True,
            ),
            assessed_requirements=Count(
                "requirement_assessments",
                filter=Q(
                    Q(
                        ~Q(
                            requirement_assessments__result=RequirementAssessment.Result.NOT_ASSESSED
                        )
                    )
                    | Q(requirement_assessments__score__isnull=False),
                    requirement_assessments__requirement__assessable=True,
                ),
                distinct=True,
            ),
        )

        auditee_folders = get_auditee_filtered_folder_ids(self.request.user)
        if auditee_folders:
            user_actors = Actor.get_all_for_user(self.request.user)
            qs = qs.filter(
                ~Q(folder_id__in=auditee_folders)
                | Q(requirement_assignments__actor__in=user_actors)
            ).distinct()

        return qs

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get status choices")
    def status(self, request):
        return Response(dict(ComplianceAssessment.Status.choices))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get score calculation method choices")
    def score_calculation_method(self, request):
        return Response(dict(ComplianceAssessment.CalculationMethod.choices))

    @action(
        detail=True,
        name="Get target frameworks mapping options with compliance distribution",
    )
    def frameworks(self, request, pk):
        audit = ComplianceAssessment.objects.get(id=pk)
        from core.mappings.engine import engine

        audit_from_results = engine.load_audit_fields(audit)
        max_depth = get_mapping_max_depth()
        data = []
        for dest_urn in sorted(
            [
                p[-1]
                for p in engine.all_paths_from(
                    source_urn=audit.framework.urn, max_depth=max_depth
                )
            ]
        ):
            best_results, _ = engine.best_mapping_inferences(
                audit_from_results,
                audit.framework.urn,
                dest_urn,
                max_depth=max_depth,
            )
            if best_results:
                framework = Framework.objects.filter(urn=dest_urn).first()
                if framework and str(framework) not in str(data):
                    assessable_urns = set(
                        framework.requirement_nodes.filter(assessable=True).values_list(
                            "urn", flat=True
                        )
                    )
                    data.append(
                        {
                            "id": framework.id,
                            "str": str(framework),
                            "results": engine.summary_results(
                                best_results, filter_urns=assessable_urns
                            ),
                            "assessable_requirements_count": len(assessable_urns),
                        }
                    )
        return Response(data, status=status.HTTP_200_OK)

    @action(detail=True, name="Get compliance assessment (audit) CSV")
    def compliance_assessment_csv(self, request, pk):
        response = HttpResponse(content_type="text/csv")
        response["Content-Disposition"] = 'attachment; filename="audit_export.csv"'

        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, ComplianceAssessment
        )

        if UUID(pk) in viewable_objects:
            writer = csv.writer(response, delimiter=";")
            columns = [
                "urn",
                "ref_id",
                "name",
                "description",
                "compliance_result",
                "extended_result",
                "requirement_progress",
                "score",
                "observations",
            ]
            writer.writerow(columns)

            compliance_assessment = ComplianceAssessment.objects.get(id=pk)
            reqs = list(
                compliance_assessment.get_requirement_assessments(
                    include_non_assessable=True
                )
            )
            req_nodes = RequirementNode.objects.in_bulk(
                [ra.requirement_id for ra in reqs]
            )
            for req in reqs:
                req_node = req_nodes.get(req.requirement_id)
                row = [
                    req_node.urn,
                    req_node.ref_id,
                    req_node.get_name_translated,
                    req_node.get_description_translated,
                ]
                if req_node.assessable:
                    row += [
                        req.result,
                        req.extended_result,
                        req.status,
                        req.score,
                        req.observation,
                    ]
                else:
                    row += ["", "", "", "", ""]
                writer.writerow(row)

            return response
        else:
            return Response(
                {"error": "Permission denied"}, status=status.HTTP_403_FORBIDDEN
            )

    @action(detail=True, methods=["get"], name="Audit as an Excel")
    def xlsx(self, request, pk):
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, ComplianceAssessment
        )
        if UUID(pk) not in viewable_objects:
            return Response(
                {"error": "Permission denied"}, status=status.HTTP_403_FORBIDDEN
            )
        audit = ComplianceAssessment.objects.get(id=pk)
        entries = []
        show_documentation_score = audit.show_documentation_score
        requirement_assessments = audit.get_requirement_assessments(
            include_non_assessable=True
        )
        req_node_ids = [req.requirement.id for req in requirement_assessments]
        req_nodes = {
            node.id: node
            for node in RequirementNode.objects.filter(id__in=req_node_ids)
        }

        has_questions = any(rn.questions for rn in req_nodes.values())

        for req in requirement_assessments:
            req_node = req_nodes.get(req.requirement.id)
            if not req_node:
                continue

            entry = {
                "urn": escape_excel_formula(req_node.urn),
                "assessable": req_node.assessable,
                "ref_id": escape_excel_formula(req_node.ref_id),
                "name": escape_excel_formula(req_node.get_name_translated),
                "description": escape_excel_formula(
                    req_node.get_description_translated
                ),
                "compliance_result": req.result,
                "extended_result": req.extended_result,
                "requirement_progress": req.status,
                "observations": escape_excel_formula(req.observation),
            }
            if show_documentation_score:
                entry["implementation_score"] = req.score
                entry["documentation_score"] = req.documentation_score
            else:
                entry["score"] = req.score

            if has_questions:
                if req_node.questions:
                    answers = req.answers or {}
                    lines = []
                    for q_urn, question in req_node.questions.items():
                        q_text = question.get("text", "")
                        if not q_text:
                            continue
                        q_type = question.get("type")
                        choices_map = {
                            c["urn"]: c.get("value", "")
                            for c in question.get("choices", [])
                        }
                        answer_value = answers.get(q_urn)
                        readable = ""
                        if answer_value:
                            if q_type in ("text", "date"):
                                readable = str(answer_value)
                            elif q_type == "multiple_choice" and isinstance(
                                answer_value, list
                            ):
                                readable = " | ".join(
                                    choices_map.get(a, a) for a in answer_value
                                )
                            else:
                                readable = choices_map.get(
                                    answer_value, str(answer_value)
                                )
                        # Show choices hint when no answer
                        if not readable:
                            choice_values = list(choices_map.values())
                            if q_type == "multiple_choice":
                                lines.append(
                                    escape_excel_formula(
                                        f"{q_text} (multiple) >> [{' / '.join(choice_values)}]"
                                    )
                                )
                            elif q_type in ("text", "date"):
                                lines.append(
                                    escape_excel_formula(f"{q_text} >> [free text]")
                                )
                            else:
                                lines.append(
                                    escape_excel_formula(
                                        f"{q_text} >> [{' / '.join(choice_values)}]"
                                    )
                                )
                        else:
                            lines.append(
                                escape_excel_formula(f"{q_text} >> {readable}")
                            )
                    entry["answers"] = "\n\n".join(lines)
                else:
                    entry["answers"] = ""

            entries.append(entry)

        df = pd.DataFrame(entries)
        buffer = io.BytesIO()

        with pd.ExcelWriter(buffer, engine="openpyxl") as writer:
            df.to_excel(writer, index=False)
            worksheet = writer.sheets["Sheet1"]

            wrap_columns = ["name", "description", "observations", "answers"]
            wrap_indices = [
                df.columns.get_loc(col) + 1 for col in wrap_columns if col in df.columns
            ]

            for col_idx in wrap_indices:
                for row_idx in range(2, len(df) + 2):
                    cell = worksheet.cell(row=row_idx, column=col_idx)
                    cell.alignment = Alignment(wrap_text=True)

            for idx, col in enumerate(df.columns):
                column_width = 40
                if col in wrap_columns:
                    column_width = 60
                worksheet.column_dimensions[
                    worksheet.cell(row=1, column=idx + 1).column_letter
                ].width = column_width

        buffer.seek(0)
        response = HttpResponse(
            buffer.getvalue(),
            content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
        )
        response["Content-Disposition"] = f'attachment; filename="{audit.name}.xlsx"'

        return response

    @action(detail=True, methods=["get"])
    def word_report(self, request, pk):
        """
        Word report generation (Exec)
        """
        lang = "en"
        if request.user.preferences.get("lang") is not None:
            lang = request.user.preferences.get("lang")
            if lang not in ["fr", "en"]:
                lang = "en"

        # Check for custom Word template override
        doc = None
        try:
            custom = CustomWordTemplate.objects.filter(
                template_key="audit_report",
                language=lang,
                is_active=True,
            ).first()
            if custom and custom.file:
                custom.file.open("rb")
                doc = DocxTemplate(io.BytesIO(custom.file.read()))
        except Exception as e:
            logger.warning(
                "Failed to load custom Word template, falling back to default",
                exc_info=e,
            )

        if doc is None:
            template_path = (
                Path(__file__).resolve().parent
                / "templates"
                / "core"
                / f"audit_report_template_{lang}.docx"
            )
            doc = DocxTemplate(template_path)
        _framework = self.get_object().framework
        tree = get_sorted_requirement_nodes(
            RequirementNode.objects.filter(framework=_framework).all(),
            RequirementAssessment.objects.filter(
                compliance_assessment=self.get_object()
            ).all(),
            _framework.max_score,
        )
        implementation_groups = self.get_object().selected_implementation_groups
        filter_graph_by_implementation_groups(tree, implementation_groups)
        context = gen_audit_context(pk, doc, tree, lang)
        doc.render(context)
        buffer_doc = io.BytesIO()
        doc.save(buffer_doc)
        buffer_doc.seek(0)

        response = StreamingHttpResponse(
            FileWrapper(buffer_doc),
            content_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
        )
        response["Content-Disposition"] = "attachment; filename=exec_report.docx"

        return response

    @action(detail=True, name="Get action plan CSV")
    def action_plan_csv(self, request, pk):
        (object_ids_view, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, ComplianceAssessment
        )
        if UUID(pk) not in object_ids_view:
            return Response(
                {"error": "Permission denied"}, status=status.HTTP_403_FORBIDDEN
            )
        compliance_assessment = ComplianceAssessment.objects.get(id=pk)
        requirement_assessments = compliance_assessment.get_requirement_assessments(
            include_non_assessable=False
        )
        queryset = AppliedControl.objects.filter(
            requirement_assessments__in=requirement_assessments
        ).distinct()

        # Use the same serializer to maintain consistency - to review
        serializer = ComplianceAssessmentActionPlanSerializer(
            queryset, many=True, context={"pk": pk}
        )

        response = HttpResponse(content_type="text/csv")
        response["Content-Disposition"] = f'attachment; filename="action_plan_{pk}.csv"'

        writer = csv.writer(response)

        writer.writerow(
            [
                "Name",
                "Description",
                "Category",
                "CSF Function",
                "Priority",
                "Status",
                "ETA",
                "Expiry date",
                "Effort",
                "Impact",
                "Cost",
                "Covered requirements",
            ]
        )

        for item in serializer.data:
            writer.writerow(
                [
                    item.get("name"),
                    item.get("description"),
                    item.get("category"),
                    item.get("csf_function"),
                    item.get("priority"),
                    item.get("status"),
                    item.get("eta"),
                    item.get("expiry_date"),
                    item.get("effort"),
                    item.get("control_impact"),
                    item.get("annual_cost"),
                    "\n".join(
                        [ra.get("str") for ra in item.get("requirement_assessments")]
                    ),
                ]
            )

        return response

    @action(detail=True, name="Get action plan XLSX")
    def action_plan_xlsx(self, request, pk):
        (object_ids_view, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, ComplianceAssessment
        )
        if UUID(pk) not in object_ids_view:
            return Response(
                {"error": "Permission denied"}, status=status.HTTP_403_FORBIDDEN
            )
        compliance_assessment = ComplianceAssessment.objects.get(id=pk)
        requirement_assessments = compliance_assessment.get_requirement_assessments(
            include_non_assessable=False
        )
        queryset = AppliedControl.objects.filter(
            requirement_assessments__in=requirement_assessments
        ).distinct()

        serializer = ComplianceAssessmentActionPlanSerializer(
            queryset, many=True, context={"pk": pk}
        )

        entries = []
        for item in serializer.data:
            entry = {
                "name": item.get("name"),
                "description": item.get("description"),
                "category": item.get("category"),
                "csf_function": item.get("csf_function"),
                "priority": item.get("priority"),
                "status": item.get("status"),
                "eta": item.get("eta"),
                "expiry_date": item.get("expiry_date"),
                "effort": item.get("effort"),
                "impact": item.get("impact"),
                "cost": item.get("annual_cost"),
                "covered_requirements": "\n".join(
                    [ra.get("str") for ra in item.get("requirement_assessments")]
                ),
            }
            entries.append(entry)

        df = pd.DataFrame(entries)
        buffer = io.BytesIO()

        with pd.ExcelWriter(buffer, engine="openpyxl") as writer:
            df.to_excel(writer, index=False)
            worksheet = writer.sheets["Sheet1"]

            wrap_columns = ["name", "description", "covered_requirements"]
            wrap_indices = [
                df.columns.get_loc(col) + 1 for col in wrap_columns if col in df.columns
            ]

            for col_idx in wrap_indices:
                for row_idx in range(2, len(df) + 2):
                    cell = worksheet.cell(row=row_idx, column=col_idx)
                    cell.alignment = Alignment(wrap_text=True)

            for idx, col in enumerate(df.columns):
                column_width = 20
                if col in wrap_columns:
                    column_width = 40
                worksheet.column_dimensions[
                    worksheet.cell(row=1, column=idx + 1).column_letter
                ].width = column_width

        buffer.seek(0)
        response = HttpResponse(
            buffer.getvalue(),
            content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
        )
        response["Content-Disposition"] = (
            f'attachment; filename="action_plan_{pk}.xlsx"'
        )
        return response

    @action(detail=True, name="Get action plan PDF")
    def action_plan_pdf(self, request, pk):
        (object_ids_view, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, ComplianceAssessment
        )
        if UUID(pk) in object_ids_view:
            context = {
                "to_do": list(),
                "in_progress": list(),
                "on_hold": list(),
                "active": list(),
                "deprecated": list(),
                "--": list(),
            }
            color_map = {
                "to_do": "#FFF8F0",
                "in_progress": "#392F5A",
                "on_hold": "#F4D06F",
                "active": "#9DD9D2",
                "deprecated": "#ff8811",
                "--": "#e5e7eb",
            }
            status = AppliedControl.Status.choices
            compliance_assessment_object: ComplianceAssessment = self.get_object()
            requirement_assessments_objects = (
                compliance_assessment_object.get_requirement_assessments(
                    include_non_assessable=True
                )
            )
            applied_controls = (
                AppliedControl.objects.filter(
                    requirement_assessments__in=requirement_assessments_objects
                )
                .distinct()
                .order_by("eta")
            )
            for applied_control in applied_controls:
                context[applied_control.status].append(
                    applied_control
                ) if applied_control.status else context["--"].append(applied_control)
            data = {
                "status_text": status,
                "color_map": color_map,
                "context": context,
                "compliance_assessment": compliance_assessment_object,
            }
            html = render_to_string("core/action_plan_pdf.html", data)
            pdf_file = HTML(string=html).write_pdf()
            response = HttpResponse(pdf_file, content_type="application/pdf")
            return response
        else:
            return Response({"error": "Permission denied"})

    @action(
        detail=True,
        methods=["post"],
        name="Send compliance assessment by mail to authors",
    )
    def mailing(self, request, pk):
        instance = self.get_object()
        if settings.EMAIL_HOST or settings.EMAIL_HOST_RESCUE:
            for author in instance.authors.all():
                try:
                    specific = author.specific
                    if hasattr(specific, "mailing"):
                        specific.mailing(
                            email_template_name="tprm/third_party_email.html",
                            subject=_(
                                "CISO Assistant: A questionnaire has been assigned to you"
                            ),
                            object="compliance-assessments",
                            object_id=instance.id,
                        )
                    else:
                        logger.warning(
                            f"Actor {author} (type: {type(specific).__name__}) has no mailing method, skipping email"
                        )
                except Exception as primary_exception:
                    logger.error(
                        f"Failed to send email to {author}: {primary_exception}"
                    )
                    raise ValidationError(
                        {"error": ["An error occurred while sending the email"]}
                    )
            return Response({"results": "mail sent"})
        raise ValidationError({"warning": ["noMailerConfigured"]})

    @action(detail=True, methods=["post"])
    def update_requirement(self, request, pk):
        compliance_assessment = get_object_or_404(self.get_queryset(), pk=pk)

        viewable_objects, _, _ = RoleAssignment.get_accessible_object_ids(
            folder=Folder.get_root_folder(),
            user=request.user,
            object_type=ComplianceAssessment,
        )
        if compliance_assessment.id not in viewable_objects:
            return Response(status=status.HTTP_403_FORBIDDEN)
        try:
            ref_id = request.data.get("ref_id")
            result = request.data.get("result")
            observation = request.data.get("observation")
            score = request.data.get("score")
            status_value = request.data.get("status")

            if not all([ref_id, result]):
                return Response(
                    {"error": "ref_id and result are required fields"},
                    status=status.HTTP_400_BAD_REQUEST,
                )
            # validate if result value is valid choice
            valid_results = [
                choice[0] for choice in RequirementAssessment.Result.choices
            ]
            if result not in valid_results:
                return Response(
                    {"error": f"invalid result value. Must be one of: {valid_results}"},
                    status=status.HTTP_400_BAD_REQUEST,
                )

            # validate if status value is valid choice
            if status_value is not None:
                valid_statuses = [
                    choice[0] for choice in RequirementAssessment.Status.choices
                ]
                if status_value not in valid_statuses:
                    return Response(
                        {
                            "error": f"invalid status value. Must be one of: {valid_statuses}"
                        },
                        status=status.HTTP_400_BAD_REQUEST,
                    )

            # validate if score value is within allowed range
            if score is not None:
                try:
                    score = int(score)
                    # Only validate range if min_score and max_score are defined
                    if (
                        compliance_assessment.min_score is not None
                        and compliance_assessment.max_score is not None
                    ):
                        if (
                            score < compliance_assessment.min_score
                            or score > compliance_assessment.max_score
                        ):
                            return Response(
                                {
                                    "error": f"Score must be between {compliance_assessment.min_score} and {compliance_assessment.max_score}"
                                },
                                status=status.HTTP_400_BAD_REQUEST,
                            )
                except (ValueError, TypeError):
                    return Response(
                        {"error": "Score must be a valid integer"},
                        status=status.HTTP_400_BAD_REQUEST,
                    )

            # Find the requirement assessment to update
            requirement_assessment = RequirementAssessment.objects.filter(
                compliance_assessment=compliance_assessment, requirement__ref_id=ref_id
            ).first()

            if not requirement_assessment:
                return Response(
                    {"error": f"Requirement with ref_id {ref_id} not found"},
                    status=status.HTTP_404_NOT_FOUND,
                )

            # Update the requirement assessment
            requirement_assessment.result = result
            requirement_assessment.observation = observation

            # Update status if provided
            if status_value is not None:
                requirement_assessment.status = status_value

            # Update score and toggle is_scored accordingly
            if score is not None:
                requirement_assessment.score = score
                requirement_assessment.is_scored = True
            elif score is None and "score" in request.data:
                # Explicitly setting score to null/empty
                requirement_assessment.score = None
                requirement_assessment.is_scored = False

            requirement_assessment.save()

            response_data = {
                "message": "Requirement updated successfully",
                "ref_id": ref_id,
                "result": result,
            }

            # Include status in response if it was updated
            if status_value is not None:
                response_data["status"] = status_value

            # Include score and is_scored in response if score was updated
            if score is not None:
                response_data["score"] = score
                response_data["is_scored"] = True
            elif score is None and "score" in request.data:
                response_data["score"] = None
                response_data["is_scored"] = False

            return Response(response_data, status=status.HTTP_200_OK)

        except RequirementAssessment.DoesNotExist:
            return Response(
                {"error": f"Requirement with ref_id {ref_id} not found"},
                status=status.HTTP_404_NOT_FOUND,
            )
        except ValidationError:
            return Response(
                {"error": "invalid input provided"}, status=status.HTTP_400_BAD_REQUEST
            )
        except Exception as e:
            logger.error(f"Unexpected error in update_requirement: {str(e)}")
            return Response(
                {"error": "An unexpected error occurred"},
                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
            )

    def perform_create(self, serializer):
        """
        Create RequirementAssessment objects for the newly created ComplianceAssessment
        with optimized database operations for different scenarios.
        """
        baseline = serializer.validated_data.pop("baseline", None)
        create_applied_controls = serializer.validated_data.pop(
            "create_applied_controls_from_suggestions", False
        )
        from core.mappings.engine import engine

        with transaction.atomic():
            instance: ComplianceAssessment = serializer.save()
            instance.create_requirement_assessments(baseline)

            if baseline and baseline.framework != instance.framework:
                source_urn = baseline.framework.urn
                audit_from_results = engine.load_audit_fields(baseline)
                dest_urn = serializer.validated_data["framework"].urn
                max_depth = get_mapping_max_depth()

                best_results, _ = engine.best_mapping_inferences(
                    audit_from_results, source_urn, dest_urn, max_depth
                )

                requirement_assessments_to_update: list[RequirementAssessment] = []

                target_requirement_assessments = RequirementAssessment.objects.filter(
                    compliance_assessment=instance,
                    requirement__urn__in=best_results["requirement_assessments"],
                )

                for req in target_requirement_assessments:
                    source = best_results["requirement_assessments"][
                        req.requirement.urn
                    ]
                    for field in [
                        "result",
                        "status",
                        "observation",
                        "score",
                        "is_scored",
                        "documentation_score",
                        "mapping_inference",
                    ]:
                        value = source.get(field)
                        if value is not None:
                            setattr(req, field, value)
                    requirement_assessments_to_update.append(req)

                RequirementAssessment.objects.bulk_update(
                    requirement_assessments_to_update,
                    [
                        "result",
                        "status",
                        "observation",
                        "score",
                        "is_scored",
                        "documentation_score",
                        "mapping_inference",
                    ],
                    batch_size=500,
                )

                for ra in requirement_assessments_to_update:
                    if best_results["requirement_assessments"][ra.requirement.urn].get(
                        "applied_controls"
                    ):
                        ra.applied_controls.add(
                            *[
                                control
                                for control in best_results["requirement_assessments"][
                                    ra.requirement.urn
                                ]["applied_controls"]
                            ]
                        )
                    if best_results["requirement_assessments"][ra.requirement.urn].get(
                        "evidences"
                    ):
                        ra.evidences.add(
                            *[
                                evidence
                                for evidence in best_results["requirement_assessments"][
                                    ra.requirement.urn
                                ]["evidences"]
                            ]
                        )
                    if best_results["requirement_assessments"][ra.requirement.urn].get(
                        "security_exceptions"
                    ):
                        ra.security_exceptions.add(
                            *[
                                exception
                                for exception in best_results[
                                    "requirement_assessments"
                                ][ra.requirement.urn]["security_exceptions"]
                            ]
                        )

            # Handle applied controls creation
            if create_applied_controls:
                # Prefetch all requirement assessments with their suggestions
                assessments = instance.requirement_assessments.all().prefetch_related(
                    "requirement__reference_controls"
                )

                # Create applied controls in bulk for each assessment
                for requirement_assessment in assessments:
                    requirement_assessment.create_applied_controls_from_suggestions()

    def perform_update(self, serializer):
        compliance_assessment = serializer.save()
        if compliance_assessment.show_documentation_score:
            ra_null_documentation_score = RequirementAssessment.objects.filter(
                compliance_assessment=compliance_assessment,
                is_scored=True,
                documentation_score__isnull=True,
            )
            ra_null_documentation_score.update(
                documentation_score=compliance_assessment.min_score
            )

    @action(detail=False, name="Compliance assessments per status")
    def per_status(self, request):
        data = assessment_per_status(request.user, ComplianceAssessment)
        return Response({"results": data})

    @action(detail=False, methods=["get"])
    def quality_check(self, request):
        """
        Returns the quality check of every compliance assessment
        """
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            folder=Folder.get_root_folder(),
            user=request.user,
            object_type=ComplianceAssessment,
        )
        compliance_assessments = ComplianceAssessment.objects.filter(
            id__in=viewable_objects
        )
        res = [
            {"id": a.id, "name": a.name, "quality_check": a.quality_check()}
            for a in compliance_assessments
        ]
        return Response({"results": res})

    @action(detail=True, methods=["get"])
    def global_score(self, request, pk):
        """Returns the global score of the compliance assessment"""
        compliance_assessment = self.get_object()
        return Response(
            {
                "score": compliance_assessment.get_global_score(),
                "max_score": compliance_assessment.max_score,
                "min_score": compliance_assessment.min_score,
                "total_max_score": compliance_assessment.get_total_max_score(),
                "scores_definition": get_referential_translation(
                    compliance_assessment.framework, "scores_definition", get_language()
                ),
                "show_documentation_score": compliance_assessment.show_documentation_score,
                "score_calculation_method": compliance_assessment.score_calculation_method,
            }
        )

    @action(detail=True, methods=["get"], url_path="quality_check")
    def quality_check_detail(self, request, pk):
        """
        Returns the quality check of a specific assessment
        """
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            folder=Folder.get_root_folder(), user=request.user, object_type=Assessment
        )
        if UUID(pk) in viewable_objects:
            compliance_assessment = self.get_object()
            return Response(compliance_assessment.quality_check())
        else:
            return Response(status=status.HTTP_403_FORBIDDEN)

    @action(detail=True, methods=["get"])
    def tree(self, request, pk):
        compliance_assessment = self.get_object()
        _framework = compliance_assessment.framework
        requirement_assessments = list(
            compliance_assessment.get_requirement_assessments(
                include_non_assessable=True
            )
        )
        # Auditee filtering: scope to assigned requirements only
        auditee_folders = get_auditee_filtered_folder_ids(request.user)
        if auditee_folders and compliance_assessment.folder_id in auditee_folders:
            user_actors = Actor.get_all_for_user(request.user)
            ra_ids = set(
                RequirementAssignment.objects.filter(
                    compliance_assessment=compliance_assessment,
                    actor__in=user_actors,
                ).values_list("requirement_assessments__id", flat=True)
            )
            requirement_assessments = [
                ra for ra in requirement_assessments if ra.id in ra_ids
            ]

        requirement_nodes = list(
            RequirementNode.objects.filter(framework=_framework)
            .select_related("framework")
            .prefetch_related("reference_controls", "threats")
            .all(),
        )
        nodes_by_urn = {node.urn: node for node in requirement_nodes}
        for node in requirement_nodes:
            parent = nodes_by_urn.get(node.parent_urn)
            if parent:
                node._parent_requirement_obj = parent
        for ra in requirement_assessments:
            req = getattr(ra, "requirement", None)
            if req:
                parent = nodes_by_urn.get(req.parent_urn)
                if parent:
                    req._parent_requirement_obj = parent
        tree = get_sorted_requirement_nodes(
            requirement_nodes,
            requirement_assessments,
            _framework.max_score,
        )
        implementation_groups = compliance_assessment.selected_implementation_groups
        return Response(
            filter_graph_by_implementation_groups(tree, implementation_groups)
        )

    @action(detail=True, methods=["get"])
    def requirements_list(self, request, pk):
        """Returns the list of requirement assessments for the different audit modes"""
        assessable = str(
            self.request.query_params.get("assessable", "false")
        ).lower() in {"true", "1", "yes"}
        compliance_assessment = self.get_object()
        requirement_assessments_objects = list(
            compliance_assessment.get_requirement_assessments(
                include_non_assessable=not assessable
            )
        )
        # Auditee filtering: scope to assigned requirements only
        auditee_folders = get_auditee_filtered_folder_ids(request.user)
        if auditee_folders and compliance_assessment.folder_id in auditee_folders:
            user_actors = Actor.get_all_for_user(request.user)
            ra_ids = set(
                RequirementAssignment.objects.filter(
                    compliance_assessment=compliance_assessment,
                    actor__in=user_actors,
                ).values_list("requirement_assessments__id", flat=True)
            )
            requirement_assessments_objects = [
                ra for ra in requirement_assessments_objects if ra.id in ra_ids
            ]

        requirements_objects = list(
            RequirementNode.objects.filter(framework=compliance_assessment.framework)
            .select_related("framework")
            .prefetch_related("reference_controls", "threats")
        )
        nodes_by_urn = {node.urn: node for node in requirements_objects}
        for node in requirements_objects:
            parent = nodes_by_urn.get(node.parent_urn)
            if parent:
                node._parent_requirement_obj = parent
        for ra in requirement_assessments_objects:
            req = getattr(ra, "requirement", None)
            if req:
                parent = nodes_by_urn.get(req.parent_urn)
                if parent:
                    req._parent_requirement_obj = parent
        requirement_assessments = RequirementAssessmentReadSerializer(
            requirement_assessments_objects, many=True
        ).data
        requirements = RequirementNodeReadSerializer(
            requirements_objects, many=True
        ).data
        requirements_list = {
            "requirements": requirements,
            "requirement_assessments": requirement_assessments,
        }
        return Response(requirements_list, status=status.HTTP_200_OK)

    @action(detail=True)
    def export(self, request, pk):
        def sanitize_filename(name):
            return regex.sub(r"[^\p{L}\p{N}\p{M}\-_.]+", "_", name)

        (object_ids_view, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, ComplianceAssessment
        )
        if UUID(pk) in object_ids_view:
            compliance_assessment = self.get_object()
            (index_content, evidences) = generate_html(compliance_assessment)
            zip_name = f"{sanitize_filename(compliance_assessment.name)}-{sanitize_filename(compliance_assessment.framework.name)}-{datetime.now():%Y-%m-%d-%H-%M}.zip"

            # Create temporary file that will be automatically deleted
            temp_file = tempfile.NamedTemporaryFile(delete=True, suffix=".zip")

            try:
                with zipfile.ZipFile(temp_file, "w") as zipf:
                    for evidence in evidences:
                        if (
                            evidence.last_revision
                            and evidence.last_revision.attachment
                            and default_storage.exists(
                                evidence.last_revision.attachment.name
                            )
                        ):
                            with default_storage.open(
                                evidence.last_revision.attachment.name
                            ) as attachment_file:
                                zipf.writestr(
                                    os.path.join(
                                        "evidences",
                                        os.path.basename(
                                            evidence.last_revision.attachment.name
                                        ),
                                    ),
                                    attachment_file.read(),
                                )
                    zipf.writestr("index.html", index_content)

                # Seek to beginning for reading
                temp_file.seek(0)

                # Create response - FileResponse will handle closing the temp_file
                response = FileResponse(
                    temp_file, as_attachment=True, filename=zip_name
                )
                return response

            except Exception:
                # Clean up on error
                temp_file.close()
                raise

        else:
            return Response({"error": "Permission denied"})

    @action(detail=True, methods=["get"])
    def donut_data(self, request, pk):
        compliance_assessment = self.get_object()
        return Response(compliance_assessment.donut_render())

    @action(detail=True, methods=["get"], url_path="is-auditee")
    def is_auditee(self, request, pk):
        """Returns whether the current user is an auditee for this compliance assessment."""
        compliance_assessment = self.get_object()
        auditee_folders = get_auditee_filtered_folder_ids(request.user)
        is_auditee = bool(
            auditee_folders and compliance_assessment.folder_id in auditee_folders
        )
        return Response({"is_auditee": is_auditee})

    @action(detail=False, methods=["get"], url_path="auditee-dashboard")
    def auditee_dashboard(self, request):
        """Returns per-assignment progress data for the auditee's dashboard."""
        user_actors = Actor.get_all_for_user(request.user)
        assignments = (
            RequirementAssignment.objects.filter(actor__in=user_actors)
            .select_related(
                "compliance_assessment",
                "compliance_assessment__framework",
                "compliance_assessment__folder",
            )
            .prefetch_related("requirement_assessments", "actor")
            .distinct()
        )

        # Only include compliance assessments the user can view
        (viewable_ca_ids, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, ComplianceAssessment
        )

        dashboard_data = []
        for assignment in assignments:
            if assignment.compliance_assessment_id not in viewable_ca_ids:
                continue

            ca = assignment.compliance_assessment
            ra_ids = assignment.requirement_assessments.values_list("id", flat=True)
            ras = RequirementAssessment.objects.filter(
                id__in=ra_ids, requirement__assessable=True
            )
            total = ras.count()
            done = ras.exclude(result="not_assessed").count()

            actor_names = ", ".join(str(a) for a in assignment.actor.all())

            dashboard_data.append(
                {
                    "id": str(ca.id),
                    "assignment_id": str(assignment.id),
                    "name": ca.name,
                    "folder": ca.folder.name if ca.folder else None,
                    "framework": ca.framework.name if ca.framework else None,
                    "status": ca.status,
                    "assignment_status": assignment.status,
                    "actor": actor_names,
                    "total_requirements": total,
                    "assessed_requirements": done,
                    "progress_percent": round(done / total * 100) if total > 0 else 0,
                }
            )

        return Response(dashboard_data)

    @action(detail=True, methods=["get"])
    def comparable_audits(self, request, pk):
        """
        Get list of compliance assessments that can be compared with this one
        (same framework, user has view permission, excludes current audit)
        """
        try:
            base_audit = self.get_object()
        except ComplianceAssessment.DoesNotExist:
            return Response(
                {"error": "Base audit not found"}, status=status.HTTP_404_NOT_FOUND
            )

        # Get viewable objects for permission checking
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, ComplianceAssessment
        )

        # Filter audits: same framework, viewable, exclude current
        comparable_audits = (
            ComplianceAssessment.objects.filter(
                framework=base_audit.framework, id__in=viewable_objects
            )
            .exclude(id=UUID(pk))
            .select_related("folder", "framework", "perimeter")
            .order_by("-created_at")
        )

        # Build response with prioritization for same perimeter
        results = []
        for audit in comparable_audits:
            result = {
                "id": str(audit.id),
                "name": audit.name,
                "ref_id": audit.ref_id,
                "version": audit.version,
                "status": audit.status,
                "perimeter": {
                    "id": str(audit.perimeter.id),
                    "str": audit.perimeter.name,
                }
                if audit.perimeter
                else None,
                "folder": {"id": str(audit.folder.id), "str": audit.folder.name},
                "created_at": audit.created_at,
                # Flag for prioritization in frontend
                "same_perimeter": (
                    base_audit.perimeter
                    and audit.perimeter
                    and audit.perimeter.id == base_audit.perimeter.id
                ),
            }
            results.append(result)

        # Sort: same perimeter first, then by creation date
        results.sort(
            key=lambda x: (not x["same_perimeter"], x["created_at"]), reverse=True
        )

        return Response({"results": results})

    @action(detail=True, methods=["get"])
    def compare(self, request, pk):
        """
        Compare two compliance assessments that use the same framework
        """
        compare_id = request.query_params.get("compare_id")

        if not compare_id:
            return Response(
                {"error": "compare_id parameter is required"},
                status=status.HTTP_400_BAD_REQUEST,
            )

        # Get viewable objects for permission checking
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, ComplianceAssessment
        )

        # Check permissions for base audit
        if UUID(pk) not in viewable_objects:
            return Response(
                {"error": "Permission denied for base audit"},
                status=status.HTTP_403_FORBIDDEN,
            )

        # Check permissions for comparison audit
        if UUID(compare_id) not in viewable_objects:
            return Response(
                {"error": "Permission denied for comparison audit"},
                status=status.HTTP_403_FORBIDDEN,
            )

        try:
            base_audit = ComplianceAssessment.objects.get(id=pk)
            compare_audit = ComplianceAssessment.objects.get(id=compare_id)
        except ComplianceAssessment.DoesNotExist:
            return Response(
                {"error": "One or both audits not found"},
                status=status.HTTP_404_NOT_FOUND,
            )

        # Validate same framework
        if base_audit.framework.id != compare_audit.framework.id:
            return Response(
                {"error": "Audits must use the same framework for comparison"},
                status=status.HTTP_400_BAD_REQUEST,
            )

        # Helper function to aggregate data by top-level requirements
        def aggregate_by_top_level(audit):
            from core.helpers import get_referential_translation

            requirement_nodes = list(
                RequirementNode.objects.filter(framework=audit.framework).all()
            )
            requirement_assessments = list(
                audit.requirement_assessments.select_related("requirement").all()
            )

            # Build mapping of requirement_id to assessment
            req_assessment_map = {
                str(ra.requirement_id): ra for ra in requirement_assessments
            }

            # Build children dictionary for quick lookup
            children_dict = {}
            for rn in requirement_nodes:
                parent = rn.parent_urn or "root"
                if parent not in children_dict:
                    children_dict[parent] = []
                children_dict[parent].append(rn)

            # Get top-level nodes (no parent)
            top_level_nodes = children_dict.get("root", [])

            # Sort by order_id if available
            for node in top_level_nodes:
                if node.order_id is None:
                    node.order_id = node.created_at.timestamp()
            top_level_nodes.sort(key=lambda x: x.order_id)

            radar_data = {
                "labels": [],
                "compliance_percentages": [],
                "maturity_scores": [],
            }

            # Collect all assessable descendants recursively
            def collect_assessable_descendants(node_urn):
                assessable = []
                node_children = children_dict.get(node_urn, [])

                for child in node_children:
                    # If this child is assessable, add its assessment
                    if child.assessable:
                        ra = req_assessment_map.get(str(child.id))
                        if ra:
                            assessable.append(ra)

                    # Recursively collect from this child's descendants
                    assessable.extend(collect_assessable_descendants(child.urn))

                return assessable

            for node in top_level_nodes:
                # Try multiple ways to get a meaningful name
                node_name = (
                    get_referential_translation(node, "name")
                    or node.name
                    or node.ref_id
                    or f"Node {node.id}"
                )
                radar_data["labels"].append(node_name)

                # Check if the node itself is assessable
                assessable_list = []
                if node.assessable:
                    ra = req_assessment_map.get(str(node.id))
                    if ra:
                        assessable_list.append(ra)

                # Add all assessable descendants
                assessable_list.extend(collect_assessable_descendants(node.urn))

                # Calculate compliance percentage (compliant or partially_compliant assessments)
                if assessable_list:
                    compliant = sum(
                        1
                        for ra in assessable_list
                        if ra.result in ["compliant", "partially_compliant"]
                    )
                    compliance_percentage = (compliant / len(assessable_list)) * 100
                else:
                    compliance_percentage = 0

                # Calculate maturity score using weights and score_calculation_method
                scored_list = [
                    ra
                    for ra in assessable_list
                    if ra.is_scored and ra.result != "not_applicable"
                ]
                if scored_list:
                    weighted_score = sum(
                        (ra.score or 0) * (ra.requirement.weight or 1)
                        for ra in scored_list
                    )
                    total_weight = sum(ra.requirement.weight or 1 for ra in scored_list)
                    if (
                        audit.score_calculation_method
                        == ComplianceAssessment.CalculationMethod.SUM
                    ):
                        maturity_score = weighted_score
                    else:
                        # Default to weighted average
                        maturity_score = (
                            weighted_score / total_weight if total_weight > 0 else 0
                        )
                else:
                    maturity_score = 0

                radar_data["compliance_percentages"].append(
                    round(compliance_percentage, 1)
                )
                radar_data["maturity_scores"].append(round(maturity_score, 1))

            return radar_data

        # Build comparison data
        comparison_data = {
            "framework": {
                "id": str(base_audit.framework.id),
                "str": base_audit.framework.name,
            },
            "base": {
                "id": str(base_audit.id),
                "name": base_audit.name,
                "version": base_audit.version,
                "status": base_audit.status,
                "perimeter": {
                    "id": str(base_audit.perimeter.id),
                    "str": base_audit.perimeter.name,
                }
                if base_audit.perimeter
                else None,
                "selected_implementation_groups": base_audit.get_selected_implementation_groups(),
                "created_at": base_audit.created_at,
                "updated_at": base_audit.updated_at,
                "observation": base_audit.observation,
                "global_score": base_audit.get_global_score(),
                "max_score": base_audit.max_score,
                "total_max_score": base_audit.get_total_max_score(),
                "score_calculation_method": base_audit.score_calculation_method,
                "donut_data": base_audit.donut_render(),
                "radar_data": aggregate_by_top_level(base_audit),
            },
            "compare": {
                "id": str(compare_audit.id),
                "name": compare_audit.name,
                "version": compare_audit.version,
                "status": compare_audit.status,
                "perimeter": {
                    "id": str(compare_audit.perimeter.id),
                    "str": compare_audit.perimeter.name,
                }
                if compare_audit.perimeter
                else None,
                "selected_implementation_groups": compare_audit.get_selected_implementation_groups(),
                "created_at": compare_audit.created_at,
                "updated_at": compare_audit.updated_at,
                "observation": compare_audit.observation,
                "global_score": compare_audit.get_global_score(),
                "max_score": compare_audit.max_score,
                "total_max_score": compare_audit.get_total_max_score(),
                "score_calculation_method": compare_audit.score_calculation_method,
                "donut_data": compare_audit.donut_render(),
                "radar_data": aggregate_by_top_level(compare_audit),
            },
        }

        # Build differences list
        differences = []

        # Get all requirement assessments from both audits
        base_ras = base_audit.requirement_assessments.select_related(
            "requirement"
        ).all()
        compare_ras_dict = {
            ra.requirement_id: ra
            for ra in compare_audit.requirement_assessments.select_related(
                "requirement"
            ).all()
        }

        for base_ra in base_ras:
            compare_ra = compare_ras_dict.get(base_ra.requirement_id)

            # Skip if no matching requirement in compare audit
            if not compare_ra:
                continue

            # Check if result or score is different
            result_different = base_ra.result != compare_ra.result
            score_different = base_ra.score != compare_ra.score

            if result_different or score_different:
                # Use safe_display_str for smart fallback: name → ref_id → description → URN
                req_name = base_ra.requirement.safe_display_str

                differences.append(
                    {
                        "requirement": {
                            "id": str(base_ra.requirement.id),
                            "ref_id": base_ra.requirement.ref_id,
                            "name": req_name,
                            "description": base_ra.requirement.description,
                        },
                        "base": {
                            "id": str(base_ra.id),
                            "result": base_ra.result,
                            "status": base_ra.status,
                            "score": base_ra.score,
                        },
                        "compare": {
                            "id": str(compare_ra.id),
                            "result": compare_ra.result,
                            "status": compare_ra.status,
                            "score": compare_ra.score,
                        },
                    }
                )

        comparison_data["differences"] = differences

        return Response(comparison_data)

    @staticmethod
    @api_view(["GET", "POST"])
    @renderer_classes([JSONRenderer])
    def create_suggested_applied_controls(request, pk):
        compliance_assessment = ComplianceAssessment.objects.get(id=pk)
        dry_run = str(request.query_params.get("dry_run", "false")).lower() in {
            "true",
            "1",
            "yes",
        }
        if request.method == "GET":
            dry_run = True
        if request.method == "GET":
            dry_run = True
        if not RoleAssignment.is_access_allowed(
            user=request.user,
            perm=Permission.objects.get(codename="add_appliedcontrol"),
            folder=compliance_assessment.folder,
        ):
            return Response(status=status.HTTP_403_FORBIDDEN)
        requirement_assessments = compliance_assessment.requirement_assessments.all()
        controls = []
        for requirement_assessment in requirement_assessments:
            controls.append(
                requirement_assessment.create_applied_controls_from_suggestions(
                    dry_run=dry_run
                )
            )
        return Response(
            AppliedControlReadSerializer(chain.from_iterable(controls), many=True).data,
            status=status.HTTP_200_OK,
        )

    @action(
        detail=True,
        methods=["get", "post"],
        url_path="syncToActions",
    )
    def sync_to_applied_controls(self, request, pk):
        dry_run = request.query_params.get("dry_run", True)
        if dry_run == "false":
            dry_run = False
        compliance_assessment = ComplianceAssessment.objects.get(id=pk)

        if not RoleAssignment.is_access_allowed(
            user=request.user,
            perm=Permission.objects.get(codename="change_requirementassessment"),
            folder=compliance_assessment.folder,
        ):
            return Response(status=status.HTTP_403_FORBIDDEN)

        changes = compliance_assessment.sync_to_applied_controls(dry_run=dry_run)
        return Response({"changes": changes})

    @action(detail=True, methods=["get"], url_path="progress_ts")
    def progress_ts(self, request, pk):
        try:
            raw = (
                HistoricalMetric.objects.filter(
                    model="ComplianceAssessment", object_id=pk
                )
                .annotate(progress=F("data__reqs__progress_perc"))
                .values("date", "progress")
                .order_by("date")
            )

            # Transform the data into the required format
            formatted_data = [
                [entry["date"].isoformat(), entry["progress"]] for entry in raw
            ]

            return Response({"data": formatted_data})

        except HistoricalMetric.DoesNotExist:
            return Response(
                {"error": "No metrics found for this assessment"},
                status=status.HTTP_404_NOT_FOUND,
            )

    @action(detail=True, methods=["get"])
    def threats_metrics(self, request, pk=None):
        compliance_assessment = self.get_object()
        self.check_object_permissions(request, compliance_assessment)

        threat_metrics = compliance_assessment.get_threats_metrics()
        if threat_metrics.get("total_unique_threats") == 0:
            return Response(threat_metrics, status=status.HTTP_200_OK)
        children = []
        for th in threat_metrics["threats"]:
            children.append(
                {
                    "name": th["name"],
                    "children": [
                        ra["requirement_name"] for ra in th["requirement_assessments"]
                    ],
                    "value": len(th["requirement_assessments"]),
                }
            )
        tree = {"name": "Threats", "children": children}
        nodes = []
        for th in threat_metrics["threats"]:
            nodes.append(
                {
                    "name": th["name"],
                    "value": len(th["requirement_assessments"]),
                    "items": [
                        f"{ra['requirement_name']} ({ra['result']})"
                        for ra in th["requirement_assessments"]
                    ],
                }
            )
        threat_metrics.update({"tree": tree, "graph": {"nodes": nodes}})

        return Response(threat_metrics, status=status.HTTP_200_OK)

    @action(detail=True, methods=["get"], url_path="section_compliance")
    def section_compliance(self, request, pk):
        """Aggregates compliance results and scores per top-level requirement group."""
        compliance_assessment = self.get_object()
        framework = compliance_assessment.framework
        ig = (
            set(compliance_assessment.selected_implementation_groups)
            if compliance_assessment.selected_implementation_groups
            else None
        )

        requirement_nodes = list(
            RequirementNode.objects.filter(framework=framework).all()
        )
        # Build children dict
        children_dict = defaultdict(list)
        for rn in requirement_nodes:
            parent = rn.parent_urn or "root"
            children_dict[parent].append(rn)

        top_level_nodes = children_dict.get("root", [])
        for node in top_level_nodes:
            if node.order_id is None:
                node.order_id = 0
        top_level_nodes.sort(key=lambda x: x.order_id)

        # Build RA map
        ras = RequirementAssessment.objects.filter(
            compliance_assessment=compliance_assessment,
            requirement__assessable=True,
        ).select_related("requirement")

        # Auditee filtering
        auditee_folders = get_auditee_filtered_folder_ids(request.user)
        ra_ids = None
        if auditee_folders and compliance_assessment.folder_id in auditee_folders:
            user_actors = Actor.get_all_for_user(request.user)
            ra_ids = set(
                RequirementAssignment.objects.filter(
                    compliance_assessment=compliance_assessment,
                    actor__in=user_actors,
                ).values_list("requirement_assessments__id", flat=True)
            )

        ra_by_req_id = {}
        for ra in ras:
            if ra_ids is not None and ra.id not in ra_ids:
                continue
            ra_by_req_id[str(ra.requirement_id)] = ra

        def collect_assessable_descendants(node_urn):
            result = []
            for child in children_dict.get(node_urn, []):
                if child.assessable:
                    ra = ra_by_req_id.get(str(child.id))
                    if ra:
                        if not ig or (ig & set(child.implementation_groups or [])):
                            result.append(ra)
                result.extend(collect_assessable_descendants(child.urn))
            return result

        sections = []
        for node in top_level_nodes:
            assessable_list = []
            if node.assessable:
                ra = ra_by_req_id.get(str(node.id))
                if ra and (not ig or (ig & set(node.implementation_groups or []))):
                    assessable_list.append(ra)
            assessable_list.extend(collect_assessable_descendants(node.urn))

            if not assessable_list:
                continue

            results = defaultdict(int)
            weighted_score = 0
            total_weight = 0
            doc_weighted_score = 0
            doc_total_weight = 0
            scored_count = 0
            is_sum = (
                compliance_assessment.score_calculation_method
                == compliance_assessment.CalculationMethod.SUM
            )
            for ra in assessable_list:
                results[ra.result] += 1
                if ra.is_scored and ra.result != "not_applicable":
                    weight = ra.requirement.weight if ra.requirement.weight else 1
                    weighted_score += (ra.score or 0) * weight
                    total_weight += weight
                    scored_count += 1
                    if compliance_assessment.show_documentation_score:
                        doc_weighted_score += (ra.documentation_score or 0) * weight
                        doc_total_weight += weight

            if is_sum:
                section_score = (
                    int(weighted_score * 10) / 10 if total_weight > 0 else None
                )
                section_doc_score = (
                    int(doc_weighted_score * 10) / 10 if doc_total_weight > 0 else None
                )
            else:
                section_score = (
                    int((weighted_score / total_weight) * 10) / 10
                    if total_weight > 0
                    else None
                )
                section_doc_score = (
                    int((doc_weighted_score / doc_total_weight) * 10) / 10
                    if doc_total_weight > 0
                    else None
                )

            node_name = (
                get_referential_translation(node, "name")
                or node.name
                or node.ref_id
                or str(node.id)
            )
            sections.append(
                {
                    "ref_id": node.ref_id,
                    "name": node_name,
                    "total_assessable": len(assessable_list),
                    "results": dict(results),
                    "score": section_score,
                    "documentation_score": section_doc_score,
                    "scored_count": scored_count,
                    "total_weight": total_weight,
                }
            )

        return Response(
            {
                "sections": sections,
                "score_calculation_method": compliance_assessment.score_calculation_method,
                "max_score": compliance_assessment.max_score,
                "min_score": compliance_assessment.min_score,
            }
        )

    @action(detail=True, methods=["get"], url_path="controls_coverage")
    def controls_coverage(self, request, pk):
        """Controls coverage analysis for this compliance assessment."""
        compliance_assessment = self.get_object()
        ig = (
            set(compliance_assessment.selected_implementation_groups)
            if compliance_assessment.selected_implementation_groups
            else None
        )

        ras = RequirementAssessment.objects.filter(
            compliance_assessment=compliance_assessment,
            requirement__assessable=True,
        ).select_related("requirement")

        # Auditee filtering
        auditee_folders = get_auditee_filtered_folder_ids(request.user)
        ra_ids = None
        if auditee_folders and compliance_assessment.folder_id in auditee_folders:
            user_actors = Actor.get_all_for_user(request.user)
            ra_ids = set(
                RequirementAssignment.objects.filter(
                    compliance_assessment=compliance_assessment,
                    actor__in=user_actors,
                ).values_list("requirement_assessments__id", flat=True)
            )

        # Filter by implementation groups and auditee
        filtered_ras = []
        for ra in ras:
            if ra_ids is not None and ra.id not in ra_ids:
                continue
            if ig and not (ig & set(ra.requirement.implementation_groups or [])):
                continue
            filtered_ras.append(ra)

        # Annotate with control counts
        ra_control_counts = (
            RequirementAssessment.objects.filter(id__in=[ra.id for ra in filtered_ras])
            .annotate(control_count=Count("applied_controls"))
            .values("id", "control_count")
        )
        count_map = {str(r["id"]): r["control_count"] for r in ra_control_counts}

        with_controls = 0
        without_controls = 0
        requirements_by_control_count = defaultdict(int)
        all_control_ids = set()

        for ra in filtered_ras:
            cc = count_map.get(str(ra.id), 0)
            if cc > 0:
                with_controls += 1
            else:
                without_controls += 1
            bucket = str(cc) if cc <= 2 else "3+"
            requirements_by_control_count[bucket] += 1

        # Get all controls linked to filtered RAs
        # Note: no IAM filtering here — coverage metrics must be comprehensive
        # to avoid misleadingly low percentages. Only aggregates are exposed.
        ra_ids_list = [ra.id for ra in filtered_ras]
        control_ids = set(
            RequirementAssessment.applied_controls.through.objects.filter(
                requirementassessment_id__in=ra_ids_list
            ).values_list("appliedcontrol_id", flat=True)
        )

        control_status_distribution = defaultdict(int)
        if control_ids:
            controls = AppliedControl.objects.filter(id__in=control_ids)
            for c in controls:
                control_status_distribution[c.status] += 1

        total = len(filtered_ras)
        return Response(
            {
                "total_assessable": total,
                "with_controls": with_controls,
                "without_controls": without_controls,
                "coverage_percent": round(with_controls / total * 100, 1)
                if total > 0
                else 0,
                "control_status_distribution": dict(control_status_distribution),
                "requirements_by_control_count": dict(requirements_by_control_count),
            }
        )

    @action(detail=True, methods=["get"], url_path="evidence_coverage")
    def evidence_coverage(self, request, pk):
        """Evidence coverage analysis — direct (on RA) and indirect (via applied controls)."""
        compliance_assessment = self.get_object()
        ig = (
            set(compliance_assessment.selected_implementation_groups)
            if compliance_assessment.selected_implementation_groups
            else None
        )

        ras = RequirementAssessment.objects.filter(
            compliance_assessment=compliance_assessment,
            requirement__assessable=True,
        ).select_related("requirement")

        # Auditee filtering
        auditee_folders = get_auditee_filtered_folder_ids(request.user)
        ra_ids = None
        if auditee_folders and compliance_assessment.folder_id in auditee_folders:
            user_actors = Actor.get_all_for_user(request.user)
            ra_ids = set(
                RequirementAssignment.objects.filter(
                    compliance_assessment=compliance_assessment,
                    actor__in=user_actors,
                ).values_list("requirement_assessments__id", flat=True)
            )

        filtered_ras = []
        for ra in ras:
            if ra_ids is not None and ra.id not in ra_ids:
                continue
            if ig and not (ig & set(ra.requirement.implementation_groups or [])):
                continue
            filtered_ras.append(ra)

        ra_ids_list = [ra.id for ra in filtered_ras]

        # Note: no IAM filtering on evidence/controls — coverage metrics must be
        # comprehensive to avoid misleadingly low percentages. Only aggregates are exposed.

        # Direct evidence: RA.evidences M2M
        ra_with_direct = set(
            RequirementAssessment.evidences.through.objects.filter(
                requirementassessment_id__in=ra_ids_list
            ).values_list("requirementassessment_id", flat=True)
        )

        # Indirect evidence: RA → applied_controls → evidences
        ra_control_pairs = list(
            RequirementAssessment.applied_controls.through.objects.filter(
                requirementassessment_id__in=ra_ids_list
            ).values_list("requirementassessment_id", "appliedcontrol_id")
        )
        control_ids = set(pair[1] for pair in ra_control_pairs)
        controls_with_evidence = (
            set(
                AppliedControl.evidences.through.objects.filter(
                    appliedcontrol_id__in=control_ids
                ).values_list("appliedcontrol_id", flat=True)
            )
            if control_ids
            else set()
        )
        ra_with_indirect = set()
        for ra_id, ctrl_id in ra_control_pairs:
            if ctrl_id in controls_with_evidence:
                ra_with_indirect.add(ra_id)

        # Combine
        ra_with_any_evidence = ra_with_direct | ra_with_indirect
        with_evidence = len(ra_with_any_evidence)
        without_evidence = len(filtered_ras) - with_evidence
        direct_only = len(ra_with_direct - ra_with_indirect)
        indirect_only = len(ra_with_indirect - ra_with_direct)
        both = len(ra_with_direct & ra_with_indirect)

        # Evidence status distribution
        all_evidence_ids = set(
            RequirementAssessment.evidences.through.objects.filter(
                requirementassessment_id__in=ra_ids_list
            ).values_list("evidence_id", flat=True)
        )
        if control_ids:
            all_evidence_ids |= set(
                AppliedControl.evidences.through.objects.filter(
                    appliedcontrol_id__in=control_ids
                ).values_list("evidence_id", flat=True)
            )
        evidence_status_distribution = defaultdict(int)
        if all_evidence_ids:
            for status_val in Evidence.objects.filter(
                id__in=all_evidence_ids
            ).values_list("status", flat=True):
                evidence_status_distribution[status_val] += 1

        total = len(filtered_ras)
        return Response(
            {
                "total_assessable": total,
                "with_evidence": with_evidence,
                "without_evidence": without_evidence,
                "coverage_percent": round(with_evidence / total * 100, 1)
                if total > 0
                else 0,
                "direct_only": direct_only,
                "indirect_only": indirect_only,
                "both": both,
                "evidence_status_distribution": dict(evidence_status_distribution),
            }
        )

    @action(detail=True, methods=["get"], url_path="exceptions_summary")
    def exceptions_summary(self, request, pk):
        """Security exceptions summary for this compliance assessment."""
        compliance_assessment = self.get_object()
        ig = (
            set(compliance_assessment.selected_implementation_groups)
            if compliance_assessment.selected_implementation_groups
            else None
        )

        ras = RequirementAssessment.objects.filter(
            compliance_assessment=compliance_assessment,
            requirement__assessable=True,
        ).select_related("requirement")

        auditee_folders = get_auditee_filtered_folder_ids(request.user)
        ra_ids = None
        if auditee_folders and compliance_assessment.folder_id in auditee_folders:
            user_actors = Actor.get_all_for_user(request.user)
            ra_ids = set(
                RequirementAssignment.objects.filter(
                    compliance_assessment=compliance_assessment,
                    actor__in=user_actors,
                ).values_list("requirement_assessments__id", flat=True)
            )

        filtered_ras = []
        for ra in ras:
            if ra_ids is not None and ra.id not in ra_ids:
                continue
            if ig and not (ig & set(ra.requirement.implementation_groups or [])):
                continue
            filtered_ras.append(ra)

        ra_ids_list = [ra.id for ra in filtered_ras]
        exception_ids = set(
            RequirementAssessment.security_exceptions.through.objects.filter(
                requirementassessment_id__in=ra_ids_list
            ).values_list("securityexception_id", flat=True)
        )
        # IAM: intersect with user-visible exceptions
        (viewable_exception_ids, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, SecurityException
        )
        exception_ids &= set(viewable_exception_ids)

        if not exception_ids:
            return Response(
                {
                    "total": 0,
                    "status_distribution": {},
                    "severity_distribution": {},
                    "exceptions": [],
                }
            )

        exceptions = SecurityException.objects.filter(id__in=exception_ids)
        status_distribution = defaultdict(int)
        severity_distribution = defaultdict(int)
        exception_list = []
        for exc in exceptions:
            status_distribution[exc.status] += 1
            severity_distribution[exc.get_severity_display()] += 1
            exception_list.append(
                {
                    "id": str(exc.id),
                    "name": exc.name,
                    "status": exc.status,
                    "severity": exc.get_severity_display(),
                    "expiration_date": str(exc.expiration_date)
                    if exc.expiration_date
                    else None,
                }
            )

        return Response(
            {
                "total": len(exception_ids),
                "status_distribution": dict(status_distribution),
                "severity_distribution": dict(severity_distribution),
                "exceptions": exception_list,
            }
        )

    @action(detail=True, methods=["get"], url_path="compliance_timeline")
    def compliance_timeline(self, request, pk):
        """Returns compliance metrics over time from HistoricalMetric snapshots."""
        compliance_assessment = self.get_object()

        # Get historical data for this assessment
        metrics = HistoricalMetric.objects.filter(
            model="ComplianceAssessment", object_id=pk
        ).order_by("date")

        timeline = []
        for m in metrics:
            data = m.data.get("reqs", {})
            timeline.append(
                {
                    "date": m.date.isoformat(),
                    "progress_perc": data.get("progress_perc", 0),
                    "score": data.get("score", 0),
                    "per_result": data.get("per_result", {}),
                    "per_status": data.get("per_status", {}),
                }
            )

        return Response({"timeline": timeline})

    @action(
        detail=True,
        methods=["get"],
        url_path="implementation_groups_breakdown",
    )
    def implementation_groups_breakdown(self, request, pk):
        """Breakdown of compliance results by implementation group."""
        compliance_assessment = self.get_object()
        framework = compliance_assessment.framework

        ig_definition = framework.implementation_groups_definition
        if not ig_definition:
            return Response({"groups": []})

        ras = RequirementAssessment.objects.filter(
            compliance_assessment=compliance_assessment,
            requirement__assessable=True,
        ).select_related("requirement")

        # Auditee filtering
        auditee_folders = get_auditee_filtered_folder_ids(request.user)
        ra_ids = None
        if auditee_folders and compliance_assessment.folder_id in auditee_folders:
            user_actors = Actor.get_all_for_user(request.user)
            ra_ids = set(
                RequirementAssignment.objects.filter(
                    compliance_assessment=compliance_assessment,
                    actor__in=user_actors,
                ).values_list("requirement_assessments__id", flat=True)
            )

        # Build filtered RA list
        filtered_ras = []
        for ra in ras:
            if ra_ids is not None and ra.id not in ra_ids:
                continue
            filtered_ras.append(ra)

        groups = []
        for group_def in ig_definition:
            group_ref_id = group_def.get("ref_id")
            group_name = group_def.get("name", group_ref_id)

            matching_ras = [
                ra
                for ra in filtered_ras
                if group_ref_id in (ra.requirement.implementation_groups or [])
            ]

            if not matching_ras:
                groups.append(
                    {
                        "ref_id": group_ref_id,
                        "name": group_name,
                        "total_assessable": 0,
                        "results": {},
                        "progress_percent": 0,
                        "score": None,
                        "documentation_score": None,
                        "scored_count": 0,
                    }
                )
                continue

            results = defaultdict(int)
            assessed = 0
            weighted_score = 0
            total_weight = 0
            doc_weighted_score = 0
            doc_total_weight = 0
            scored_count = 0
            is_sum = (
                compliance_assessment.score_calculation_method
                == compliance_assessment.CalculationMethod.SUM
            )

            for ra in matching_ras:
                results[ra.result] += 1
                if ra.result != "not_assessed":
                    assessed += 1
                if ra.is_scored and ra.result != "not_applicable":
                    weight = ra.requirement.weight if ra.requirement.weight else 1
                    weighted_score += (ra.score or 0) * weight
                    total_weight += weight
                    scored_count += 1
                    if compliance_assessment.show_documentation_score:
                        doc_weighted_score += (ra.documentation_score or 0) * weight
                        doc_total_weight += weight

            total = len(matching_ras)
            if is_sum:
                group_score = (
                    int(weighted_score * 10) / 10 if total_weight > 0 else None
                )
                group_doc_score = (
                    int(doc_weighted_score * 10) / 10 if doc_total_weight > 0 else None
                )
            else:
                group_score = (
                    int((weighted_score / total_weight) * 10) / 10
                    if total_weight > 0
                    else None
                )
                group_doc_score = (
                    int((doc_weighted_score / doc_total_weight) * 10) / 10
                    if doc_total_weight > 0
                    else None
                )

            groups.append(
                {
                    "ref_id": group_ref_id,
                    "name": group_name,
                    "total_assessable": total,
                    "results": dict(results),
                    "progress_percent": round(assessed / total * 100)
                    if total > 0
                    else 0,
                    "score": group_score,
                    "documentation_score": group_doc_score,
                    "scored_count": scored_count,
                }
            )

        return Response(
            {
                "groups": groups,
                "score_calculation_method": compliance_assessment.score_calculation_method,
                "max_score": compliance_assessment.max_score,
                "min_score": compliance_assessment.min_score,
            }
        )

    @action(detail=False, methods=["get"], name="Get compliance analytics")
    def analytics(self, request):
        """
        Returns compliance analytics data grouped by framework and domain
        """
        folder_id = request.query_params.get("folder", None)
        analytics_data = get_compliance_analytics(request.user, folder_id)
        return Response(analytics_data, status=status.HTTP_200_OK)


class RequirementAssessmentViewSet(BaseModelViewSet):
    """
    API endpoint that allows requirement assessments to be viewed or edited.
    """

    model = RequirementAssessment
    filterset_fields = [
        "folder",
        "folder__name",
        "evidences",
        "compliance_assessment",
        "applied_controls",
        "security_exceptions",
        "requirement__ref_id",
        "result",
        "extended_result",
        "compliance_assessment__ref_id",
        "compliance_assessment__perimeter",
        "compliance_assessment__perimeter__name",
        "compliance_assessment__assets__ref_id",
        "requirement__assessable",
    ]
    search_fields = [
        "requirement__name",
        "requirement__description",
        "requirement__ref_id",
    ]

    def get_queryset(self):
        """Optimize queries for table view and serializer - high-impact due to many nested relationships"""
        qs = (
            super()
            .get_queryset()
            .select_related(
                "folder",
                "folder__parent_folder",  # For get_folder_full_path() optimization
                "compliance_assessment",  # Displayed in table and serialized
                "compliance_assessment__perimeter",  # perimeter field uses compliance_assessment.perimeter
                "compliance_assessment__perimeter__folder",  # Nested FieldsRelatedField optimization
                "requirement",  # Used for name (__str__), description, assessable in table
            )
            .prefetch_related(
                "evidences",  # ManyToManyField serialized as FieldsRelatedField
                "applied_controls",  # ManyToManyField to AppliedControl
                "security_exceptions",  # ManyToManyField serialized as FieldsRelatedField
            )
        )
        auditee_folders = get_auditee_filtered_folder_ids(self.request.user)
        if auditee_folders:
            user_actors = Actor.get_all_for_user(self.request.user)
            qs = qs.filter(
                ~Q(folder_id__in=auditee_folders)
                | Q(assignments__actor__in=user_actors)
            ).distinct()
        return qs

    def update(self, request, *args, **kwargs):
        response = super().update(request, *args, **kwargs)
        cache.clear()
        return response

    @action(detail=False, name="Get updatable measures")
    def updatables(self, request):
        (_, object_ids_change, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, AppliedControl
        )

        return Response({"results": object_ids_change})

    @action(
        detail=False, name="Something"
    )  # Write a good name for the "name" keyword argument
    def per_status(self, request):
        data = applied_control_per_status(request.user)
        return Response({"results": data})

    @action(detail=False, name="Get the ordered todo applied controls")
    def todo(self, request):
        (object_ids_view, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, AppliedControl
        )

        measures = sorted(
            AppliedControl.objects.filter(id__in=object_ids_view)
            .exclude(status="done")
            .order_by("eta"),
            key=lambda mtg: mtg.get_ranking_score(),
            reverse=True,
        )

        """measures = [{
            key: getattr(mtg,key)
            for key in [
                "id","folder","reference_control","type","status","effort","cost","name","description","eta","link","created_at","updated_at"
            ]
        } for mtg in measures]
        for i in range(len(measures)) :
            measures[i]["id"] = str(measures[i]["id"])
            measures[i]["folder"] = str(measures[i]["folder"].name)
            for key in ["created_at","updated_at","eta"] :
                measures[i][key] = str(measures[i][key])"""

        ranking_scores = {str(mtg.id): mtg.get_ranking_score() for mtg in measures}

        measures = [AppliedControlReadSerializer(mtg).data for mtg in measures]

        for i in range(len(measures)):
            measures[i]["ranking_score"] = ranking_scores[measures[i]["id"]]

        """
        The serializer of AppliedControl isn't applied automatically for this function
        """

        return Response({"results": measures})

    @action(detail=False, name="Get the secuity measures to review")
    def to_review(self, request):
        measures = measures_to_review(request.user)
        measures = [AppliedControlReadSerializer(mtg).data for mtg in measures]

        """
        The serializer of AppliedControl isn't applied automatically for this function
        """

        return Response({"results": measures})

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get status choices")
    def status(self, request):
        return Response(dict(RequirementAssessment.Status.choices))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get result choices")
    def result(self, request):
        return Response(dict(RequirementAssessment.Result.choices))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get extended result choices")
    def extended_result(self, request):
        return Response(dict(RequirementAssessment.ExtendedResult.choices))

    @staticmethod
    @api_view(["GET", "POST"])
    @renderer_classes([JSONRenderer])
    def create_suggested_applied_controls(request, pk):
        requirement_assessment = RequirementAssessment.objects.get(id=pk)
        dry_run = str(request.query_params.get("dry_run", "false")).lower() in {
            "true",
            "1",
            "yes",
        }
        if not RoleAssignment.is_access_allowed(
            user=request.user,
            perm=Permission.objects.get(codename="add_appliedcontrol"),
            folder=requirement_assessment.folder,
        ):
            return Response(status=status.HTTP_403_FORBIDDEN)
        controls = requirement_assessment.create_applied_controls_from_suggestions(
            dry_run=dry_run
        )
        return Response(
            AppliedControlReadSerializer(controls, many=True).data,
            status=status.HTTP_200_OK,
        )


class RequirementMappingSetViewSet(BaseModelViewSet):
    model = StoredLibrary

    filterset_fields = [
        "provider",
    ]

    def get_serializer_class(self, **kwargs):
        return RequirementMappingSetReadSerializer

    def get_queryset(self):
        return (
            super()
            .get_queryset()
            .filter(
                Q(content__requirement_mapping_set__isnull=False)
                | Q(content__requirement_mapping_sets__isnull=False)
            )
        )

    @action(detail=False, name="Get provider choices")
    def provider(self, request):
        providers = set(
            StoredLibrary.objects.filter(
                provider__isnull=False,
                objects_meta__requirement_mapping_sets__isnull=False,
            ).values_list("provider", flat=True)
        )
        return Response({p: p for p in providers})

    @action(detail=False, methods=["get"], url_path="graph-data")
    def graph_data_list(self, request):
        from core.mappings.engine import engine

        max_depth = get_mapping_max_depth()

        all_paths = engine.get_mapping_graph(max_depth=max_depth)

        all_framework_urns = set()
        for path in all_paths:
            all_framework_urns.update(path)

        framework_details = {}

        framework_libs_qs = StoredLibrary.objects.filter(
            content__framework__urn__in=all_framework_urns,
            content__framework__isnull=False,
            content__requirement_mapping_set__isnull=True,
            content__requirement_mapping_sets__isnull=True,
        )

        for lib in framework_libs_qs:
            try:
                fw_content = lib.content["framework"]
                urn = fw_content.get("urn")

                if not urn:
                    continue

                display_name = fw_content.get("name", urn)

                display_value = fw_content.get("description", display_name)

                framework_details[urn] = {
                    "name": display_name,
                    "value": display_value,
                    "pk": lib.pk,
                }
            except (KeyError, TypeError):
                # Skip this library if content is malformed
                continue

        nodes = []
        urn_to_index = {}  # Map URNs to their index in the `nodes` list

        # Sort for a consistent node order
        sorted_urns = sorted(list(all_framework_urns))

        for i, urn in enumerate(sorted_urns):
            urn_to_index[urn] = i

            details = framework_details.get(urn, {"name": urn, "value": urn})

            nodes.append(
                {
                    "name": details["name"],  # Display name for the node
                    "category": 0
                    if urn in engine.frameworks.keys()
                    else 1,  # All nodes are in the "Frameworks" category
                    "value": details["value"],  # Tooltip content
                    "urn": urn,  # Pass URN as extra data
                    "pk": details.get("pk"),
                    # `symbolSize` will be added below after degrees are calculated
                }
            )

        links = []
        link_set = set()  # Use a set to prevent duplicate links

        node_degrees = [0] * len(nodes)

        for path in all_paths:
            # A path [A, B, C] creates links (A, B) and (B, C)
            for i in range(len(path) - 1):
                source_urn = path[i]
                target_urn = path[i + 1]
                link_tuple = (source_urn, target_urn)

                if link_tuple not in link_set:
                    link_set.add(link_tuple)

                    # Get the integer index for source and target
                    source_index = urn_to_index.get(source_urn)
                    target_index = urn_to_index.get(target_urn)

                    if source_index is not None and target_index is not None:
                        links.append(
                            {
                                "source": source_index,
                                "target": target_index,
                                "value": "maps to",
                            }
                        )

                        node_degrees[source_index] += 1
                        node_degrees[target_index] += 1

        min_size = 10
        max_size = 30

        non_zero_degrees = [d for d in node_degrees if d > 0]

        if not non_zero_degrees:
            min_degree = 0
            max_degree = 0
        else:
            min_degree = min(non_zero_degrees)
            max_degree = max(node_degrees)

        degree_range = max_degree - min_degree
        size_range = max_size - min_size

        for i, node in enumerate(nodes):
            degree = node_degrees[i]

            if degree == 0:
                # Assign min size for nodes with no connections (if any)
                node["symbolSize"] = min_size
            elif degree_range == 0:
                # All nodes have the same degree, assign min_size
                node["symbolSize"] = min_size
            else:
                # Scale the size proportionally
                normalized_degree = (degree - min_degree) / degree_range
                scaled_size = min_size + (normalized_degree * size_range)
                node["symbolSize"] = int(round(scaled_size))

        categories = [{"name": "importedFrameworks"}, {"name": "notImportedFrameworks"}]

        return Response({"nodes": nodes, "links": links, "categories": categories})

    @action(detail=True, methods=["get"], url_path="graph_data")
    def graph_data(self, request, pk=None):
        obj = StoredLibrary.objects.get(id=pk)

        mapping_set = obj.content.get(
            "requirement_mapping_sets",
            [obj.content.get("requirement_mapping_set", {})],
        )[0]

        source_framework_lib = StoredLibrary.objects.filter(
            content__framework__urn=mapping_set["source_framework_urn"],
            content__framework__isnull=False,
            content__requirement_mapping_set__isnull=True,
            content__requirement_mapping_sets__isnull=True,
        ).first()
        if not source_framework_lib:
            raise NotFound("Source framework library not found")

        target_framework_lib = StoredLibrary.objects.filter(
            content__framework__urn=mapping_set["target_framework_urn"],
            content__framework__isnull=False,
            content__requirement_mapping_set__isnull=True,
            content__requirement_mapping_sets__isnull=True,
        ).first()
        if not target_framework_lib:
            raise NotFound("Target framework library not found")

        source_framework = source_framework_lib.content["framework"]
        target_framework = target_framework_lib.content["framework"]

        source_nodes_dict = {
            n.get("urn"): n for n in source_framework["requirement_nodes"]
        }
        target_nodes_dict = {
            n.get("urn"): n for n in target_framework["requirement_nodes"]
        }

        nodes = []
        links = []
        snodes_idx = dict()
        tnodes_idx = dict()
        categories = [
            {
                "name": source_framework["name"],
            },
            {
                "name": target_framework["name"],
            },
        ]
        N = 0
        for req in source_framework["requirement_nodes"]:
            if not req.get("assessable", False):
                continue
            nodes.append(
                {
                    "name": req.get("ref_id"),
                    "category": 0,
                    "value": req.get("name", req.get("description")),
                }
            )
            snodes_idx[req.get("urn")] = N
            N += 1

        for req in target_framework["requirement_nodes"]:
            if not req.get("assessable", False):
                continue
            nodes.append(
                {
                    "name": req.get("ref_id"),
                    "category": 1,
                    "value": req.get("name", req.get("description")),
                }
            )
            tnodes_idx[req.get("urn")] = N
            N += 1
        req_mappings = mapping_set.get("requirement_mappings", [])
        for mapping in req_mappings:
            source_urn = mapping.get("source_requirement_urn")
            target_urn = mapping.get("target_requirement_urn")
            source_node = source_nodes_dict.get(source_urn)
            target_node = target_nodes_dict.get(target_urn)

            if (
                mapping.get("source_requirement_urn") not in snodes_idx
                or mapping.get("target_requirement_urn") not in tnodes_idx
                or not source_node
                or not target_node
            ):
                continue

            links.append(
                {
                    "source": snodes_idx[source_node.get("urn")],
                    "target": tnodes_idx[target_node.get("urn")],
                    "value": mapping.get("relationship"),
                }
            )

        # Calculate coverage in both directions
        source_assessable_count = len(snodes_idx)
        target_assessable_count = len(tnodes_idx)

        linked_source_urns = set(
            mapping.get("source_requirement_urn")
            for mapping in req_mappings
            if mapping.get("source_requirement_urn") in snodes_idx
        )
        linked_target_urns = set(
            mapping.get("target_requirement_urn")
            for mapping in req_mappings
            if mapping.get("target_requirement_urn") in tnodes_idx
        )

        source_coverage = (
            round(len(linked_source_urns) / source_assessable_count * 100)
            if source_assessable_count > 0
            else 0
        )
        target_coverage = (
            round(len(linked_target_urns) / target_assessable_count * 100)
            if target_assessable_count > 0
            else 0
        )

        meta = {
            "display_name": f"{source_framework['name']} ➜ {target_framework['name']}",
            "source_framework": source_framework["name"],
            "target_framework": target_framework["name"],
            "source_coverage": source_coverage,
            "target_coverage": target_coverage,
            "source_total": source_assessable_count,
            "source_linked": len(linked_source_urns),
            "target_total": target_assessable_count,
            "target_linked": len(linked_target_urns),
        }

        return Response(
            {"nodes": nodes, "links": links, "categories": categories, "meta": meta}
        )


@api_view(["GET"])
@permission_classes([permissions.AllowAny])
def get_csrf_token(request):
    """
    API endpoint that returns the CSRF token.
    """
    return Response({"csrfToken": csrf.get_token(request)})


def get_disk_usage():
    try:
        path = Path(settings.BASE_DIR) / "db"
        usage = shutil.disk_usage(path)
        return usage
    except PermissionError:
        logger.error(
            "Permission issue: cannot access the path to retrieve the disk_usage info"
        )
        return None
    except FileNotFoundError:
        logger.error(
            "Path issue: cannot access the path to retrieve the disk_usage info"
        )
        return None


@api_view(["GET"])
@permission_classes([permissions.AllowAny])
def healthcheck(request):
    return Response({"status": "ok"})


@api_view(["GET"])
def get_build(request):
    """
    API endpoint that returns the build version of the application.
    """
    BUILD = settings.BUILD
    VERSION = settings.VERSION
    default_db_engine = settings.DATABASES["default"]["ENGINE"]
    if "postgresql" in default_db_engine:
        database_type = "P-FS"
    elif "sqlite" in default_db_engine:
        database_type = "S-FS"
    else:
        database_type = "Unknown"

    disk_info = get_disk_usage()

    if disk_info:
        total, used, free = disk_info
        disk_response = {
            "diskSpace": f"{humanize.naturalsize(total)}",
            "diskUsed": f"{humanize.naturalsize(used)} ({int((used / total) * 100)} %)",
        }
    else:
        disk_response = {
            "diskSpace": "Unable to retrieve disk usage",
        }

    return Response(
        {
            "version": VERSION,
            "build": BUILD,
            "infrastructure": database_type,
            **disk_response,
        }
    )


# NOTE: Important functions/classes from old views.py, to be reviewed


def generate_html(
    compliance_assessment: ComplianceAssessment,
) -> Tuple[str, list[Evidence]]:
    selected_evidences = []

    requirement_nodes = RequirementNode.objects.filter(
        framework=compliance_assessment.framework
    )

    assessments = RequirementAssessment.objects.filter(
        compliance_assessment=compliance_assessment,
    ).all()

    implementation_groups = compliance_assessment.selected_implementation_groups
    graph = get_sorted_requirement_nodes(
        list(requirement_nodes),
        list(assessments),
        compliance_assessment.framework.max_score,
    )
    graph = filter_graph_by_implementation_groups(graph, implementation_groups)
    flattened_graph = flatten_dict(graph)

    requirement_nodes = requirement_nodes.filter(urn__in=flattened_graph.values())
    assessments = assessments.filter(requirement__urn__in=flattened_graph.values())

    node_per_urn = {r.urn: r for r in requirement_nodes}
    ancestors = {}
    for a in assessments:
        ancestors[a] = set()
        req = a.requirement
        while req:
            ancestors[a].add(req)
            p = req.parent_urn
            req = None if not (p) else node_per_urn[p]

    def generate_data_rec(requirement_node: RequirementNode):
        selected_evidences = []
        children_nodes = [
            req for req in requirement_nodes if req.parent_urn == requirement_node.urn
        ]

        node_data = {
            "requirement_node": requirement_node,
            "children": [],
            "assessments": None,
            "bar_graph": None,
            "direct_evidences": [],
            "applied_controls": [],
            "result": "",
            "status": "",
            "color_class": "",
        }

        node_data["bar_graph"] = True if children_nodes else False

        if requirement_node.assessable:
            assessment = RequirementAssessment.objects.filter(
                requirement__urn=requirement_node.urn,
                compliance_assessment=compliance_assessment,
            ).first()

            if assessment:
                node_data["assessments"] = assessment
                node_data["result"] = assessment.get_result_display()
                node_data["status"] = assessment.get_status_display()
                node_data["result_color_class"] = color_css_class(assessment.result)
                node_data["status_color_class"] = color_css_class(assessment.status)
                direct_evidences = assessment.evidences.all()
                if direct_evidences:
                    selected_evidences += direct_evidences
                    node_data["direct_evidences"] = direct_evidences

                measures = assessment.applied_controls.all()
                if measures:
                    applied_controls = []
                    for measure in measures:
                        evidences = measure.evidences.all()
                        applied_controls.append(
                            {
                                "measure": measure,
                                "evidences": evidences,
                            }
                        )
                        selected_evidences += evidences
                    node_data["applied_controls"] = applied_controls

        for child_node in children_nodes:
            child_data, child_evidences = generate_data_rec(child_node)
            node_data["children"].append(child_data)
            selected_evidences += child_evidences

        return node_data, selected_evidences

    top_level_nodes = [req for req in requirement_nodes if not req.parent_urn]
    update_translations_in_object(top_level_nodes)
    top_level_nodes_data = []
    for requirement_node in top_level_nodes:
        node_data, node_evidences = generate_data_rec(requirement_node)
        top_level_nodes_data.append(node_data)
        selected_evidences += node_evidences

    data = {
        "compliance_assessment": compliance_assessment,
        "top_level_nodes": top_level_nodes_data,
        "assessments": assessments,
        "ancestors": ancestors,
    }

    return render_to_string("core/audit_report.html", data), list(
        set(selected_evidences)
    )


def export_mp_csv(request):
    response = HttpResponse(content_type="text/csv")
    response["Content-Disposition"] = 'attachment; filename="MP.csv"'

    writer = csv.writer(response, delimiter=";")
    columns = [
        "measure_id",
        "measure_name",
        "measure_desc",
        "category",
        "csf_function",
        "reference_control",
        "eta",
        "priority",
        "effort",
        "control_impact",
        "annual_cost",
        "link",
        "status",
    ]
    writer.writerow(columns)
    (
        object_ids_view,
        object_ids_change,
        object_ids_delete,
    ) = RoleAssignment.get_accessible_object_ids(
        Folder.get_root_folder(), request.user, AppliedControl
    )
    for mtg in AppliedControl.objects.filter(id__in=object_ids_view):
        row = [
            mtg.id,
            mtg.name,
            mtg.description,
            mtg.category,
            mtg.csf_function,
            mtg.priority,
            mtg.reference_control,
            mtg.eta,
            mtg.effort,
            mtg.control_impact,
            mtg.annual_cost,
            mtg.link,
            mtg.status,
        ]
        writer.writerow(row)

    return response


class SecurityExceptionViewSet(ExportMixin, BaseModelViewSet):
    """
    API endpoint that allows security exceptions to be viewed or edited.
    """

    model = SecurityException
    filterset_fields = [
        "name",
        "requirement_assessments",
        "risk_scenarios",
        "owners",
        "approver",
        "folder",
        "severity",
        "status",
        "genericcollection",
    ]
    search_fields = ["name", "description", "ref_id"]

    export_config = {
        "fields": {
            "internal_id": {"source": "id", "label": "internal_id"},
            "ref_id": {"source": "ref_id", "label": "ref_id", "escape": True},
            "name": {"source": "name", "label": "name", "escape": True},
            "description": {
                "source": "description",
                "label": "description",
                "escape": True,
            },
            "severity": {"source": "get_severity_display", "label": "severity"},
            "status": {"source": "get_status_display", "label": "status"},
            "expiration_date": {
                "source": "expiration_date",
                "label": "expiration_date",
            },
            "owners": {
                "source": "owners",
                "label": "owners",
                "format": lambda qs: ",".join(
                    escape_excel_formula(str(o)) for o in qs.all()
                ),
            },
            "approver": {
                "source": "approver.email",
                "label": "approver",
                "escape": True,
            },
            "folder": {
                "source": "folder.name",
                "label": "folder",
                "escape": True,
            },
        },
        "filename": "security_exceptions_export",
        "select_related": ["folder", "approver"],
        "prefetch_related": ["owners"],
    }

    @action(detail=False, name="Get severity choices")
    def severity(self, request):
        return Response(dict(Severity.choices))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get status choices")
    def status(self, request):
        return Response(dict(SecurityException.Status.choices))

    def get_queryset(self):
        return (
            super()
            .get_queryset()
            .select_related("folder")
            .prefetch_related(
                "assets",
                "applied_controls",
                "vulnerabilities",
                "risk_scenarios",
                "requirement_assessments",
                "owners",
            )
        )

    @action(detail=False, name="Get security exception Sankey data")
    def sankey_data(self, request):
        folder_id = request.query_params.get("folder", None)
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, SecurityException
        )
        queryset = SecurityException.objects.filter(id__in=viewable_objects)

        if folder_id:
            folder = Folder.objects.get(id=folder_id)
            queryset = queryset.filter(folder=folder)

        # Get severity and status combinations
        from django.db.models import Count

        combinations = (
            queryset.values("severity", "status")
            .annotate(count=Count("id"))
            .filter(count__gt=0, status__isnull=False)
            .exclude(status="")
        )

        # Build Sankey data structure
        nodes = []
        links = []
        node_names = set()

        # Create maps for severity and status labels
        severity_choice_map = {choice[0]: choice[1] for choice in Severity.choices}
        status_choice_map = {
            choice[0]: choice[1] for choice in SecurityException.Status.choices
        }

        # Create severity and status nodes only for data that exists
        severity_map = {}
        status_map = {}

        for combo in combinations:
            # Create severity node if not already created
            if combo["severity"] not in severity_map:
                severity_label = f"Severity: {severity_choice_map.get(combo['severity'], 'Unknown').title()}"
                severity_map[combo["severity"]] = severity_label
                node_names.add(severity_label)

            # Create status node if not already created
            if combo["status"] not in status_map:
                status_label = f"Status: {status_choice_map.get(combo['status'], 'Unknown').title()}"
                status_map[combo["status"]] = status_label
                node_names.add(status_label)

        # Convert node names to indexed list
        nodes = [{"name": name} for name in sorted(node_names)]
        node_index = {name: i for i, name in enumerate(sorted(node_names))}

        # Create links
        for combo in combinations:
            severity_name = severity_map[combo["severity"]]
            status_name = status_map[combo["status"]]

            links.append(
                {
                    "source": node_index[severity_name],
                    "target": node_index[status_name],
                    "value": combo["count"],
                }
            )

        return Response({"results": {"nodes": nodes, "links": links}})


class FindingsAssessmentViewSet(BaseModelViewSet):
    model = FindingsAssessment
    filterset_fields = [
        "name",
        "ref_id",
        "category",
        "perimeter",
        "folder",
        "authors",
        "status",
        "evidences",
        "genericcollection",
    ]
    search_fields = ["name", "description", "ref_id"]

    def get_queryset(self):
        dealt_with_statuses = [
            Finding.Status.MITIGATED,
            Finding.Status.RESOLVED,
            Finding.Status.DISMISSED,
            Finding.Status.CLOSED,
        ]
        return (
            super()
            .get_queryset()
            .select_related("folder", "perimeter")
            .prefetch_related(
                "evidences",
                "authors",
            )
            .annotate(
                _total_findings=Count("findings"),
                _dealt_findings=Count(
                    "findings",
                    filter=Q(findings__status__in=dealt_with_statuses),
                    distinct=True,
                ),
                treatment_progress=Case(
                    When(_total_findings=0, then=Value(0)),
                    default=100 * F("_dealt_findings") / F("_total_findings"),
                    output_field=IntegerField(),
                ),
            )
        )

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get status choices")
    def status(self, request):
        return Response(dict(FindingsAssessment.Status.choices))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get category choices")
    def category(self, request):
        return Response(dict(FindingsAssessment.Category.choices))

    @action(detail=True, name="Get Follow up metrics")
    def metrics(self, request, pk=None):
        assessment = self.get_object()
        raw_metrics = assessment.get_findings_metrics()

        def format_severity_data(metrics):
            severity_colors = {
                "info": "#3B82F6",
                "low": "#59BBB2",
                "medium": "#F5C481",
                "high": "#E6686D",
                "critical": "#C71E1D",
                "undefined": "#CCCCCC",
            }

            severity_chart_data = []
            for severity, count in metrics["severity_distribution"].items():
                severity_chart_data.append(
                    {
                        "name": severity.capitalize(),
                        "value": count,
                        "color": severity_colors.get(severity, "#CCCCCC"),
                    }
                )

            return severity_chart_data

        def format_status_data(metrics):
            status_mapping = {
                "identified": {"localName": "identified", "color": "#F5C481"},
                "confirmed": {"localName": "confirmed", "color": "#E6686D"},
                "assigned": {"localName": "assigned", "color": "#fab998"},
                "in_progress": {"localName": "inProgress", "color": "#fac858"},
                "mitigated": {
                    "localName": "mitigated",
                    "color": "hsl(80deg, 80%, 60%)",
                },
                "resolved": {"localName": "resolved", "color": "hsl(120deg, 80%, 45%)"},
                "closed": {"localName": "closed", "color": "hsl(120deg, 60%, 35%)"},
                "dismissed": {"localName": "dismissed", "color": "#5470c6"},
                "deprecated": {"localName": "deprecated", "color": "#91cc75"},
                "--": {"localName": "undefined", "color": "#CCCCCC"},
            }

            status_values = []

            for status, count in metrics["status_distribution"].items():
                mapping_info = status_mapping.get(
                    status, {"localName": status, "color": "#CCCCCC"}
                )
                status_values.append(
                    {
                        "value": count,
                        "localName": mapping_info["localName"],
                        "itemStyle": {"color": mapping_info["color"]},
                    }
                )

            return {"values": status_values}

        formatted_metrics = {
            "raw_metrics": raw_metrics,
            "severity_chart_data": format_severity_data(raw_metrics),
            "status_chart_data": format_status_data(raw_metrics),
        }

        return Response(formatted_metrics)

    @action(detail=True, methods=["get"], name="Findings Assessment as Excel")
    def xlsx(self, request, pk):
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, FindingsAssessment
        )
        if UUID(pk) not in viewable_objects:
            return Response(
                {"error": "Permission denied"}, status=status.HTTP_403_FORBIDDEN
            )

        findings_assessment = FindingsAssessment.objects.get(id=pk)
        findings = Finding.objects.filter(findings_assessment=pk).order_by("ref_id")

        # Prepare data for Excel export
        entries = []
        for finding in findings:
            entry = {
                "ref_id": finding.ref_id,
                "name": finding.name,
                "description": finding.description,
                "status": finding.get_status_display(),
                "severity": finding.get_severity_display(),
                "folder": finding.folder.name if finding.folder else "",
                "applied_controls": "\n".join(
                    [
                        f"{ac.name} [{ac.get_status_display().lower()}]"
                        for ac in finding.applied_controls.all()
                    ]
                ),
                "evidences": "\n".join([ev.name for ev in finding.evidences.all()]),
                "created_at": finding.created_at.strftime("%Y-%m-%d %H:%M:%S")
                if finding.created_at
                else "",
                "eta": finding.eta.strftime("%Y-%m-%d") if finding.eta else "",
                "due_date": finding.due_date.strftime("%Y-%m-%d")
                if finding.due_date
                else "",
            }
            entries.append(entry)

        df = pd.DataFrame(entries)
        buffer = io.BytesIO()

        # Create ExcelWriter with openpyxl engine
        with pd.ExcelWriter(buffer, engine="openpyxl") as writer:
            df.to_excel(writer, index=False, sheet_name="Findings")

            # Get the worksheet
            worksheet = writer.sheets["Findings"]

            # Apply text wrapping to columns with line breaks
            wrap_columns = ["name", "description", "applied_controls", "evidences"]
            wrap_indices = [
                df.columns.get_loc(col) + 1 for col in wrap_columns if col in df.columns
            ]

            for col_idx in wrap_indices:
                for row_idx in range(2, len(df) + 2):
                    cell = worksheet.cell(row=row_idx, column=col_idx)
                    cell.alignment = Alignment(wrap_text=True)

            # Adjust column widths
            for idx, col in enumerate(df.columns):
                column_width = 40 if col in wrap_columns else 20
                worksheet.column_dimensions[
                    worksheet.cell(row=1, column=idx + 1).column_letter
                ].width = column_width

        buffer.seek(0)
        response = HttpResponse(
            buffer.getvalue(),
            content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
        )
        response["Content-Disposition"] = (
            f'attachment; filename="{findings_assessment.name}_findings.xlsx"'
        )
        return response

    @action(detail=True, methods=["get"], name="Findings Assessment as Markdown")
    def md(self, request, pk):
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, FindingsAssessment
        )
        if UUID(pk) not in viewable_objects:
            return Response(
                {"error": "Permission denied"}, status=status.HTTP_403_FORBIDDEN
            )

        findings_assessment = (
            FindingsAssessment.objects.select_related("folder")
            .prefetch_related("authors", "reviewers")
            .get(id=pk)
        )
        findings = (
            Finding.objects.filter(findings_assessment_id=pk)
            .select_related("folder")
            .prefetch_related("applied_controls", "evidences")
            .order_by("ref_id")
        )

        # Calculate closed and open findings counts
        closed_statuses = [
            Finding.Status.DISMISSED,
            Finding.Status.MITIGATED,
            Finding.Status.RESOLVED,
            Finding.Status.CLOSED,
            Finding.Status.DEPRECATED,
        ]
        open_statuses = [
            Finding.Status.UNDEFINED,
            Finding.Status.IDENTIFIED,
            Finding.Status.CONFIRMED,
            Finding.Status.ASSIGNED,
            Finding.Status.IN_PROGRESS,
        ]

        closed_findings_count = findings.filter(status__in=closed_statuses).count()
        open_findings_count = findings.filter(status__in=open_statuses).count()

        # Generate Markdown content
        md_content = f"# {findings_assessment.name}\n\n"

        # Assessment metadata
        md_content += "## Assessment Information\n\n"
        md_content += f"- **Name**: {findings_assessment.name}\n"
        md_content += f"- **Reference ID**: {findings_assessment.ref_id or 'N/A'}\n"
        md_content += f"- **Description**: {findings_assessment.description or 'N/A'}\n"
        md_content += f"- **Category**: {findings_assessment.get_category_display()}\n"
        md_content += f"- **Status**: {findings_assessment.get_status_display()}\n"
        md_content += f"- **Folder**: {findings_assessment.folder.name if findings_assessment.folder else 'N/A'}\n"
        if findings_assessment.authors.exists():
            md_content += f"- **Authors**: {', '.join([str(actor) for actor in findings_assessment.authors.all()])}\n"
        if findings_assessment.reviewers.exists():
            md_content += f"- **Reviewers**: {', '.join([str(actor) for actor in findings_assessment.reviewers.all()])}\n"
        md_content += f"- **Created**: {findings_assessment.created_at.strftime('%Y-%m-%d %H:%M:%S') if findings_assessment.created_at else 'N/A'}\n\n"

        # Metrics summary
        metrics = findings_assessment.get_findings_metrics()
        md_content += "## Summary\n\n"
        md_content += "| Metric | Count |\n"
        md_content += "|--------|-------|\n"
        md_content += f"| Total Findings | {metrics['total_count']} |\n"
        md_content += f"| Closed Findings | {closed_findings_count} |\n"
        md_content += f"| Open Findings | {open_findings_count} |\n\n"

        # Severity distribution
        if metrics["severity_distribution"]:
            md_content += "### Severity Distribution\n\n"
            md_content += "| Severity | Count |\n"
            md_content += "|----------|-------|\n"
            for severity, count in metrics["severity_distribution"].items():
                if count > 0:
                    md_content += f"| {severity.capitalize()} | {count} |\n"
            md_content += "\n"

        # Status distribution
        if metrics["status_distribution"]:
            md_content += "### Status Distribution\n\n"
            md_content += "| Status | Count |\n"
            md_content += "|--------|-------|\n"
            status_choices = dict(Finding.Status.choices)
            for status, count in metrics["status_distribution"].items():
                if count > 0:
                    status_display = status_choices.get(
                        status, status.replace("_", " ").title()
                    )
                    md_content += f"| {status_display} | {count} |\n"
            md_content += "\n"

        # Findings details
        md_content += "## Findings\n\n"
        for finding in findings:
            md_content += f"### {finding.ref_id or 'N/A'} - {finding.name}\n\n"
            md_content += f"- **Status**: {finding.get_status_display()}\n"
            md_content += f"- **Severity**: {finding.get_severity_display()}\n"
            md_content += f"- **Description**: {finding.description or 'N/A'}\n"
            md_content += f"- **Observation**: {finding.observation or 'N/A'}\n"
            if finding.applied_controls.exists():
                md_content += "- **Applied Controls**:\n"
                for ac in finding.applied_controls.all():
                    md_content += f"  - {ac.name} [{ac.get_status_display().lower()}]\n"
            if finding.evidences.exists():
                md_content += f"- **Evidences**: {', '.join([ev.name for ev in finding.evidences.all()])}\n"
            if finding.eta:
                md_content += f"- **ETA**: {finding.eta.strftime('%Y-%m-%d')}\n"
            if finding.due_date:
                md_content += (
                    f"- **Due Date**: {finding.due_date.strftime('%Y-%m-%d')}\n"
                )
            md_content += "\n"

        response = HttpResponse(md_content, content_type="text/markdown")
        safe_name = slugify(findings_assessment.name) or "findings_assessment"
        response["Content-Disposition"] = (
            f'attachment; filename="{safe_name}_findings.md"'
        )
        return response

    @action(detail=True, methods=["get"], name="Findings Assessment as PDF")
    def pdf(self, request, pk):
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, FindingsAssessment
        )
        if UUID(pk) not in viewable_objects:
            return Response(
                {"error": "Permission denied"}, status=status.HTTP_403_FORBIDDEN
            )

        findings_assessment = (
            FindingsAssessment.objects.select_related("folder")
            .prefetch_related("authors", "reviewers")
            .get(id=pk)
        )
        findings = (
            Finding.objects.filter(findings_assessment_id=pk)
            .select_related("folder")
            .prefetch_related("applied_controls", "evidences")
            .order_by("ref_id")
        )
        metrics = findings_assessment.get_findings_metrics()

        # Calculate closed and open findings counts
        closed_statuses = [
            Finding.Status.DISMISSED,
            Finding.Status.MITIGATED,
            Finding.Status.RESOLVED,
            Finding.Status.CLOSED,
            Finding.Status.DEPRECATED,
        ]
        open_statuses = [
            Finding.Status.UNDEFINED,
            Finding.Status.IDENTIFIED,
            Finding.Status.CONFIRMED,
            Finding.Status.ASSIGNED,
            Finding.Status.IN_PROGRESS,
        ]

        closed_findings_count = findings.filter(status__in=closed_statuses).count()
        open_findings_count = findings.filter(status__in=open_statuses).count()

        # Process status distribution with display names
        status_choices = dict(Finding.Status.choices)
        processed_status_distribution = []
        for status, count in metrics["status_distribution"].items():
            if count > 0:
                display_name = status_choices.get(
                    status, status.replace("_", " ").title()
                )
                processed_status_distribution.append(
                    {"status": status, "display_name": display_name, "count": count}
                )

        context = {
            "findings_assessment": findings_assessment,
            "findings": findings,
            "metrics": metrics,
            "total_findings": metrics["total_count"],
            "closed_findings_count": closed_findings_count,
            "open_findings_count": open_findings_count,
            "unresolved_important": metrics["unresolved_important_count"],
            "severity_distribution": metrics["severity_distribution"],
            "status_distribution": metrics["status_distribution"],
            "processed_status_distribution": processed_status_distribution,
            "finding_status_choices": dict(Finding.Status.choices),
        }

        html = render_to_string("core/findings_assessment_pdf.html", context)
        pdf_file = HTML(string=html).write_pdf()
        response = HttpResponse(pdf_file, content_type="application/pdf")
        safe_name = slugify(findings_assessment.name) or "findings_assessment"
        response["Content-Disposition"] = (
            f'attachment; filename="{safe_name}_findings.pdf"'
        )
        return response

    @action(detail=False, methods=["get"], name="Get sunburst data")
    def sunburst_data(self, request):
        """
        Returns FindingsAssessment data structured for sunburst visualization:
        Categories (pentest, audit, self-identified) -> Status
        """
        folder_id = request.query_params.get("folder", None)

        # Get viewable findings assessments
        scoped_folder = (
            Folder.objects.get(id=folder_id) if folder_id else Folder.get_root_folder()
        )
        (object_ids, _, _) = RoleAssignment.get_accessible_object_ids(
            scoped_folder, request.user, FindingsAssessment
        )

        findings_assessments = FindingsAssessment.objects.filter(id__in=object_ids)

        # Color mapping for statuses
        status_colors = {
            "planned": "#BFDBFE",
            "in_progress": "#5470c6",
            "in_review": "#BBF7D0",
            "done": "#46D39A",
            "deprecated": "#E55759",
        }

        # Build hierarchical structure: category -> status
        category_data = {}

        for fa in findings_assessments:
            category = fa.category if fa.category else "--"
            status = fa.status if fa.status else "planned"

            if category not in category_data:
                category_data[category] = {}

            if status not in category_data[category]:
                category_data[category][status] = 0

            category_data[category][status] += 1

        # Convert to sunburst format
        sunburst_data = []
        for category, statuses in category_data.items():
            children = []
            for status, count in statuses.items():
                if count > 0:
                    children.append(
                        {
                            "name": status,
                            "value": count,
                            "itemStyle": {"color": status_colors.get(status, "#CCC")},
                        }
                    )

            if children:
                sunburst_data.append({"name": category, "children": children})

        return Response(sunburst_data)


class FindingViewSet(BaseModelViewSet):
    model = Finding
    filterset_fields = [
        "name",
        "owner",
        "folder",
        "status",
        "severity",
        "priority",
        "findings_assessment",
        "filtering_labels",
        "applied_controls",
        "evidences",
    ]
    ordering = ["ref_id"]

    def get_queryset(self) -> models.query.QuerySet:
        return (
            super()
            .get_queryset()
            .select_related("folder", "findings_assessment")
            .prefetch_related("filtering_labels", "applied_controls", "evidences")
        )

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get status choices")
    def status(self, request):
        return Response(dict(Finding.Status.choices))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get severity choices")
    def severity(self, request):
        return Response(dict(Severity.choices))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get priority choices")
    def priority(self, request):
        return Response(dict(Finding.PRIORITY))

    @action(detail=False, name="Get all findings owners")
    def owner(self, request):
        return Response(
            ActorReadSerializer(
                Actor.objects.filter(findings__isnull=False).distinct(),
                many=True,
            ).data
        )

    @action(detail=False, name="Get findings sankey data")
    def sankey_data(self, request):
        """
        Returns findings data structured for Sankey diagram:
        Category -> Severity -> Status
        """
        folder_id = request.query_params.get("folder", None)
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, Finding
        )
        queryset = Finding.objects.filter(id__in=viewable_objects).select_related(
            "findings_assessment"
        )

        if folder_id:
            folder = Folder.objects.get(id=folder_id)
            queryset = queryset.filter(folder=folder)

        # Build Sankey data structure

        # Get category -> severity -> status combinations
        nodes = []
        links = []
        node_names = set()

        # Create maps for choices
        category_choice_map = {
            choice[0]: choice[1] for choice in FindingsAssessment.Category.choices
        }
        severity_choice_map = {choice[0]: choice[1] for choice in Severity.choices}
        status_choice_map = {choice[0]: choice[1] for choice in Finding.Status.choices}

        # Track category -> severity links
        category_severity_counts = {}
        # Track severity -> status links
        severity_status_counts = {}

        for finding in queryset:
            # Get category from parent findings assessment
            category_value = (
                finding.findings_assessment.category
                if finding.findings_assessment
                else "--"
            )
            category_label = f"Category: {category_choice_map.get(category_value, 'Unknown').title()}"

            # Get severity
            severity_value = finding.severity if finding.severity else "--"
            severity_label = f"Severity: {severity_choice_map.get(severity_value, 'Unknown').title()}"

            # Get status
            status_value = finding.status if finding.status else "--"
            status_label = (
                f"Status: {status_choice_map.get(status_value, 'Unknown').title()}"
            )

            # Add to node sets
            node_names.add(category_label)
            node_names.add(severity_label)
            node_names.add(status_label)

            # Track links
            cat_sev_key = f"{category_label}||{severity_label}"
            category_severity_counts[cat_sev_key] = (
                category_severity_counts.get(cat_sev_key, 0) + 1
            )

            sev_stat_key = f"{severity_label}||{status_label}"
            severity_status_counts[sev_stat_key] = (
                severity_status_counts.get(sev_stat_key, 0) + 1
            )

        # Convert node names to indexed list
        nodes = [{"name": name} for name in sorted(node_names)]
        node_index = {name: i for i, name in enumerate(sorted(node_names))}

        # Create links for category -> severity
        for key, value in category_severity_counts.items():
            category, severity = key.split("||")
            links.append(
                {
                    "source": node_index[category],
                    "target": node_index[severity],
                    "value": value,
                }
            )

        # Create links for severity -> status
        for key, value in severity_status_counts.items():
            severity, status = key.split("||")
            links.append(
                {
                    "source": node_index[severity],
                    "target": node_index[status],
                    "value": value,
                }
            )

        return Response({"results": {"nodes": nodes, "links": links}})


class IncidentViewSet(ExportMixin, BaseModelViewSet):
    model = Incident
    search_fields = ["name", "description", "ref_id"]
    filterset_fields = [
        "folder",
        "status",
        "severity",
        "qualifications",
        "detection",
        "owners",
        "entities",
        "assets",
        "filtering_labels",
    ]

    export_config = {
        "fields": {
            "internal_id": {"source": "id", "label": "internal_id"},
            "ref_id": {"source": "ref_id", "label": "ref_id", "escape": True},
            "name": {"source": "name", "label": "name", "escape": True},
            "description": {
                "source": "description",
                "label": "description",
                "escape": True,
            },
            "status": {"source": "get_status_display", "label": "status"},
            "severity": {"source": "get_severity_display", "label": "severity"},
            "detection": {"source": "get_detection_display", "label": "detection"},
            "reported_at": {
                "source": "reported_at",
                "label": "reported_at",
                "format": lambda dt: (
                    dt.replace(tzinfo=None)
                    if dt and hasattr(dt, "tzinfo") and dt.tzinfo
                    else dt
                ),
            },
            "owners": {
                "source": "owners",
                "label": "owners",
                "format": lambda qs: ",".join(
                    escape_excel_formula(str(o)) for o in qs.all()
                ),
            },
            "folder": {"source": "folder.name", "label": "folder", "escape": True},
            "qualifications": {
                "source": "qualifications",
                "label": "qualifications",
                "format": lambda qs: ",".join(
                    escape_excel_formula(q.name) for q in qs.all()
                ),
            },
            "threats": {
                "source": "threats",
                "label": "threats",
                "format": lambda qs: ",".join(
                    escape_excel_formula(t.name) for t in qs.all()
                ),
            },
            "assets": {
                "source": "assets",
                "label": "assets",
                "format": lambda qs: ",".join(
                    escape_excel_formula(a.name) for a in qs.all()
                ),
            },
            "entities": {
                "source": "entities",
                "label": "entities",
                "format": lambda qs: ",".join(
                    escape_excel_formula(e.name) for e in qs.all()
                ),
            },
            "link": {"source": "link", "label": "link", "escape": True},
            "labels": {
                "source": "filtering_labels",
                "label": "labels",
                "format": lambda qs: ",".join(
                    escape_excel_formula(o.label) for o in qs.all()
                ),
            },
        },
        "filename": "incidents_export",
        "select_related": ["folder"],
        "prefetch_related": [
            "owners",
            "qualifications",
            "threats",
            "assets",
            "entities",
        ],
    }

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get status choices")
    def status(self, request):
        return Response(dict(Incident.Status.choices))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get severity choices")
    def severity(self, request):
        return Response(dict(Incident.Severity.choices))

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get detection channel choices")
    def detection(self, request):
        return Response(dict(Incident.Detection.choices))

    def perform_update(self, serializer):
        previous_instance = self.get_object()
        previous_status = previous_instance.status
        previous_severity = previous_instance.severity

        instance = serializer.save()

        if previous_status != instance.status and previous_status is not None:
            TimelineEntry.objects.create(
                incident=instance,
                entry=f"{previous_instance.get_status_display()}->{instance.get_status_display()}",
                entry_type=TimelineEntry.EntryType.STATUS_CHANGED,
                author=self.request.user.actor,
                timestamp=now(),
            )

        if previous_severity != instance.severity and previous_severity is not None:
            TimelineEntry.objects.create(
                incident=instance,
                entry=f"{previous_instance.get_severity_display()}->{instance.get_severity_display()}",
                entry_type=TimelineEntry.EntryType.SEVERITY_CHANGED,
                author=self.request.user.actor,
                timestamp=now(),
            )

        return super().perform_update(serializer)

    @action(detail=True, methods=["get"], name="Incident as PDF")
    def pdf(self, request, pk):
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, Incident
        )
        if UUID(pk) not in viewable_objects:
            return Response(
                {"error": "Permission denied"}, status=status.HTTP_403_FORBIDDEN
            )

        incident = (
            Incident.objects.select_related("folder")
            .prefetch_related(
                "owners", "entities", "assets", "threats", "qualifications"
            )
            .get(id=pk)
        )

        timeline_entries = (
            TimelineEntry.objects.filter(incident_id=pk)
            .select_related("author")
            .prefetch_related("evidences")
            .order_by("timestamp")
        )

        # Count timeline entry types
        detection_count = timeline_entries.filter(
            entry_type=TimelineEntry.EntryType.DETECTION
        ).count()
        mitigation_count = timeline_entries.filter(
            entry_type=TimelineEntry.EntryType.MITIGATION
        ).count()

        context = {
            "incident": incident,
            "timeline_entries": timeline_entries,
            "detection_count": detection_count,
            "mitigation_count": mitigation_count,
        }

        html = render_to_string("core/incident_pdf.html", context)
        pdf_file = HTML(string=html).write_pdf()
        response = HttpResponse(pdf_file, content_type="application/pdf")
        safe_name = slugify(incident.name) or "incident"
        response["Content-Disposition"] = (
            f'attachment; filename="{safe_name}_report.pdf"'
        )
        return response

    @action(detail=True, methods=["get"], name="Incident as Markdown")
    def md(self, request, pk):
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, Incident
        )
        if UUID(pk) not in viewable_objects:
            return Response(
                {"error": "Permission denied"}, status=status.HTTP_403_FORBIDDEN
            )

        incident = (
            Incident.objects.select_related("folder")
            .prefetch_related("owners", "entities", "assets", "threats")
            .get(id=pk)
        )

        timeline_entries = (
            TimelineEntry.objects.filter(incident_id=pk)
            .select_related("author")
            .prefetch_related("evidences")
            .order_by("timestamp")
        )

        # Generate Markdown content
        md_content = f"# {incident.name}\n\n"

        # Incident metadata
        md_content += "## Incident Information\n\n"
        md_content += f"- **Name**: {incident.name}\n"
        md_content += f"- **Reference ID**: {incident.ref_id or 'N/A'}\n"
        md_content += f"- **Description**: {incident.description or 'N/A'}\n"
        md_content += f"- **Status**: {incident.get_status_display()}\n"
        md_content += f"- **Severity**: {incident.get_severity_display()}\n"
        md_content += f"- **Detection**: {incident.get_detection_display() or 'N/A'}\n"
        md_content += (
            f"- **Domain**: {incident.folder.name if incident.folder else 'N/A'}\n"
        )

        if incident.owners.exists():
            md_content += f"- **Owners**: {', '.join([str(actor) for actor in incident.owners.all()])}\n"

        if incident.entities.exists():
            md_content += f"- **Related Entities**: {', '.join([entity.name for entity in incident.entities.all()])}\n"

        md_content += f"- **Created**: {incident.created_at.strftime('%Y-%m-%d %H:%M:%S') if incident.created_at else 'N/A'}\n"
        md_content += f"- **Last Updated**: {incident.updated_at.strftime('%Y-%m-%d %H:%M:%S') if incident.updated_at else 'N/A'}\n\n"

        # Affected Assets
        if incident.assets.exists():
            md_content += "## Affected Assets\n\n"
            md_content += "| Name | Type |\n"
            md_content += "|------|------|\n"
            for asset in incident.assets.all():
                md_content += f"| {asset.name} | {asset.get_type_display()} |\n"
            md_content += "\n"

        # Related Threats
        if incident.threats.exists():
            md_content += "## Related Threats\n\n"
            md_content += "| Name | Reference ID |\n"
            md_content += "|------|------|\n"
            for threat in incident.threats.all():
                md_content += f"| {threat.name} | {threat.ref_id or 'N/A'} |\n"
            md_content += "\n"

        # Timeline
        if timeline_entries:
            md_content += "## Timeline\n\n"
            md_content += "*Events are listed in chronological order*\n\n"

            for entry in timeline_entries:
                md_content += f"### {entry.timestamp.strftime('%Y-%m-%d %H:%M:%S')} - {entry.entry}\n\n"
                md_content += f"**Type**: {entry.get_entry_type_display()}\n\n"

                if entry.author:
                    md_content += f"**Author**: {str(entry.author)}\n\n"

                if entry.observation:
                    md_content += f"**Observation**: {entry.observation}\n\n"

                if entry.evidences.exists():
                    md_content += "**Associated Evidence**:\n"
                    for evidence in entry.evidences.all():
                        md_content += f"- {evidence.name}\n"
                    md_content += "\n"

                md_content += "---\n\n"
        else:
            md_content += "## Timeline\n\n"
            md_content += "*No timeline events found for this incident.*\n\n"

        response = HttpResponse(md_content, content_type="text/markdown")
        safe_name = slugify(incident.name) or "incident"
        response["Content-Disposition"] = (
            f'attachment; filename="{safe_name}_report.md"'
        )
        return response

    @action(detail=False, name="Get incident detection breakdown")
    def detection_breakdown(self, request):
        folder_id = request.query_params.get("folder", None)
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, Incident
        )
        queryset = Incident.objects.filter(id__in=viewable_objects)

        if folder_id:
            folder = Folder.objects.get(id=folder_id)
            queryset = queryset.filter(folder=folder)

        detection_stats = []
        for detection_choice in Incident.Detection.choices:
            count = queryset.filter(detection=detection_choice[0]).count()
            detection_stats.append(
                {
                    "name": detection_choice[1],
                    "value": count,
                    "itemStyle": {
                        "color": "#3B82F6"
                        if detection_choice[0] == "internally_detected"
                        else "#EF4444"
                    },
                }
            )

        return Response({"results": detection_stats})

    @action(detail=False, name="Get monthly incident metrics")
    def monthly_metrics(self, request):
        folder_id = request.query_params.get("folder", None)
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, Incident
        )
        queryset = Incident.objects.filter(id__in=viewable_objects)

        if folder_id:
            folder = Folder.objects.get(id=folder_id)
            queryset = queryset.filter(folder=folder)

        # Get incidents with reported_at dates
        incidents_with_dates = queryset.filter(reported_at__isnull=False).order_by(
            "reported_at"
        )

        if not incidents_with_dates.exists():
            return Response(
                {
                    "results": {
                        "months": [],
                        "monthly_counts": [],
                        "cumulative_counts": [],
                    }
                }
            )

        # Group by month
        from collections import defaultdict
        from datetime import datetime

        monthly_counts = defaultdict(int)

        for incident in incidents_with_dates:
            month_key = incident.reported_at.strftime("%Y-%m")
            monthly_counts[month_key] += 1

        # Sort months and calculate cumulative
        sorted_months = sorted(monthly_counts.keys())
        cumulative_count = 0
        cumulative_counts = []

        for month in sorted_months:
            cumulative_count += monthly_counts[month]
            cumulative_counts.append(cumulative_count)

        # Format months for display
        formatted_months = []
        for month in sorted_months:
            date_obj = datetime.strptime(month, "%Y-%m")
            formatted_months.append(date_obj.strftime("%b %Y"))

        return Response(
            {
                "results": {
                    "months": formatted_months,
                    "monthly_counts": [
                        monthly_counts[month] for month in sorted_months
                    ],
                    "cumulative_counts": cumulative_counts,
                }
            }
        )

    @action(detail=False, name="Get incident summary statistics")
    def summary_stats(self, request):
        folder_id = request.query_params.get("folder", None)
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, Incident
        )
        queryset = Incident.objects.filter(id__in=viewable_objects)

        if folder_id:
            folder = Folder.objects.get(id=folder_id)
            queryset = queryset.filter(folder=folder)

        # Total incidents
        total_incidents = queryset.count()

        # Incidents this month
        from datetime import date
        import calendar

        today = date.today()
        first_day = today.replace(day=1)
        last_day = today.replace(day=calendar.monthrange(today.year, today.month)[1])

        incidents_this_month = queryset.filter(
            reported_at__date__gte=first_day, reported_at__date__lte=last_day
        ).count()

        # Currently open incidents (not closed or dismissed)
        open_incidents = queryset.exclude(
            status__in=[Incident.Status.CLOSED, Incident.Status.DISMISSED]
        ).count()

        return Response(
            {
                "results": {
                    "total_incidents": total_incidents,
                    "incidents_this_month": incidents_this_month,
                    "open_incidents": open_incidents,
                }
            }
        )

    @action(detail=False, name="Get incident severity breakdown")
    def severity_breakdown(self, request):
        folder_id = request.query_params.get("folder", None)
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, Incident
        )
        queryset = Incident.objects.filter(id__in=viewable_objects)

        if folder_id:
            folder = Folder.objects.get(id=folder_id)
            queryset = queryset.filter(folder=folder)

        # Define severity colors
        severity_colors = {
            1: "#DC2626",  # Critical - Red
            2: "#EA580C",  # Major - Orange
            3: "#D97706",  # Moderate - Amber
            4: "#65A30D",  # Minor - Lime
            5: "#16A34A",  # Low - Green
            6: "#6B7280",  # Unknown - Gray
        }

        severity_stats = []
        for severity_choice in Incident.Severity.choices:
            count = queryset.filter(severity=severity_choice[0]).count()
            severity_stats.append(
                {
                    "name": severity_choice[1],
                    "value": count,
                    "itemStyle": {
                        "color": severity_colors.get(severity_choice[0], "#6B7280")
                    },
                }
            )

        return Response({"results": severity_stats})

    @action(detail=False, name="Get incident qualifications breakdown")
    def qualifications_breakdown(self, request):
        folder_id = request.query_params.get("folder", None)
        (viewable_objects, _, _) = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, Incident
        )
        queryset = Incident.objects.filter(id__in=viewable_objects)

        if folder_id:
            folder = Folder.objects.get(id=folder_id)
            queryset = queryset.filter(folder=folder)

        # Get all unique qualifications used in incidents
        from django.db.models import Count

        qualifications_stats = []

        # Get qualification counts
        qualification_counts = (
            queryset.values("qualifications__name")
            .annotate(count=Count("id", distinct=True))
            .filter(qualifications__name__isnull=False)
            .order_by("-count")
        )

        # Format for radar chart
        labels = []
        values = []
        for item in qualification_counts:
            if item["qualifications__name"]:
                values.append(item["count"])

        # Create labels with proper format for radar chart
        max_offset = max(values, default=0)
        for item in qualification_counts:
            if item["qualifications__name"]:
                labels.append({"name": item["qualifications__name"], "max": max_offset})

        return Response({"results": {"labels": labels, "values": values}})


class TimelineEntryViewSet(BaseModelViewSet):
    model = TimelineEntry
    filterset_fields = ["incident"]
    search_fields = ["entry", "entry_type"]
    ordering = ["-timestamp"]

    def get_queryset(self):
        return (
            super()
            .get_queryset()
            .select_related("folder", "incident", "author")
            .prefetch_related("evidences")
        )

    @action(detail=False, name="Get entry type choices")
    def entry_type(self, request):
        return Response(dict(TimelineEntry.EntryType.get_manual_entry_types()))

    def perform_create(self, serializer):
        instance = serializer.save(author=self.request.user.actor)
        dispatch_webhook_event(instance, "created", serializer)
        return instance

    def perform_destroy(self, instance):
        if instance.entry_type in [
            TimelineEntry.EntryType.SEVERITY_CHANGED,
            TimelineEntry.EntryType.STATUS_CHANGED,
        ]:
            raise ValidationError({"error": "cannotDeleteAutoTimelineEntry"})
        return super().perform_destroy(instance)


class CommentViewSet(BaseModelViewSet):
    model = Comment
    filterset_fields = [
        "requirement_assessment",
        "risk_scenario",
        "applied_control",
        "finding",
        "is_active",
        "author",
    ]
    search_fields = ["body"]
    ordering = ["created_at"]

    def get_queryset(self):
        return (
            super()
            .get_queryset()
            .select_related(
                "folder",
                "author",
                "requirement_assessment",
                "risk_scenario",
                "applied_control",
                "finding",
            )
        )

    def perform_create(self, serializer):
        serializer.save(author=self.request.user)

    def perform_update(self, serializer):
        if serializer.instance.author_id != self.request.user.id:
            raise PermissionDenied({"error": "You can only edit your own comments."})
        super().perform_update(serializer)

    def perform_destroy(self, instance):
        if instance.author_id != self.request.user.id:
            if not self.request.user.is_admin():
                raise PermissionDenied(
                    {"error": "You can only delete your own comments."}
                )
        return super().perform_destroy(instance)


class TaskTemplateFilter(GenericFilterSet):
    folder = df.ModelMultipleChoiceFilter(queryset=Folder.objects.all())

    next_occurrence_status = df.MultipleChoiceFilter(
        choices=TaskNode.TASK_STATUS_CHOICES, method="filter_next_occurrence_status"
    )
    last_occurrence_status = df.MultipleChoiceFilter(
        choices=TaskNode.TASK_STATUS_CHOICES, method="filter_last_occurrence_status"
    )

    class Meta:
        model = TaskTemplate
        fields = [
            "name",
            "assigned_to",
            "is_recurrent",
            "applied_controls",
            "last_occurrence_status",
            "next_occurrence_status",
            "evidences",
            "objectives",
        ]

    def filter_last_occurrence_status(self, queryset, name, values):
        start = timezone.now().date()
        status_subquery = (
            TaskNode.objects.filter(task_template=OuterRef("pk"), due_date__lt=start)
            .order_by("-due_date")
            .values("status")[:1]
        )

        return queryset.annotate(last_status=Subquery(status_subquery)).filter(
            last_status__in=values
        )

    def filter_next_occurrence_status(self, queryset, name, values):
        start = timezone.now().date()
        status_subquery = (
            TaskNode.objects.filter(task_template=OuterRef("pk"), due_date__gte=start)
            .order_by("due_date")
            .values("status")[:1]
        )

        return queryset.annotate(next_status=Subquery(status_subquery)).filter(
            next_status__in=values
        )


class TaskTemplateViewSet(ExportMixin, BaseModelViewSet):
    model = TaskTemplate
    filterset_fields = [
        "assigned_to",
        "is_recurrent",
        "folder",
        "applied_controls",
        "evidences",
    ]
    search_fields = ["ref_id", "name"]
    filterset_class = TaskTemplateFilter

    export_config = {
        "filename": "task_templates",
        "select_related": ["folder"],
        "prefetch_related": [
            "assigned_to",
            "evidences",
            "applied_controls",
            "compliance_assessments",
            "risk_assessments",
            "assets",
            "findings_assessment",
        ],
        "fields": {
            "ref_id": {"source": "ref_id", "label": "ref_id", "escape": True},
            "name": {"source": "name", "label": "name", "escape": True},
            "description": {
                "source": "description",
                "label": "description",
                "escape": True,
            },
            "folder": {"source": "folder.name", "label": "folder", "escape": True},
            "is_recurrent": {
                "source": "is_recurrent",
                "label": "is_recurrent",
                "format": lambda x: "Yes" if x else "No",
            },
            "task_date": {
                "source": "task_date",
                "label": "task_date",
                "format": lambda x: x.strftime("%Y-%m-%d") if x else "",
            },
            "schedule_frequency": {
                "source": "schedule",
                "label": "schedule_frequency",
                "format": lambda s: s.get("frequency", "") if s else "",
            },
            "schedule_interval": {
                "source": "schedule",
                "label": "schedule_interval",
                "format": lambda s: str(s.get("interval", "")) if s else "",
            },
            "schedule_end_date": {
                "source": "schedule",
                "label": "schedule_end_date",
                "format": lambda s: s.get("end_date", "") if s else "",
            },
            "next_occurrence": {
                "source": "get_next_occurrence",
                "label": "next_occurrence",
                "format": lambda x: x.strftime("%Y-%m-%d") if x else "",
            },
            "next_occurrence_status": {
                "source": "get_next_occurrence_status",
                "label": "next_occurrence_status",
            },
            "enabled": {
                "source": "enabled",
                "label": "enabled",
                "format": lambda x: "Yes" if x else "No",
            },
            "assigned_to": {
                "source": "assigned_to.all",
                "label": "assigned_to",
                "format": lambda actors: (
                    ", ".join([str(a) for a in actors]) if actors else ""
                ),
            },
            "assets": {
                "source": "assets.all",
                "label": "assets",
                "format": lambda items: (
                    ", ".join(escape_excel_formula(str(item)) for item in items)
                    if items
                    else ""
                ),
            },
            "applied_controls": {
                "source": "applied_controls.all",
                "label": "applied_controls",
                "format": lambda items: (
                    ", ".join(escape_excel_formula(str(item)) for item in items)
                    if items
                    else ""
                ),
            },
            "evidences": {
                "source": "evidences.all",
                "label": "evidences",
                "format": lambda items: (
                    ", ".join(escape_excel_formula(item.name) for item in items)
                    if items
                    else ""
                ),
            },
            "compliance_assessments": {
                "source": "compliance_assessments.all",
                "label": "compliance_assessments",
                "format": lambda items: (
                    ", ".join(escape_excel_formula(str(item)) for item in items)
                    if items
                    else ""
                ),
            },
            "risk_assessments": {
                "source": "risk_assessments.all",
                "label": "risk_assessments",
                "format": lambda items: (
                    ", ".join(escape_excel_formula(str(item)) for item in items)
                    if items
                    else ""
                ),
            },
            "findings_assessment": {
                "source": "findings_assessment.all",
                "label": "findings_assessment",
                "format": lambda items: (
                    ", ".join(escape_excel_formula(str(item)) for item in items)
                    if items
                    else ""
                ),
            },
            "link": {"source": "link", "label": "link", "escape": True},
        },
        "wrap_columns": ["name", "description"],
        "task_node_fields": {
            "due_date": {
                "source": "due_date",
                "label": "due_date",
                "format": lambda x: x.strftime("%Y-%m-%d") if x else "",
            },
            "status": {"source": "status", "label": "status"},
            "observation": {
                "source": "observation",
                "label": "observation",
                "escape": True,
            },
            "assigned_to": {
                "source": "task_template.assigned_to.all",
                "label": "assigned_to",
                "format": lambda actors: (
                    ", ".join([str(a) for a in actors]) if actors else ""
                ),
            },
            "expected_evidence": {
                "source": "task_template.evidences.all",
                "label": "expected_evidence",
                # Note: Formatting with done/pending status is handled specially in export_tasks_xlsx
                "format": lambda items: (
                    ", ".join([item.name for item in items]) if items else ""
                ),
            },
            # "evidences": {
            #     "source": "evidences.all",
            #     "label": "evidences",
            #     "format": lambda items: ", ".join([item.name for item in items])
            #     if items
            #     else "",
            # },
            "assets": {
                "source": "task_template.assets.all",
                "label": "assets",
                "format": lambda items: (
                    ", ".join(escape_excel_formula(str(item)) for item in items)
                    if items
                    else ""
                ),
            },
            "applied_controls": {
                "source": "task_template.applied_controls.all",
                "label": "applied_controls",
                "format": lambda items: (
                    ", ".join(escape_excel_formula(str(item)) for item in items)
                    if items
                    else ""
                ),
            },
            "compliance_assessments": {
                "source": "task_template.compliance_assessments.all",
                "label": "compliance_assessments",
                "format": lambda items: (
                    ", ".join(escape_excel_formula(str(item)) for item in items)
                    if items
                    else ""
                ),
            },
            "risk_assessments": {
                "source": "task_template.risk_assessments.all",
                "label": "risk_assessments",
                "format": lambda items: (
                    ", ".join(escape_excel_formula(str(item)) for item in items)
                    if items
                    else ""
                ),
            },
            "findings_assessment": {
                "source": "task_template.findings_assessment.all",
                "label": "findings_assessment",
                "format": lambda items: (
                    ", ".join(escape_excel_formula(str(item)) for item in items)
                    if items
                    else ""
                ),
            },
        },
    }

    def get_queryset(self):
        qs = super().get_queryset()
        ordering = self.request.query_params.get("ordering", "")

        if any(
            f in ordering
            for f in (
                "next_occurrence",
                "last_occurrence_status",
                "next_occurrence_status",
            )
        ):
            today = timezone.localdate()
            qs = qs.annotate(
                next_occurrence=Case(
                    When(is_recurrent=False, then=Min("tasknode__due_date")),
                    When(
                        is_recurrent=True,
                        then=Min(
                            "tasknode__due_date",
                            filter=Q(tasknode__due_date__gte=today),
                        ),
                    ),
                    default=Value(None),
                    output_field=models.DateField(),
                ),
                last_occurrence_status=Subquery(
                    TaskNode.objects.filter(
                        task_template=OuterRef("pk"), due_date__lt=today
                    )
                    .order_by("-due_date")
                    .values("status")[:1]
                ),
                next_occurrence_status=Case(
                    When(
                        is_recurrent=False,
                        then=Subquery(
                            TaskNode.objects.filter(task_template=OuterRef("pk"))
                            .order_by("due_date")
                            .values("status")[:1]
                        ),
                    ),
                    When(
                        is_recurrent=True,
                        then=Subquery(
                            TaskNode.objects.filter(
                                task_template=OuterRef("pk"),
                                due_date__gte=today,
                            )
                            .order_by("due_date")
                            .values("status")[:1]
                        ),
                    ),
                    default=Value(None),
                    output_field=models.CharField(),
                ),
            )

        return qs

    def task_calendar(self, task_templates, start=None, end=None):
        """Generate calendar of tasks for the given templates."""
        today = timezone.localdate()
        task_templates = list(task_templates)
        task_templates_by_id = {
            str(template.id): template for template in task_templates
        }
        tasks_list = []
        for template in task_templates:
            if not template.is_recurrent:
                if not template.task_date:
                    continue
                tasks_list.append(_create_task_dict(template, template.task_date))
                continue

            start_date_param = start or template.task_date or today
            end_date_param = end or template.schedule.get("end_date")

            if not end_date_param:
                start_date = datetime.strptime(str(start_date_param), "%Y-%m-%d").date()
                end_date_param = (start_date + rd.relativedelta(months=1)).strftime(
                    "%Y-%m-%d"
                )

            try:
                start_date = datetime.strptime(str(start_date_param), "%Y-%m-%d").date()
                end_date = datetime.strptime(str(end_date_param), "%Y-%m-%d").date()
            except ValueError:
                return {"error": "Invalid date format. Use YYYY-MM-DD"}

            tasks = _generate_occurrences(template, start_date, end_date)
            tasks_list.extend(tasks)

            # Preserve TaskNodes manually rescheduled by the user
            # (due_date != scheduled_date)
            # Unmodified nodes (due_date == scheduled_date)
            # can be safely garbage-collected on schedule changes.
            existing_nodes = TaskNode.objects.filter(
                task_template=template,
            ).filter(
                Q(
                    scheduled_date__gte=start_date,
                    scheduled_date__lte=end_date,
                )
                | (
                    Q(due_date__gte=start_date, due_date__lte=end_date)
                    & (
                        Q(scheduled_date__isnull=True)
                        | ~Q(due_date=F("scheduled_date"))
                    )
                )
            )
            generated_scheduled_dates = {t["due_date"] for t in tasks}
            for node in existing_nodes:
                if node.due_date != node.scheduled_date:
                    # Always preserve user-rescheduled nodes
                    node.to_delete = False
                    node.save(update_fields=["to_delete"])
                    if node.due_date and start_date <= node.due_date <= end_date:
                        tasks_list.append(TaskNodeReadSerializer(node).data)
                elif node.scheduled_date not in generated_scheduled_dates:
                    effective_date = node.due_date or node.scheduled_date
                    if effective_date and effective_date < today:
                        # Preserve past nodes whose slot was removed
                        node.to_delete = False
                        node.save(update_fields=["to_delete"])
                        tasks_list.append(TaskNodeReadSerializer(node).data)

        def _parse_due_date(val):
            """Normalize due_date to a date object"""
            if isinstance(val, str):
                return datetime.strptime(val, "%Y-%m-%d").date()
            return val

        # Sort tasks by due date, skip entries without a due_date
        sorted_tasks = sorted(
            [t for t in tasks_list if t.get("due_date")],
            key=lambda x: _parse_due_date(x["due_date"]),
        )

        # Build a set of (template_id, date) identifiers for virtual tasks to materialize.
        # Every virtual task in the calendar range gets a DB record so it is
        # clickable/editable in the UI.
        # NOTE: All virtual tasks in the range are materialized. Currently the
        # calendar UI fetches one month at a time; if that changes, consider
        # adding an upper bound here.
        tasks_to_process_ids = set()
        for task in sorted_tasks:
            if not task.get("virtual"):
                continue
            template_id = task.get("task_template")
            task_date = task.get("due_date")
            if template_id and task_date:
                tasks_to_process_ids.add((template_id, task_date))

        processed_tasks_identifiers = set()
        materialized_node_ids = {
            str(task["id"])
            for task in tasks_list
            if task and not task.get("virtual") and task.get("id")
        }

        for i in range(len(tasks_list)):
            task = tasks_list[i]
            task_date = task.get("due_date")
            if not task_date:
                continue

            # Already-serialized TaskNodes from the DB don't need processing
            if not task.get("virtual"):
                continue

            task_template_id = task["task_template"]
            task_identifier = (task_template_id, task_date)

            if task_identifier in processed_tasks_identifiers:
                continue

            if task_identifier in tasks_to_process_ids:
                processed_tasks_identifiers.add(task_identifier)

                task_template = task_templates_by_id[str(task_template_id)]

                # Check if a node already exists for this recurrence slot
                rescheduled_node = TaskNode.objects.filter(
                    task_template=task_template,
                    scheduled_date=task_date,
                ).first()
                if rescheduled_node:
                    if rescheduled_node.due_date != task_date:
                        # Node was rescheduled — already preserved in
                        # the existing_nodes loop, drop the virtual entry.
                        tasks_list[i] = None
                        continue
                    task_node = rescheduled_node
                else:
                    try:
                        task_node, created = TaskNode.objects.get_or_create(
                            task_template=task_template,
                            due_date=task_date,
                            defaults={
                                "scheduled_date": task_date,
                                "status": "pending",
                                "folder": task_template.folder,
                            },
                        )
                    except IntegrityError:
                        existing_node = TaskNode.objects.filter(
                            task_template=task_template,
                            due_date=task_date,
                        ).first()
                        if not existing_node:
                            tasks_list[i] = None
                            continue
                        task_node = existing_node
                if str(task_node.id) in materialized_node_ids:
                    tasks_list[i] = None
                    continue
                materialized_node_ids.add(str(task_node.id))
                task_node.to_delete = False
                task_node.save(update_fields=["to_delete"])
                tasks_list[i] = TaskNodeReadSerializer(task_node).data

        return [task for task in tasks_list if task is not None]

    def _sync_task_nodes(self, task_template: TaskTemplate):
        if task_template.is_recurrent:
            with transaction.atomic():
                today = timezone.localdate()
                # Soft-delete future TaskNode instances for re-evaluation.
                TaskNode.objects.filter(task_template=task_template).filter(
                    Q(scheduled_date__gte=today)
                    | Q(scheduled_date__isnull=True, due_date__gte=today)
                ).update(to_delete=True)
                # Determine the end date based on the frequency
                start_date = task_template.task_date
                if task_template.is_recurrent:
                    if task_template.schedule["frequency"] == "DAILY":
                        delta = rd.relativedelta(months=2)
                    elif task_template.schedule["frequency"] == "WEEKLY":
                        delta = rd.relativedelta(months=4)
                    elif task_template.schedule["frequency"] == "MONTHLY":
                        delta = rd.relativedelta(years=1)
                    elif task_template.schedule["frequency"] == "YEARLY":
                        delta = rd.relativedelta(years=5)

                    end_date_param = task_template.schedule.get("end_date")
                    if end_date_param:
                        end_date = datetime.strptime(end_date_param, "%Y-%m-%d").date()
                    else:
                        end_date = today + delta
                    # Ensure end_date is not before the calculated delta
                    min_end_date = today + delta
                    if end_date < min_end_date:
                        end_date = min_end_date
                else:
                    end_date = start_date
                # Generate the task nodes
                self.task_calendar(
                    task_templates=self.get_queryset().filter(id=task_template.id),
                    start=start_date,
                    end=end_date,
                )

                # garbage-collect — only delete untouched nodes
                TaskNode.objects.filter(
                    to_delete=True,
                    task_template=task_template,
                    status="pending",
                    due_date=F("scheduled_date"),
                ).filter(
                    Q(observation__isnull=True) | Q(observation=""),
                ).exclude(
                    evidences__isnull=False,
                ).exclude(
                    evidence_revisions__isnull=False,
                ).delete()
                # Clear to_delete on surviving nodes
                TaskNode.objects.filter(
                    to_delete=True, task_template=task_template
                ).update(to_delete=False)

    @action(
        detail=False,
        name="Get tasks for the calendar",
        url_path="calendar/(?P<start>.+)/(?P<end>.+)",
    )
    def calendar(
        self,
        request,
        start=None,
        end=None,
    ):
        if start is None:
            start = timezone.now().date()
        if end is None:
            end = timezone.now().date() + relativedelta.relativedelta(months=1)
        return Response(
            self.task_calendar(
                task_templates=self.get_queryset().filter(enabled=True),
                start=start,
                end=end,
            )
        )

    def perform_update(self, serializer):
        task_template = serializer.save()
        self._sync_task_nodes(task_template)

    @action(detail=False, name="Export Tasks with Nodes as Multi-Sheet XLSX")
    def export_xlsx(self, request):
        """
        Export task templates with a summary sheet and individual sheets for each template's task nodes.
        """
        if not self.export_config:
            return HttpResponse(
                status=501, content="Export not configured for this model"
            )

        queryset = self._get_export_queryset()

        wb = Workbook()
        wb.remove(wb.active)

        summary_ws = wb.create_sheet(title="Summary")
        main_fields = self.export_config["fields"]
        main_headers = [f.get("label", name) for name, f in main_fields.items()]
        summary_ws.append(main_headers)

        wrap_alignment = Alignment(wrap_text=True, vertical="top")
        wrap_columns = self.export_config.get("wrap_columns", ["name", "description"])

        used_sheet_names = {"Summary"}
        task_node_fields = self.export_config.get("task_node_fields", {})

        template_counter = 0
        for obj in queryset:
            row = [
                self._resolve_field_value(obj, field_config)
                for field_config in main_fields.values()
            ]
            summary_ws.append(row)

            # Only include past task nodes due_date <= today
            today = timezone.localdate()

            task_nodes = (
                TaskNode.objects.filter(task_template=obj, due_date__lte=today)
                .select_related("task_template", "task_template__folder")
                .prefetch_related(
                    "evidences",
                    "task_template__assigned_to",
                    "task_template__evidences__revisions",  # Prefetch revisions for expected evidences
                    "task_template__assets",
                    "task_template__applied_controls",
                    "task_template__compliance_assessments",
                    "task_template__risk_assessments",
                    "task_template__findings_assessment",
                )
                .order_by("due_date")
            )

            if not task_nodes.exists():
                continue

            template_counter += 1
            task_name = obj.name or f"Template_{obj.pk}"
            # Format: "1-task_name", "2-task_name", etc.
            base_name = f"{template_counter}-{task_name}"

            INVALID_CHARS = r"[\\\/\?\*\[\]:]"
            base_name = re.sub(INVALID_CHARS, "", base_name)

            # Excel sheet names have a 31 character limit
            sheet_name = base_name[:31]
            counter = 1
            while sheet_name in used_sheet_names:
                suffix = f" ({counter})"
                sheet_name = base_name[: 31 - len(suffix)] + suffix
                counter += 1
            used_sheet_names.add(sheet_name)

            detail_ws = wb.create_sheet(title=sheet_name)
            detail_headers = [
                f.get("label", name) for name, f in task_node_fields.items()
            ]
            detail_ws.append(detail_headers)

            for task_node in task_nodes:
                detail_row = []
                for field_name, field_config in task_node_fields.items():
                    if field_name == "expected_evidence":
                        # Special handling for expected evidence with done/pending status
                        evidences = task_node.task_template.evidences.all()
                        evidence_statuses = []
                        for evidence in evidences:
                            last_revision = evidence.last_revision
                            if last_revision and last_revision.task_node == task_node:
                                status = "done"
                            else:
                                status = "pending"
                            evidence_statuses.append(f"{evidence.name} ({status})")
                        value = (
                            ", ".join(evidence_statuses) if evidence_statuses else ""
                        )
                    else:
                        value = self._resolve_field_value(task_node, field_config)
                    detail_row.append(value)
                detail_ws.append(detail_row)

            for col_idx, header in enumerate(detail_headers, 1):
                if header.lower() in wrap_columns:
                    for row_idx in range(1, detail_ws.max_row + 1):
                        detail_ws.cell(
                            row=row_idx, column=col_idx
                        ).alignment = wrap_alignment

        for col_idx, header in enumerate(main_headers, 1):
            if header.lower() in wrap_columns:
                for row_idx in range(1, summary_ws.max_row + 1):
                    summary_ws.cell(
                        row=row_idx, column=col_idx
                    ).alignment = wrap_alignment

        buffer = io.BytesIO()
        wb.save(buffer)
        buffer.seek(0)

        filename = f"{self.export_config.get('filename', 'export')}_multi_sheet.xlsx"
        response = HttpResponse(
            buffer.getvalue(),
            content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
        )
        response["Content-Disposition"] = f'attachment; filename="{filename}"'
        return response

    def perform_create(self, serializer):
        super().perform_create(serializer)
        self._sync_task_nodes(serializer.instance)

    @action(detail=True, name="Get write data")
    def object(self, request, pk):
        serializer_class = self.get_serializer_class(action="update")
        self._sync_task_nodes(
            self.get_object()
        )  # Synchronize task nodes when fetching a task template
        return Response(serializer_class(super().get_object()).data)

    @action(detail=False, name="Get all task template assigned_to actors")
    def assigned_to(self, request):
        return Response(
            ActorReadSerializer(
                Actor.objects.filter(assigned_task_templates__isnull=False).distinct(),
                many=True,
            ).data
        )

    @action(detail=False, name="Yearly tasks review")
    def yearly_review(self, request):
        """Get recurrent task templates grouped by folder for yearly review."""
        # Get date range from query params or use current year
        current_year = timezone.now().year

        try:
            start_month = int(request.query_params.get("start_month", 1))
            start_year = int(request.query_params.get("start_year", current_year))
            end_month = int(request.query_params.get("end_month", 12))
            end_year = int(request.query_params.get("end_year", current_year))
        except ValueError:
            # Default to current year if any parsing fails
            start_month, start_year = 1, current_year
            end_month, end_year = 12, current_year

        # Validate month values
        start_month = max(1, min(12, start_month))
        end_month = max(1, min(12, end_month))

        year_start = date(start_year, start_month, 1)
        # Get last day of end_month
        if end_month == 12:
            year_end = date(end_year, 12, 31)
        else:
            year_end = date(end_year, end_month + 1, 1) - timedelta(days=1)

        # Get folder filter from query params
        folder_id = request.query_params.get("folder")

        # Get all viewable recurrent task templates
        queryset = self.get_queryset().filter(
            is_recurrent=True,
            enabled=True,
        )

        # Apply folder filter if provided
        if folder_id:
            queryset = queryset.filter(folder_id=folder_id)

        task_templates = queryset.select_related("folder").prefetch_related(
            "assigned_to"
        )

        # Group by folder
        folders_dict = defaultdict(list)
        for template in task_templates:
            folder_id = str(template.folder.id)
            folder_name = str(template.folder)

            # Use TaskTemplateReadSerializer for proper serialization
            template_data = TaskTemplateReadSerializer(template).data
            # Add schedule field (excluded by default in read serializer)
            template_data["schedule"] = template.schedule

            # Get TaskNodes for this template in the current year
            task_nodes = TaskNode.objects.filter(
                task_template=template, due_date__gte=year_start, due_date__lte=year_end
            ).values("due_date", "status")

            # Group by month and determine status
            monthly_status = {}
            # Generate list of (year, month) tuples for the date range
            months_in_range = []
            current = date(start_year, start_month, 1)
            end = date(end_year, end_month, 1)
            while current <= end:
                months_in_range.append((current.year, current.month))
                if current.month == 12:
                    current = date(current.year + 1, 1, 1)
                else:
                    current = date(current.year, current.month + 1, 1)

            for year, month in months_in_range:
                # Create a unique key for year-month combination
                key = f"{year}-{month:02d}"
                month_nodes = [
                    node
                    for node in task_nodes
                    if node["due_date"].year == year and node["due_date"].month == month
                ]

                if not month_nodes:
                    monthly_status[key] = None
                else:
                    # Simple aggregation logic
                    statuses = [node["status"] for node in month_nodes]
                    # If all completed, show completed (green)
                    if all(s == "completed" for s in statuses):
                        monthly_status[key] = "completed"
                    # If any in_progress or mix of statuses, show in_progress (orange)
                    elif "in_progress" in statuses or "completed" in statuses:
                        monthly_status[key] = "in_progress"
                    # If all pending, show pending (red)
                    elif "pending" in statuses:
                        monthly_status[key] = "pending"
                    else:
                        monthly_status[key] = statuses[0] if statuses else None

            template_data["monthly_status"] = monthly_status

            if folder_id not in folders_dict:
                folders_dict[folder_id] = {
                    "folder_id": folder_id,
                    "folder_name": folder_name,
                    "tasks": [],
                }
            folders_dict[folder_id]["tasks"].append(template_data)

        # Convert to list and sort by folder name
        result = list(folders_dict.values())
        result.sort(key=lambda x: x["folder_name"])

        # Include date range metadata in response
        return Response(
            {
                "folders": result,
                "start_month": start_month,
                "start_year": start_year,
                "end_month": end_month,
                "end_year": end_year,
            }
        )

    @action(detail=False, name="Task templates per status")
    def per_status(self, request):
        data = task_template_per_status(request.user)
        return Response({"results": data})

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get Task Node status choices")
    def status(srlf, request):
        return Response(dict(TaskNode.TASK_STATUS_CHOICES))


class TaskNodeFilterSet(GenericFilterSet):
    past = df.BooleanFilter(method="filter_past")

    def filter_past(self, queryset, name, value):
        if value is True:
            return queryset.filter(due_date__lt=date.today()).order_by("due_date")
        elif value is False:
            return queryset.filter(due_date__gte=date.today()).order_by("due_date")
        return queryset

    class Meta:
        model = TaskNode
        fields = {
            "status": ["exact"],
            "task_template": ["exact"],
            "due_date": ["exact", "lte", "gte", "lt", "gt", "month", "year"],
        }


class TaskNodeViewSet(BaseModelViewSet):
    model = TaskNode
    filterset_class = TaskNodeFilterSet
    search_fields = ["observation"]
    ordering = ["due_date"]

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get Task Node status choices")
    def status(srlf, request):
        return Response(dict(TaskNode.TASK_STATUS_CHOICES))

    @action(
        detail=True, name="Remove/Move evidence to expected evidence", methods=["post"]
    )
    def remove_evidence(self, request, pk):
        task_node = TaskNode.objects.get(id=pk)
        evidence_id = request.data.get("evidence_id")
        to_move = request.data.get("move", False)
        evidence = Evidence.objects.get(id=evidence_id)
        task_node.evidences.remove(evidence)
        if to_move:
            task_node.task_template.evidences.add(evidence)
        return Response(status=status.HTTP_200_OK)

    def perform_create(self, serializer):
        instance: TaskNode = serializer.save()
        instance.save()
        return super().perform_create(serializer)


class TaskNodeEvidenceList(generics.ListAPIView):
    serializer_class = EvidenceReadSerializer
    filterset_fields = {
        "folder": ["exact"],
        "status": ["exact"],
        "owner": ["exact"],
        "name": ["icontains"],
        "expiry_date": ["exact", "lte", "gte"],
        "created_at": ["exact", "lte", "gte"],
        "updated_at": ["exact", "lte", "gte"],
    }
    search_fields = ["name", "description"]
    ordering_fields = ["name", "status", "updated_at", "expiry_date"]
    ordering = ["name"]

    filter_backends = [
        DjangoFilterBackend,
        filters.SearchFilter,
        filters.OrderingFilter,
    ]

    def get_serializer_context(self):
        context = super().get_serializer_context()
        context.update({"pk": self.kwargs["pk"]})
        return context

    def get_queryset(self):
        """RBAC not automatic as we don't inherit from BaseModelViewSet -> enforce it explicitly"""
        task_node_id = self.kwargs["pk"]

        if not RoleAssignment.is_object_readable(
            self.request.user,
            TaskNode,
            task_node_id,
        ):
            raise PermissionDenied()

        task_node = TaskNode.objects.get(id=task_node_id)

        # Get task node's template
        task_template = task_node.task_template

        # Get visible evidences to filter result
        viewable_evidences, _, _ = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), self.request.user, Evidence
        )
        return Evidence.objects.filter(
            id__in=task_template.evidences.filter(
                id__in=viewable_evidences
            ).values_list("id", flat=True)
        )


class TerminologyViewSet(BaseModelViewSet):
    model = Terminology
    filterset_fields = ["field_path", "folder", "is_visible", "builtin"]
    search_fields = ["name", "description"]
    ordering = ["field_path", "name"]

    @method_decorator(cache_page(60 * LONG_CACHE_TTL))
    @action(detail=False, name="Get class name choices")
    def field_path(self, request):
        return Response(dict(Terminology.FieldPath.choices))


class RequirementAssignmentViewSet(BaseModelViewSet):
    """
    API endpoint that allows requirement assignments to be viewed or edited.
    Requirement assignments delegate groups of requirement assessments to specific actors.
    """

    permission_overrides = {
        "set_status": "transition_requirementassignment",
        "requirements_list": "view_requirementassignment",
    }

    model = RequirementAssignment
    filterset_fields = [
        "folder",
        "compliance_assessment",
        "actor",
        "status",
    ]

    def get_queryset(self):
        qs = (
            super()
            .get_queryset()
            .select_related(
                "folder",
                "compliance_assessment",
            )
            .prefetch_related(
                "actor",
                "requirement_assessments",
                Prefetch(
                    "events",
                    queryset=RequirementAssignmentEvent.objects.select_related(
                        "event_actor"
                    ),
                ),
            )
        )
        auditee_folders = get_auditee_filtered_folder_ids(self.request.user)
        if auditee_folders:
            user_actors = Actor.get_all_for_user(self.request.user)
            qs = qs.filter(
                ~Q(folder_id__in=auditee_folders) | Q(actor__in=user_actors)
            ).distinct()
        return qs

    EDITABLE_STATUSES = ("draft", "in_progress")

    def update(self, request, *args, **kwargs):
        assignment = self.get_object()
        if assignment.status not in self.EDITABLE_STATUSES:
            return Response(
                {"error": f"Cannot edit assignment in '{assignment.status}' status."},
                status=status.HTTP_403_FORBIDDEN,
            )
        return super().update(request, *args, **kwargs)

    def partial_update(self, request, *args, **kwargs):
        assignment = self.get_object()
        if assignment.status not in self.EDITABLE_STATUSES:
            return Response(
                {"error": f"Cannot edit assignment in '{assignment.status}' status."},
                status=status.HTTP_403_FORBIDDEN,
            )
        return super().partial_update(request, *args, **kwargs)

    def destroy(self, request, *args, **kwargs):
        assignment = self.get_object()
        if assignment.status not in self.EDITABLE_STATUSES:
            return Response(
                {"error": f"Cannot delete assignment in '{assignment.status}' status."},
                status=status.HTTP_403_FORBIDDEN,
            )
        return super().destroy(request, *args, **kwargs)

    # Valid transitions: (from_status, to_status) → config
    # reviewer_only: auditee-only users are forbidden
    # actor_only: only assigned actors can perform this transition
    # check_completion: all assessable requirements must be assessed
    # observation: "clear" removes it, "optional" keeps if provided, "required" must be provided
    TRANSITIONS = {
        ("draft", "in_progress"): {"reviewer_only": True},
        ("in_progress", "submitted"): {
            "actor_only": True,
            "check_completion": True,
            "observation": "clear",
        },
        ("changes_requested", "submitted"): {
            "actor_only": True,
            "check_completion": True,
            "observation": "clear",
        },
        ("submitted", "closed"): {
            "reviewer_only": True,
            "observation": "optional",
        },
        ("submitted", "changes_requested"): {
            "reviewer_only": True,
            "observation": "required",
        },
        ("closed", "submitted"): {
            "reviewer_only": True,
            "observation": "clear",
        },
    }

    @action(detail=True, methods=["post"], url_path="set_status")
    def set_status(self, request, pk=None):
        """Transition assignment to a new status.

        Accepts {"status": "<target_status>", "reviewer_observation": "..."}.
        Valid transitions and their constraints are defined in TRANSITIONS.
        """
        assignment = self.get_object()
        target = request.data.get("status")
        if not target:
            return Response(
                {"error": "Missing 'status' field."},
                status=status.HTTP_400_BAD_REQUEST,
            )

        key = (assignment.status, target)
        config = self.TRANSITIONS.get(key)
        if config is None:
            return Response(
                {
                    "error": f"Invalid transition from '{assignment.status}' to '{target}'."
                },
                status=status.HTTP_400_BAD_REQUEST,
            )

        # Reviewer-only check: auditee-only users are forbidden
        if config.get("reviewer_only"):
            auditee_folders = get_auditee_filtered_folder_ids(request.user)
            if (
                auditee_folders
                and assignment.compliance_assessment.folder_id in auditee_folders
            ):
                return Response(
                    {"error": "Auditee users cannot perform this action."},
                    status=status.HTTP_403_FORBIDDEN,
                )

        # Actor-only check: user must be an assigned actor
        if config.get("actor_only"):
            user_actors = Actor.get_all_for_user(request.user)
            if not assignment.actor.filter(id__in=[a.id for a in user_actors]).exists():
                return Response(
                    {"error": "You are not assigned to this assignment."},
                    status=status.HTTP_403_FORBIDDEN,
                )

        # Completion check: all assessable requirements must be assessed
        if config.get("check_completion"):
            unassessed_count = assignment.requirement_assessments.filter(
                requirement__assessable=True, result="not_assessed"
            ).count()
            if unassessed_count > 0:
                return Response(
                    {
                        "error": f"{unassessed_count} requirement(s) have not been assessed yet.",
                        "unassessed_count": unassessed_count,
                    },
                    status=status.HTTP_400_BAD_REQUEST,
                )

        # Handle reviewer_observation (stored as event_notes)
        obs_rule = config.get("observation")
        observation = request.data.get("reviewer_observation")
        if obs_rule == "required" and not observation:
            return Response(
                {"error": "Reviewer observation is required for this transition."},
                status=status.HTTP_400_BAD_REQUEST,
            )

        # Apply transition
        assignment.status = target
        assignment.save(update_fields=["status"])

        # Create event for traceability
        RequirementAssignmentEvent.objects.create(
            assignment=assignment,
            event_type=target,
            event_actor=request.user,
            event_notes=observation if obs_rule in ("required", "optional") else None,
            folder=assignment.folder,
        )

        # Send notification
        try:
            self._send_transition_notification(assignment, key, observation or "")
        except Exception:
            logger.error(
                "Failed to send assignment notification",
                assignment_id=str(assignment.id),
                transition=f"{key[0]} -> {key[1]}",
                exc_info=True,
            )

        return Response({"status": target})

    @action(detail=True, methods=["get"])
    def requirements_list(self, request, pk=None):
        """Returns the scoped requirements list for this assignment.

        Authorization is enforced by get_queryset() which filters
        auditee-only users to their own assignments.
        """
        assignment = self.get_object()
        compliance_assessment = assignment.compliance_assessment

        assigned_ra_ids = set(
            assignment.requirement_assessments.values_list("id", flat=True)
        )

        requirement_assessments_objects = list(
            compliance_assessment.get_requirement_assessments(
                include_non_assessable=True
            )
        )
        requirement_assessments_objects = [
            ra for ra in requirement_assessments_objects if ra.id in assigned_ra_ids
        ]

        requirements_objects = list(
            RequirementNode.objects.filter(framework=compliance_assessment.framework)
            .select_related("framework")
            .prefetch_related("reference_controls", "threats")
        )
        nodes_by_urn = {node.urn: node for node in requirements_objects}
        for node in requirements_objects:
            parent = nodes_by_urn.get(node.parent_urn)
            if parent:
                node._parent_requirement_obj = parent
        for ra in requirement_assessments_objects:
            req = getattr(ra, "requirement", None)
            if req:
                parent = nodes_by_urn.get(req.parent_urn)
                if parent:
                    req._parent_requirement_obj = parent

        requirement_assessments = RequirementAssessmentReadSerializer(
            requirement_assessments_objects, many=True
        ).data
        requirements = RequirementNodeReadSerializer(
            requirements_objects, many=True
        ).data

        return Response(
            {
                "requirements": requirements,
                "requirement_assessments": requirement_assessments,
            },
            status=status.HTTP_200_OK,
        )

    @staticmethod
    def _send_transition_notification(assignment, transition_key, observation=""):
        from_status, to_status = transition_key
        if to_status == "in_progress":
            from core.tasks import send_assignment_activated_notification

            send_assignment_activated_notification(assignment.id)
        elif to_status == "submitted" and from_status in (
            "in_progress",
            "changes_requested",
        ):
            from core.tasks import send_assignment_submitted_notification

            send_assignment_submitted_notification(assignment.id)
        elif to_status in ("closed", "changes_requested") or (
            to_status == "submitted" and from_status == "closed"
        ):
            from core.tasks import send_assignment_reviewed_notification

            decision = "reopened" if from_status == "closed" else to_status
            send_assignment_reviewed_notification(assignment.id, decision, observation)


# ---------------------------------------------------------------------------
# Universal Search
# ---------------------------------------------------------------------------


def _search_entry(model, slug, *, ref_id=False, limit=200, extra_search=None):
    """Helper to build a search config entry."""
    return {
        "model": model,
        "slug": slug,
        "ref_id": ref_id,
        "limit": limit,
        # Additional fields to search on (icontains) beyond name/description/ref_id.
        # These are ORM lookup paths, e.g. "framework__name" for a FK join.
        "extra_search": extra_search or [],
    }


SEARCHABLE_MODELS = [
    # limit: per-model cap on broad fetch. Large tables (RequirementNode,
    # ReferenceControl) get a tighter limit to avoid starving smaller models.
    # extra_search: additional fields to query via icontains for this model.
    # --- Organization ---
    _search_entry(Folder, "folders"),
    _search_entry(Perimeter, "perimeters", ref_id=True),
    # --- Catalog ---
    _search_entry(Framework, "frameworks", ref_id=True),
    _search_entry(Threat, "threats", ref_id=True, limit=100, extra_search=["provider"]),
    _search_entry(ReferenceControl, "reference-controls", ref_id=True, limit=100),
    _search_entry(RiskMatrix, "risk-matrices", ref_id=True, limit=50),
    _search_entry(RequirementNode, "requirement-nodes", ref_id=True, limit=100),
    # --- Assets ---
    _search_entry(Asset, "assets", ref_id=True),
    _search_entry(Vulnerability, "vulnerabilities", ref_id=True, limit=100),
    # --- Operations ---
    _search_entry(AppliedControl, "applied-controls", ref_id=True),
    _search_entry(Policy, "policies", ref_id=True),
    _search_entry(Incident, "incidents", ref_id=True),
    _search_entry(Finding, "findings", ref_id=True),
    _search_entry(SecurityException, "security-exceptions", ref_id=True),
    _search_entry(TaskTemplate, "task-templates", ref_id=True, limit=100),
    _search_entry(Evidence, "evidences"),
    # --- Governance ---
    _search_entry(RiskAcceptance, "risk-acceptances"),
    # --- Risk ---
    _search_entry(RiskAssessment, "risk-assessments", ref_id=True),
    _search_entry(
        RiskScenario,
        "risk-scenarios",
        ref_id=True,
        extra_search=["risk_assessment__name"],
    ),
    # --- Compliance ---
    _search_entry(
        ComplianceAssessment,
        "compliance-assessments",
        ref_id=True,
        extra_search=["framework__name"],
    ),
    # --- TPRM ---
    _search_entry(Entity, "entities", ref_id=True),
    _search_entry(Solution, "solutions", extra_search=["provider_entity__name"]),
    _search_entry(Contract, "contracts", ref_id=True),
    # --- EBIOS RM ---
    _search_entry(EbiosRMStudy, "ebios-rm"),
    _search_entry(FearedEvent, "feared-events"),
    _search_entry(StrategicScenario, "strategic-scenarios"),
    _search_entry(AttackPath, "attack-paths"),
    # --- Privacy ---
    _search_entry(Processing, "processings", ref_id=True),
    _search_entry(DataBreach, "data-breaches", ref_id=True),
    _search_entry(RightRequest, "right-requests", ref_id=True),
    # --- Resilience ---
    _search_entry(BusinessImpactAnalysis, "business-impact-analysis"),
]


_ACCENT_MAP = {
    "a": "[aàáâãäåæ]",
    "e": "[eèéêë]",
    "i": "[iìíîï]",
    "o": "[oòóôõöø]",
    "u": "[uùúûü]",
    "c": "[cç]",
    "n": "[nñ]",
    "y": "[yýÿ]",
    "s": "[sß]",
}


def _accent_regex(word: str) -> str:
    """Convert a word to a regex pattern that matches accented variants.

    E.g. "referentiel" -> "r[eèéêë]f[eèéêë]r[eèéêë][nñ]t[iìíîï][eèéêë]l"
    Works on both SQLite and PostgreSQL with __iregex.
    """
    return "".join(_ACCENT_MAP.get(c, re.escape(c)) for c in word.lower())


@api_view(["GET"])
@permission_classes([permissions.IsAuthenticated])
def global_search(request):
    """
    Universal fuzzy search across all searchable models.

    GET /api/search/?q=firewall+policy&type=applied-controls,assets
    """
    from rapidfuzz import fuzz

    EMPTY_RESPONSE = {"results": [], "query": "", "count": 0, "total_candidates": 0}

    # Cap query length to avoid excessive icontains processing across all models.
    # 200 chars is far beyond any realistic search; words capped at 20 since each
    # word generates 2-3 Q objects per model (~1000+ total at the limit).
    # Minimum 2 chars to avoid single-character fan-out (e.g. q=a matching everything).
    query = request.query_params.get("q", "").strip()[:200]
    if len(query) < 2:
        return Response(EMPTY_RESPONSE)

    type_filter = request.query_params.get("type", "")
    allowed_types = set(type_filter.split(",")) if type_filter else None

    words = query.split()[:20]
    # Build prefix set for typo-tolerant broad fetch (first 3 chars of each word)
    prefixes = {w[:3].lower() for w in words if len(w) >= 3}

    candidates = []

    for entry in SEARCHABLE_MODELS:
        model_class = entry["model"]
        url_slug = entry["slug"]
        has_ref_id = entry["ref_id"]
        max_per_model = entry["limit"]
        extra_search = entry["extra_search"]

        if allowed_types and url_slug not in allowed_types:
            continue

        # Permission-aware queryset
        accessible_ids = RoleAssignment.get_accessible_object_ids(
            Folder.get_root_folder(), request.user, model_class
        )[0]
        qs = model_class.objects.filter(id__in=accessible_ids)

        # ComplianceAssessment has extra auditee scoping: users with only the
        # auditee role in a folder can only see assessments where they have a
        # requirement assignment. Mirror the logic from ComplianceAssessmentViewSet.
        if model_class is ComplianceAssessment:
            auditee_folders = get_auditee_filtered_folder_ids(request.user)
            if auditee_folders:
                user_actors = Actor.get_all_for_user(request.user)
                qs = qs.filter(
                    ~Q(folder_id__in=auditee_folders)
                    | Q(requirement_assignments__actor__in=user_actors)
                ).distinct()

        # Build Q filter for each word on searchable fields.
        # Uses iregex with accent-folding character classes so that e.g.
        # "referentiel" matches "RÉFÉRENTIEL" on SQLite (whose LIKE is
        # ASCII-only and can't fold accents).
        field_names = {f.name for f in model_class._meta.get_fields()}
        searchable = ["name", "description"]
        if has_ref_id:
            searchable.append("ref_id")
        # Models with i18n translations store localized names/descriptions in a
        # JSONField. Searching it catches all translated variants at once.
        if "translations" in field_names:
            searchable.append("translations")
        searchable.extend(extra_search)

        q_filter = Q()
        for word in words:
            pattern = _accent_regex(word)
            word_q = Q()
            for field in searchable:
                word_q |= Q(**{f"{field}__iregex": pattern})
            q_filter |= word_q

        # Also add prefix-based matching for typo tolerance (name + ref_id only)
        for prefix in prefixes:
            prefix_pattern = _accent_regex(prefix)
            prefix_q = Q(**{"name__iregex": prefix_pattern})
            if has_ref_id:
                prefix_q |= Q(**{"ref_id__iregex": prefix_pattern})
            q_filter |= prefix_q

        # Detect optional display fields present on this model (reuses field_names from above)
        extra_fields = []
        if has_ref_id:
            extra_fields.append("ref_id")
        if "folder" in field_names:
            extra_fields.append("folder__name")
        if "provider_entity" in field_names:
            extra_fields.append("provider_entity__name")
        if model_class is RequirementNode:
            extra_fields.append("framework_id")

        results = qs.filter(q_filter).values(
            "id",
            "name",
            "description",
            *extra_fields,
        )[:max_per_model]

        for row in results:
            candidates.append(
                {
                    "type": url_slug,
                    "id": str(row["id"]),
                    "name": row["name"] or "",
                    "ref_id": row.get("ref_id", "") or "",
                    "description": row.get("description") or "",
                    "folder": row.get("folder__name", "") or "",
                    "provider": row.get("provider_entity__name", "") or "",
                    "framework_id": str(row["framework_id"])
                    if row.get("framework_id")
                    else None,
                }
            )

    # Strip accents/diacritics for accent-insensitive matching and scoring.
    # SQLite's LIKE is only ASCII case-insensitive, and rapidfuzz treats
    # accented chars as distinct — normalizing levels the playing field.
    import unicodedata

    def strip_accents(s: str) -> str:
        return "".join(
            c
            for c in unicodedata.normalize("NFD", s)
            if unicodedata.category(c) != "Mn"
        ).lower()

    query_norm = strip_accents(query)

    # Score and rank with rapidfuzz
    for candidate in candidates:
        name_norm = strip_accents(candidate["name"])
        ref_norm = strip_accents(candidate["ref_id"])
        desc_norm = strip_accents(candidate["description"])

        # Fuzzy scores (on normalized strings for accent-insensitive comparison)
        name_score = fuzz.WRatio(query_norm, name_norm)
        ref_score = fuzz.WRatio(query_norm, ref_norm) if ref_norm else 0
        desc_score = fuzz.partial_ratio(query_norm, desc_norm) * 0.4

        # Substring bonus: literal match in name or ref_id should rank very
        # high. WRatio penalizes long names unfairly (e.g. "recyf" vs
        # "RECYF : REFERENTIEL CYBER France..." scores only 24).
        substring_bonus = 0
        if query_norm in name_norm:
            substring_bonus = 95 if name_norm.startswith(query_norm) else 90
        if query_norm in ref_norm:
            substring_bonus = max(substring_bonus, 95)

        candidate["score"] = max(name_score, ref_score, desc_score, substring_bonus)

    # Sort by score descending, return top 50
    candidates.sort(key=lambda c: c["score"], reverse=True)
    top_results = candidates[:50]

    # Build final response: add URLs and truncate descriptions for the wire
    for r in top_results:
        if r["type"] == "requirement-nodes" and r.get("framework_id"):
            # Link to parent framework (no standalone requirement detail page yet)
            r["url"] = f"/frameworks/{r['framework_id']}"
        else:
            r["url"] = f"/{r['type']}/{r['id']}"
        r["description"] = r["description"][:200]

    return Response(
        {
            "results": top_results,
            "query": query,
            "count": len(top_results),
            "total_candidates": len(candidates),
        }
    )