Merge branch 'master' into feat/ability-yaml-upload

Author: fionamccrae

.gitignore

@@ -20,6 +20,7 @@ conf/*.yml
 !conf/default.yml
 data/object_store
 data/fact_store
+data/cookie_storage
 data/results/*
 !data/results/.gitkeep
 data/payloads/*

app/api/v2/handlers/payload_api.py

@@ -4,6 +4,7 @@
 import pathlib
 import re
 from io import IOBase
+from typing import Optional
 
 import aiohttp_apispec
 from aiohttp import web
@@ -27,14 +28,16 @@ def add_routes(self, app: web.Application):
 
     @aiohttp_apispec.docs(tags=['payloads'],
                           summary='Retrieve payloads',
-                          description='Retrieves all stored payloads.')
+                          description='Retrieves all stored payloads. Supports optional filtering by name '
+                                      '(case-insensitive substring match via the `name` query parameter).')
     @aiohttp_apispec.querystring_schema(PayloadQuerySchema)
     @aiohttp_apispec.response_schema(PayloadSchema(),
                                      description='Returns a list of all payloads in PayloadSchema format.')
     async def get_payloads(self, request: web.Request):
         sort: bool = request['querystring'].get('sort')
         exclude_plugins: bool = request['querystring'].get('exclude_plugins')
         add_path: bool = request['querystring'].get('add_path')
+        name_filter: Optional[str] = request['querystring'].get('name')
 
         cwd = pathlib.Path.cwd()
         payload_dirs = [cwd / 'data' / 'payloads']
@@ -52,6 +55,11 @@ async def get_payloads(self, request: web.Request):
         }
 
         payloads = list(payloads)
+
+        if name_filter:
+            name_filter_lower = name_filter.lower()
+            payloads = [p for p in payloads if name_filter_lower in pathlib.PurePath(p).name.lower()]
+
         if sort:
             payloads.sort()
 

app/api/v2/managers/base_api_manager.py

@@ -1,12 +1,14 @@
 import logging
 import os
+import re
 import uuid
 import yaml
 
 from marshmallow.schema import SchemaMeta
 from typing import Any, List
 from base64 import b64encode, b64decode
 
+from app.api.v2.errors import DataValidationError
 from app.utility.base_world import BaseWorld
 
 
@@ -64,6 +66,7 @@ def create_object_from_schema(self, schema: SchemaMeta, data: dict, access: Base
 
     async def create_on_disk_object(self, data: dict, access: dict, ram_key: str, id_property: str, obj_class: type):
         obj_id = data.get(id_property) or str(uuid.uuid4())
+        obj_id = self._sanitize_id(obj_id)
         data[id_property] = obj_id
 
         file_path = await self._get_new_object_file_path(data[id_property], ram_key)
@@ -121,18 +124,34 @@ async def remove_object_from_memory_by_id(self, identifier: str, ram_key: str, i
         await self._data_svc.remove(ram_key, {id_property: identifier})
 
     async def remove_object_from_disk_by_id(self, identifier: str, ram_key: str):
+        identifier = self._sanitize_id(identifier)
         file_path = await self._get_existing_object_file_path(identifier, ram_key)
 
         if os.path.exists(file_path):
             os.remove(file_path)
 
+    @staticmethod
+    def _sanitize_id(obj_id) -> str:
+        '''Removes any non-alphanumeric characters and non-hyphen/underscore.'''
+        if not isinstance(obj_id, str):
+            raise DataValidationError(message=f'Invalid id type: expected str, got {type(obj_id).__name__}', name='id', value=obj_id)
+        original_id = obj_id
+        obj_id = re.sub(r'[^a-zA-Z0-9_-]', '', obj_id)
+        if not obj_id:
+            raise DataValidationError(message=f"Invalid id: {obj_id!r}", name='id', value=obj_id)
+        if original_id != obj_id:
+            logging.getLogger(DEFAULT_LOGGER_NAME).warning(f"Sanitized ID: {obj_id}")
+        return obj_id
+
     @staticmethod
     async def _get_new_object_file_path(identifier: str, ram_key: str) -> str:
         """Create file path for new object"""
+        identifier = BaseApiManager._sanitize_id(identifier)
         return os.path.join('data', ram_key, f'{identifier}.yml')
 
     async def _get_existing_object_file_path(self, identifier: str, ram_key: str) -> str:
         """Find file path for existing object (by id)"""
+        identifier = self._sanitize_id(identifier)
         _, file_path = await self._file_svc.find_file_path(f'{identifier}.yml', location=ram_key)
         if not file_path:
             file_path = await self._get_new_object_file_path(identifier, ram_key)

app/api/v2/schemas/payload_schemas.py

@@ -5,6 +5,7 @@ class PayloadQuerySchema(schema.Schema):
     sort = fields.Boolean(required=False, load_default=False)
     exclude_plugins = fields.Boolean(required=False, load_default=False)
     add_path = fields.Boolean(required=False, load_default=False)
+    name = fields.String(required=False, load_default=None, allow_none=True)
 
 
 class PayloadSchema(schema.Schema):

app/service/auth_svc.py

@@ -1,6 +1,6 @@
-import base64
 from collections import namedtuple
 from importlib import import_module
+import os
 
 from aiohttp import web, web_request
 from aiohttp.web_exceptions import HTTPUnauthorized, HTTPForbidden
@@ -10,7 +10,6 @@
 from aiohttp_security.abc import AbstractAuthorizationPolicy
 from aiohttp_session import setup as setup_session
 from aiohttp_session.cookie_storage import EncryptedCookieStorage
-from cryptography import fernet
 
 from app.service.interfaces.i_auth_svc import AuthServiceInterface
 from app.service.interfaces.i_login_handler import LoginHandlerInterface
@@ -73,9 +72,35 @@ async def apply(self, app, users):
                 for username, password in user.items():
                     await self.create_user(username, password, group)
         app.user_map = self.user_map
-        fernet_key = fernet.Fernet.generate_key()
-        secret_key = base64.urlsafe_b64decode(fernet_key)
-        storage = EncryptedCookieStorage(secret_key, cookie_name=COOKIE_SESSION)
+        cookie_file = 'cookie_storage'
+        expiration_days = self.get_config('session_expiration_days')
+        file_svc = self.get_service('file_svc')
+        cookie_path = os.path.join('data', cookie_file)
+
+        # Safely calculate max_age in seconds, allowing for fractional days
+        try:
+            max_age = int(float(expiration_days) * 86400) if expiration_days else None
+        except (ValueError, TypeError):
+            max_age = None
+        try:
+            if os.path.exists(cookie_path):
+                secret_key = file_svc._read(cookie_path)
+                self.log.debug('Loaded persistent session key from data/cookie_storage')
+            else:
+                # Generate a new random 32-byte key for AES encryption if no valid key is found in the config or data folder
+                secret_key = os.urandom(32)
+                file_svc._save(cookie_path, secret_key, encrypt=True)
+                self.log.debug('Generated and saved new persistent session key.')
+        except Exception as e:
+            # Fallback if file operations fail
+            self.log.warning('Could not manage persistent key file, falling back to ephemeral: %s', e)
+            secret_key = os.urandom(32)
+        if len(secret_key) != 32:
+            secret_key = os.urandom(32)
+            self.log.warning('Loaded session key is not 32 bytes long. Generating new key.')
+
+        # Pass max_age to the storage initializer
+        storage = EncryptedCookieStorage(secret_key, cookie_name=COOKIE_SESSION, max_age=max_age)
         setup_session(app, storage)
         policy = SessionIdentityPolicy()
         setup_security(app, policy, DictionaryAuthorizationPolicy(self.user_map))

app/service/data_svc.py

@@ -35,6 +35,7 @@
     'data/results/*',
     'data/sources/*',
     'data/object_store',
+    'data/cookie_storage',
 )
 
 PAYLOADS_CONFIG_STANDARD_KEY = 'standard_payloads'

conf/default.yml

@@ -27,6 +27,7 @@ app.contact.websocket: 0.0.0.0:7012
 auth.login.handler.module: default
 crypt_salt: REPLACE_WITH_RANDOM_VALUE
 encryption_key: ADMIN123
+session_expiration_days: 7
 exfil_dir: /tmp/caldera
 host: 0.0.0.0
 objects.planners.default: atomic

package-lock.json

@@ -1,12 +1,12 @@
 aiohttp-jinja2==1.5.1
-aiohttp==3.13.3
+aiohttp==3.13.4
 aiohttp_session==2.12.0
 aiohttp-security==0.4.0
 aiohttp-apispec==3.0.0b2
 argon2-cffi==25.1.0
 jinja2==3.1.6
 pyyaml==6.0.1
-cryptography==46.0.5
+cryptography==46.0.7
 websockets==15.0
 Sphinx==7.1.2
 sphinx_rtd_theme==1.3.0
@@ -15,15 +15,15 @@ marshmallow==3.26.2
 dirhash==0.2.1
 marshmallow-enum==1.5.1
 ldap3==2.9.1
-pyasn1~=0.5.1
+pyasn1==0.6.3
 reportlab==4.0.4  # debrief
 rich==13.7.0
 lxml==6.0.2  # debrief
 svglib==1.5.1  # debrief
 Markdown==3.8.1  # training
 dnspython==2.6.1
 asyncssh==2.20.0
-aioftp~=0.20.0
+aioftp==0.27.2
 packaging==23.2
 croniter~=3.0.3
 setuptools==78.1.1

requirements.txt

@@ -1,4 +1,5 @@
 import os
+import pathlib
 import tempfile
 from http import HTTPStatus
 
@@ -49,6 +50,35 @@ async def test_get_payloads(self, api_v2_client, api_cookies, expected_payload_f
 
         assert filtered_payload_file_names == expected_payload_file_names
 
+    @pytest.mark.parametrize('query_name', ['payload_', 'PAYLOAD_'])
+    async def test_get_payloads_name_filter(self, api_v2_client, api_cookies, expected_payload_file_names, query_name):
+        resp = await api_v2_client.get(f'/api/v2/payloads?name={query_name}', cookies=api_cookies)
+        assert resp.status == HTTPStatus.OK
+        payload_file_names = await resp.json()
+
+        # All expected payloads should be present
+        assert expected_payload_file_names <= set(payload_file_names)
+        # Every returned payload must match the filter (no false positives)
+        assert all('payload_' in pathlib.PurePath(p).name.lower() for p in payload_file_names)
+
+    async def test_get_payloads_name_filter_no_match(self, api_v2_client, api_cookies):
+        resp = await api_v2_client.get('/api/v2/payloads?name=__no_match_xyzzy__', cookies=api_cookies)
+        assert resp.status == HTTPStatus.OK
+        assert await resp.json() == []
+
+    async def test_get_payloads_name_filter_with_sort_and_add_path(
+            self, api_v2_client, api_cookies, expected_payload_file_names):
+        resp = await api_v2_client.get('/api/v2/payloads?name=payload_&sort=true&add_path=true', cookies=api_cookies)
+        assert resp.status == HTTPStatus.OK
+        payload_paths = await resp.json()
+
+        # Results should be sorted
+        assert payload_paths == sorted(payload_paths)
+        # Every returned path's filename must match the filter
+        assert all('payload_' in pathlib.PurePath(p).name.lower() for p in payload_paths)
+        # Results should contain paths (not bare filenames)
+        assert all(os.sep in p or '/' in p for p in payload_paths)
+
     async def test_unauthorized_get_payloads(self, api_v2_client):
         resp = await api_v2_client.get('/api/v2/payloads')
         assert resp.status == HTTPStatus.UNAUTHORIZED