Simplify DEFAULT_IOC_TYPES to common indicators (#340)

* Simplify DEFAULT_IOC_TYPES to common indicators

Reduce DEFAULT_IOC_TYPES from 30 entries down to 9 common indicator
types (domains, urls, ipv4s, ipv6s, email_addresses, md5s, sha1s,
sha256s, cves) to speed parsing in the typical case. Add a new
SUPPORTED_IOC_TYPES constant exposing the full list of parseable types
so callers can opt back into the obscure types via included_ioc_types.

This is a breaking change in the find_iocs() default behavior.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

* Improve cli and fix lint errors

---------

Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>

Author: fhightower

ioc_finder/__init__.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 
 from .ioc_finder import (
+    DEFAULT_IOC_TYPES,
+    SUPPORTED_IOC_TYPES,
     find_iocs,
     parse_asns,
     parse_authentihashes_,
@@ -37,6 +39,8 @@
 )
 
 __all__ = [
+    "DEFAULT_IOC_TYPES",
+    "SUPPORTED_IOC_TYPES",
     "find_iocs",
     "parse_asns",
     "parse_authentihashes_",

ioc_finder/ioc_finder.py

@@ -126,7 +126,7 @@
 # using `Mapping` b/c it is covariant (https://mypy.readthedocs.io/en/stable/generics.html#variance-of-generic-types)
 IndicatorData = Mapping[str, IndicatorList | IndicatorDict]
 
-DEFAULT_IOC_TYPES = [
+SUPPORTED_IOC_TYPES = [
     "asns",
     "attack_mitigations",
     "attack_tactics",
@@ -159,6 +159,18 @@
     "xmpp_addresses",
 ]
 
+DEFAULT_IOC_TYPES = [
+    "cves",
+    "domains",
+    "email_addresses",
+    "ipv4s",
+    "ipv6s",
+    "md5s",
+    "sha1s",
+    "sha256s",
+    "urls",
+]
+
 
 def _deduplicate(indicator_list: Iterable) -> list:
     """Deduplicate the list of observables."""
@@ -555,6 +567,12 @@ def parse_tlp_labels(text):
     help="Using this flag will parse URLs with and without a scheme (default is True)",
     default=True,
 )
+@click.option(
+    "--all",
+    "parse_all",
+    is_flag=True,
+    help="Parse every supported indicator type instead of the common defaults.",
+)
 def cli_find_iocs(
     text,
     no_url_domain_parsing,
@@ -563,6 +581,7 @@ def cli_find_iocs(
     no_cidr_address_parsing,
     no_xmpp_addr_domain_parsing,
     parse_urls_without_scheme,
+    parse_all,
 ):
     """CLI interface for parsing observables."""
     stdin_text = click.get_text_stream("stdin")
@@ -572,6 +591,7 @@ def cli_find_iocs(
         text = "\n".join(stdin_text)
         # text = '\n'.join([line for line in stdin_text])
 
+    included_ioc_types = list(SUPPORTED_IOC_TYPES if parse_all else DEFAULT_IOC_TYPES)
     iocs = find_iocs(
         text,
         parse_domain_from_url=not no_url_domain_parsing,
@@ -580,6 +600,7 @@ def cli_find_iocs(
         parse_address_from_cidr=not no_cidr_address_parsing,
         parse_domain_name_from_xmpp_address=not no_xmpp_addr_domain_parsing,
         parse_urls_without_scheme=parse_urls_without_scheme,
+        included_ioc_types=included_ioc_types,
     )
     ioc_string = json.dumps(iocs, indent=4, sort_keys=True)
     print(ioc_string)
@@ -613,9 +634,10 @@ def find_iocs(
             addresses. Only applicable when ``"domains"`` is in ``included_ioc_types``.
         parse_urls_without_scheme: Whether to parse URLs without a scheme. Only applicable
             when ``"urls"`` or ``"urls_complete"`` is in ``included_ioc_types``.
-        included_ioc_types: Collection of IOC type names to parse. If ``None``, all
-            default types are parsed. See ``DEFAULT_IOC_TYPES`` for valid values.
-            When specified, the boolean options above only take effect if their
+        included_ioc_types: Collection of IOC type names to parse. If ``None``,
+            the common default types are parsed (see ``DEFAULT_IOC_TYPES``). For
+            the full list of parseable types, see ``SUPPORTED_IOC_TYPES``. When
+            specified, the boolean options above only take effect if their
             corresponding IOC type is included.
     """
     if included_ioc_types is None:

tests/find_iocs_cases/feature__included_ioc_types.py

@@ -5,7 +5,7 @@
 
 from pytest import param
 
-from ioc_finder.ioc_finder import DEFAULT_IOC_TYPES
+from ioc_finder.ioc_finder import SUPPORTED_IOC_TYPES
 
 IOC_EXAMPLES = {
     "domains": ["abc.py", "bar.com", "example.com", "foo.com", "swissjabber.de"],
@@ -67,7 +67,7 @@
 
 individual_included_ioc_types_tests = []
 
-for type_ in DEFAULT_IOC_TYPES:
+for type_ in SUPPORTED_IOC_TYPES:
     individual_included_ioc_types_tests.append(
         param(
             all_ioc_text,

tests/test_cli.py

@@ -31,7 +31,7 @@ def test_cli_without_domain_from_url_parsing():
     runner = CliRunner()
     result = runner.invoke(
         ioc_finder.cli_find_iocs,
-        ["This is just an example.com https://example.org/test/bingo.php", "--no_url_domain_parsing"],
+        ["This is just an example.com https://example.org/test/bingo.php", "--no_url_domain_parsing", "--all"],
     )
     assert result.exit_code == 0
     print(result.output.strip())
@@ -115,7 +115,7 @@ def test_cli_disabling_parsing_urls_without_scheme():
 
 def test_cli_parses_imphashes_by_default():
     runner = CliRunner()
-    result = runner.invoke(ioc_finder.cli_find_iocs, ["imphash 18ddf28a71089acdbab5038f58044c0a"])
+    result = runner.invoke(ioc_finder.cli_find_iocs, ["imphash 18ddf28a71089acdbab5038f58044c0a", "--all"])
     assert result.exit_code == 0
     json_results = json.loads(result.output.strip())
     assert json_results["imphashes"] == ["18ddf28a71089acdbab5038f58044c0a"]

tests/test_edge_cases.py

@@ -2,8 +2,13 @@
 
 import pytest
 
-from ioc_finder import find_iocs
-from ioc_finder.ioc_finder import DEFAULT_IOC_TYPES
+from ioc_finder import find_iocs as _find_iocs
+from ioc_finder.ioc_finder import SUPPORTED_IOC_TYPES
+
+
+def find_iocs(*args, **kwargs):
+    kwargs.setdefault("included_ioc_types", SUPPORTED_IOC_TYPES)
+    return _find_iocs(*args, **kwargs)
 
 
 @pytest.fixture
@@ -490,14 +495,14 @@ def test_google_casing_deduplication():
 def test_excluding_imphashes_from_included_ioc_types():
     """Omitting 'imphashes' from included_ioc_types disables imphash parsing."""
     s = "imphash 18ddf28a71089acdbab5038f58044c0a"
-    types_without_imphashes = [t for t in DEFAULT_IOC_TYPES if t != "imphashes"]
+    types_without_imphashes = [t for t in SUPPORTED_IOC_TYPES if t != "imphashes"]
     iocs = find_iocs(s, included_ioc_types=types_without_imphashes)
     assert "imphashes" not in iocs
 
 
 def test_excluding_authentihashes_from_included_ioc_types():
     s = "authentihash 3f1b149d07e7e8636636b8b7f7043c40ed64a10b28986181fb046c498432c2d4"
-    types_without_authentihashes = [t for t in DEFAULT_IOC_TYPES if t != "authentihashes"]
+    types_without_authentihashes = [t for t in SUPPORTED_IOC_TYPES if t != "authentihashes"]
     iocs = find_iocs(s, included_ioc_types=types_without_authentihashes)
     assert "authentihashes" not in iocs
 

tests/test_find_iocs.py

@@ -2,12 +2,17 @@
 
 import pytest
 
-from ioc_finder import find_iocs
-from ioc_finder.ioc_finder import IndicatorDict, IndicatorList
+from ioc_finder import find_iocs as _find_iocs
+from ioc_finder.ioc_finder import SUPPORTED_IOC_TYPES, IndicatorDict, IndicatorList
 
 from .find_iocs_cases import ALL_TESTS
 
 
+def find_iocs(*args, **kwargs):
+    kwargs.setdefault("included_ioc_types", SUPPORTED_IOC_TYPES)
+    return _find_iocs(*args, **kwargs)
+
+
 @pytest.mark.parametrize("text, results, args", ALL_TESTS)
 def test_find_iocs(text: str, results: Dict, args: Dict) -> None:
     # Parse input

tests/test_ioc_finder.py

@@ -1,4 +1,10 @@
-from ioc_finder import find_iocs
+from ioc_finder import find_iocs as _find_iocs
+from ioc_finder.ioc_finder import SUPPORTED_IOC_TYPES
+
+
+def find_iocs(*args, **kwargs):
+    kwargs.setdefault("included_ioc_types", SUPPORTED_IOC_TYPES)
+    return _find_iocs(*args, **kwargs)
 
 
 def test_tlp_labels():

tests/test_urls.py

@@ -2,7 +2,14 @@
 
 from d8s_lists import iterables_have_same_items
 
-from ioc_finder import find_iocs
+from ioc_finder import find_iocs as _find_iocs
+from ioc_finder.ioc_finder import SUPPORTED_IOC_TYPES
+
+
+def find_iocs(*args, **kwargs):
+    kwargs.setdefault("included_ioc_types", SUPPORTED_IOC_TYPES)
+    return _find_iocs(*args, **kwargs)
+
 
 # VALID_URLS = [
 #     'http://foo.com/blah_blah',