add sample results, minor fixes

omkar-334 · omkar-334 · commit c84f59de496e · 2025-08-22T00:42:06.000+05:30
diff --git a/benchmark/main.py b/benchmark/main.py
@@ -1,46 +1,18 @@
 # Import benchmark modules
-from benchmark.operators import (
-    classify_video_zero_shot,
-    cluster_embeddings,
-    detect_lewd_images,
-    detect_text_in_image_tesseract,
-    dimension_reduction,
-    image_vec_rep_resnet,
-    vid_vec_rep_clip,
-    video_hash_tmk,
-)
+from benchmark.operators import all_operators
 from benchmark.report import BenchmarkReport
 
 
 def main():
     report = BenchmarkReport()
 
-    # List of benchmark modules to run
-    benchmark_modules = [
-        ("vid_vec_rep_clip", vid_vec_rep_clip),
-        ("video_hash_tmk", video_hash_tmk),
-        ("image_vec_rep_resnet", image_vec_rep_resnet),
-        ("detect_text_in_image_tesseract", detect_text_in_image_tesseract),
-        ("detect_lewd_images", detect_lewd_images),
-        ("classify_video_zero_shot", classify_video_zero_shot),
-        ("dimension_reduction", dimension_reduction),
-        ("cluster_embeddings", cluster_embeddings),
-    ]
-
-    for name, module in benchmark_modules:
-        print(f"\n=== Benchmarking {name} ===")
-        results = module.benchmark()
-        for result in results:
-            report.add(result)
+    for operator in all_operators:
+        results = operator.benchmark()
+        report.extend(results)
 
     report.save_json()
     report.save_markdown()
 
 
 if __name__ == "__main__":
     main()
-
-
-import operators
-for op in operators.__all__:
-    print(f"Loaded operator: {op}")
diff --git a/benchmark/operators/__init__.py b/benchmark/operators/__init__.py
@@ -1 +1,21 @@
-# Benchmark modules for individual operators
+from . import (
+    classify_video_zero_shot,
+    cluster_embeddings,
+    detect_lewd_images,
+    detect_text_in_image_tesseract,
+    dimension_reduction,
+    image_vec_rep_resnet,
+    vid_vec_rep_clip,
+    video_hash_tmk,
+)
+
+all_operators = [
+    classify_video_zero_shot,
+    cluster_embeddings,
+    detect_lewd_images,
+    detect_text_in_image_tesseract,
+    dimension_reduction,
+    image_vec_rep_resnet,
+    vid_vec_rep_clip,
+    video_hash_tmk,
+]
diff --git a/benchmark/operators/classify_video_zero_shot.py b/benchmark/operators/classify_video_zero_shot.py
@@ -2,7 +2,7 @@
 from operators.classify_video_zero_shot import VideoClassifier
 
 
-def benchmark():
+def benchmark() -> list[dict]:
     """Benchmark the VideoClassifier operator."""
     test_data = [
         {"file": "test/image1.jpg", "labels": ["label1", "label2"]},
diff --git a/benchmark/operators/cluster_embeddings.py b/benchmark/operators/cluster_embeddings.py
@@ -3,7 +3,7 @@
 from operators.cluster_embeddings import ClusterEmbeddings
 
 
-def benchmark():
+def benchmark() -> list[dict]:
     """Benchmark the ClusterEmbeddings operator."""
     results = []
 
@@ -29,8 +29,11 @@ def benchmark():
         # Test with different clustering methods
         for method in ["kmeans", "dbscan", "agglomerative"]:
             runtime_kwargs = {
-                "embeddings": embeddings,
-                "method": method,
+                "input_data": [
+                    {"payload": f"sample_{i}", "embedding": embedding.tolist()}
+                    for i, embedding in enumerate(embeddings)
+                ],
+                "modality": "video",
             }
 
             # For kmeans, specify the number of clusters
diff --git a/benchmark/operators/detect_lewd_images.py b/benchmark/operators/detect_lewd_images.py
@@ -2,7 +2,7 @@
 from operators.detect_lewd_images import LewdImageDetector
 
 
-def benchmark():
+def benchmark() -> list[dict]:
     """Benchmark the LewdImageDetector operator."""
     test_data = [
         {"file": "test_images/image1.jpg"},
@@ -18,12 +18,4 @@ def benchmark():
         )
         results.append(result)
 
-        if result["status"] == "success":
-            print(
-                f"  Time: {result['execution']['execution_time_seconds']:.2f}s, "
-                f"Memory: {result['execution']['memory_change_mb']:.2f}MB"
-            )
-        else:
-            print(f"  Failed: {result.get('error', 'Unknown error')}")
-
     return results
diff --git a/benchmark/operators/detect_text_in_image_tesseract.py b/benchmark/operators/detect_text_in_image_tesseract.py
@@ -2,7 +2,7 @@
 from operators.detect_text_in_image_tesseract import ImageTextDetector
 
 
-def benchmark():
+def benchmark() -> list[dict]:
     """Benchmark the ImageTextDetector operator."""
     test_data = [
         {"file": "test_images/image1.jpg"},
@@ -18,5 +18,4 @@ def benchmark():
         )
         results.append(result)
 
-
     return results
diff --git a/benchmark/operators/dimension_reduction.py b/benchmark/operators/dimension_reduction.py
@@ -1,9 +1,11 @@
+import random
+
 from benchmark.data_generator import DataGenerator
 from benchmark.profiler import Profiler
 from operators.dimension_reduction import DimensionReduction
 
 
-def benchmark():
+def benchmark() -> list[dict]:
     """Benchmark the DimensionReduction operator."""
     results = []
 
@@ -18,18 +20,28 @@ def benchmark():
     for dataset_name, embeddings in test_datasets.items():
         print(f"Processing: {dataset_name} (shape: {embeddings.shape})")
 
-        # Test with different reduction methods
-        for method in ["tsne", "pca", "umap"]:
+        # Test with different reduction methods (only supported ones)
+        for method in ["tsne", "umap"]:
+            # Convert embeddings to the format expected by the operator
             runtime_kwargs = {
-                "embeddings": embeddings,
-                "method": method,
-                "n_components": 2,
+                "input_data": [
+                    {"payload": f"sample_{i}", "embedding": embedding.tolist()}
+                    for i, embedding in enumerate(embeddings)
+                ],
+            }
+
+            operator_kwargs = {
+                "model_type": method,
+                "params": {
+                    "n_components": random.randint(2, 3),
+                },
             }
 
             result = Profiler.benchmark_operator(
                 operator_class=DimensionReduction,
                 operator_name=f"dimension_reduction_{method}",
                 runtime_kwargs=runtime_kwargs,
+                operator_kwargs=operator_kwargs,
             )
 
             # Add dataset info to result
diff --git a/benchmark/operators/image_vec_rep_resnet.py b/benchmark/operators/image_vec_rep_resnet.py
@@ -2,7 +2,7 @@
 from operators.image_vec_rep_resnet import ImageVecRepResnet
 
 
-def benchmark():
+def benchmark() -> list[dict]:
     """Benchmark the ImageVecRepResnet operator."""
     test_data = [
         {"file": "test_images/image1.jpg"},
diff --git a/benchmark/operators/vid_vec_rep_clip.py b/benchmark/operators/vid_vec_rep_clip.py
@@ -2,7 +2,7 @@
 from operators.vid_vec_rep_clip import VidVecRepClip
 
 
-def benchmark():
+def benchmark() -> list[dict]:
     """Benchmark the VidVecRepClip operator."""
     test_data = [
         {"file": "test_videos/video1.mp4"},
@@ -12,12 +12,11 @@ def benchmark():
     results = []
 
     for test_item in test_data:
-
         result = Profiler.benchmark_operator(
             operator_class=VidVecRepClip,
             operator_name="vid_vec_rep_clip",
             runtime_kwargs=test_item,
         )
         results.append(result)
-        
+
     return results
diff --git a/benchmark/operators/video_hash_tmk.py b/benchmark/operators/video_hash_tmk.py
@@ -2,7 +2,7 @@
 from operators.video_hash_tmk import VideoHashTmk
 
 
-def benchmark():
+def benchmark() -> list[dict]:
     """Benchmark the VideoHashTmk operator."""
     test_data = [
         {"file": "test_videos/video1.mp4"},
diff --git a/benchmark/profiler.py b/benchmark/profiler.py
@@ -7,9 +7,10 @@
 from typing import Any, Callable
 
 import psutil
-from feluda.operator import Operator
 from memory_profiler import memory_usage
 
+from feluda.operator import Operator
+
 
 class Profiler:
     """Uniform profiler for benchmarking operators."""
@@ -73,15 +74,17 @@ def run_func():
     def benchmark_operator(
         operator_class: type[Operator],
         operator_name: str,
-        runtime_kwargs: dict,
+        runtime_kwargs: dict = {},
         operator_kwargs: dict = {},
     ) -> dict[str, Any]:
         """Benchmark a single operator with given test data."""
+        print(f"Benchmarking {operator_name}...")
         results = {
             "operator": operator_name,
             "timestamp": datetime.now().isoformat(),
         }
-        try:
+        # try:
+        if True:
             # Initialization profiling
             init_start = time.perf_counter()
             init_mem_before = Profiler._get_memory_mb()
@@ -115,8 +118,8 @@ def run_operator():
 
             results["status"] = "success"
 
-        except Exception as e:
-            results["status"] = "failed"
-            results["error"] = str(e)
+        # except Exception as e:
+        #     results["status"] = "failed"
+        #     results["error"] = str(e)
 
         return results
diff --git a/benchmark/report.py b/benchmark/report.py
@@ -8,7 +8,8 @@
 from typing import Any
 
 import numpy as np
-from profiler import Profiler
+
+from benchmark.profiler import Profiler
 
 
 class BenchmarkReport:
@@ -18,10 +19,14 @@ def __init__(self) -> None:
         self.results: list[dict[str, Any]] = []
         self.system_info: dict[str, Any] = Profiler.get_system_info()
 
-    def add_results(self, result: dict[str, Any]) -> None:
+    def add(self, result: dict[str, Any]) -> None:
         """Add a benchmark result to the report."""
         self.results.append(result)
 
+    def extend(self, results: list[dict[str, Any]]) -> None:
+        """Extend the benchmark report with multiple results."""
+        self.results.extend(results)
+
     def generate_summary(self) -> dict[str, Any]:
         """Generate summary statistics for all benchmarked operators."""
         summary = {
@@ -76,8 +81,8 @@ def generate_summary(self) -> dict[str, Any]:
     def save_json(self, filepath: str | Path = None) -> None:
         """Save the benchmark summary as JSON."""
         if not filepath:
-            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-            filepath = f"results/benchmark_results_{timestamp}.json"
+            timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+            filepath = f"benchmark/results/benchmark_results_{timestamp}.json"
         Path(filepath).write_text(
             json.dumps(self.generate_summary(), indent=2, default=str),
             encoding="utf-8",
@@ -86,8 +91,8 @@ def save_json(self, filepath: str | Path = None) -> None:
     def save_markdown(self, filepath: str | Path = None) -> None:
         """Save the benchmark summary as a Markdown report."""
         if not filepath:
-            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-            filepath = f"results/benchmark_results_{timestamp}.md"
+            timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+            filepath = f"benchmark/results/benchmark_results_{timestamp}.md"
         summary = self.generate_summary()
         sysinfo = self.system_info
 
diff --git a/benchmark/results/benchmark_results_20250822_003531.md b/benchmark/results/benchmark_results_20250822_003531.md
diff --git a/benchmark/results/benchmark_results_20250822_003743.md b/benchmark/results/benchmark_results_20250822_003743.md
diff --git a/pyproject.toml b/pyproject.toml
