Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions .github/workflows/benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,46 +7,46 @@ jobs:
benchmark:
runs-on: ubuntu-latest
timeout-minutes: 120

steps:
- uses: actions/checkout@v4

- name: Install uv
uses: astral-sh/setup-uv@v5

- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y ffmpeg tesseract-ocr tesseract-ocr-hin tesseract-ocr-tam tesseract-ocr-tel

- name: Setup Python environment
run: |
uv venv
source .venv/bin/activate
uv pip install ".[all]" ".[dev]"
uv pip install gdown

- name: Download benchmark videos
run: |
gdown --folder "https://drive.google.com/drive/folders/1XozZbqSzJVzTD4QgGZNOR6U5U7_rf5O3" -O benchmark/videos

- name: Run benchmarks
run: |
source .venv/bin/activate
cd benchmark
python main.py

- name: Upload benchmark results
uses: actions/upload-artifact@v4
with:
name: benchmark-results-${{ github.run_id }}
path: benchmark/results/
retention-days: 90

- name: Cleanup caches
if: always()
run: |
rm -rf ~/.cache/torch/hub
rm -rf ~/.cache/huggingface/hub
rm -rf benchmark/videos
rm -rf benchmark/images
rm -rf benchmark/images
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -158,4 +158,4 @@ src/core/operators/cnn/panns_data
# uv
uv.lock
benchmark/images/
benchmark/videos/
benchmark/videos/
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,10 @@ operators :
label : "Operators"
parameters :
- name : "Video Vector Representation"
type : "vid_vec_rep_clip"
type : "vid_vec_rep"
parameters: {}
- name : "Image Vector Representation"
type : "image_vec_rep_resnet"
type : "image_vec_rep"
parameters: {}
```

Expand All @@ -82,7 +82,7 @@ feluda = Feluda(config_path)
feluda.setup()

# Access an operator and run a task
operator = feluda.operators.get()["vid_vec_rep_clip"]
operator = feluda.operators.get()["vid_vec_rep"]
result = operator.run("path/to/example.mp4")
print(result)
```
Expand Down
10 changes: 5 additions & 5 deletions benchmark/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ def benchmark() -> list[dict]:
test_data = [
{"file": "test/image1.jpg", "labels": ["label1", "label2"]},
{"file": "test/image2.jpg", "labels": ["label1", "label3"]},
]
]
for test_item in test_data:
# Profile the operator for each test_item and add it to results.
result = Profiler.benchmark_operator(
Expand All @@ -105,8 +105,8 @@ def benchmark() -> list[dict]:
runtime_kwargs=test_item,
operator_kwargs=
)
results.append(result)

results.append(result)
return results
```

Expand Down Expand Up @@ -166,8 +166,8 @@ Provides human-readable summaries with:
pip install psutil memory_profiler numpy pillow
```

or
or

```bash
pip install feluda['dev']
```
```
16 changes: 8 additions & 8 deletions benchmark/operators/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,20 @@
# classify_video_zero_shot,
cluster_embeddings,
detect_lewd_images,
detect_text_in_image_tesseract,
detect_text_in_image,
dimension_reduction,
image_vec_rep_resnet,
vid_vec_rep_clip,
video_hash_tmk,
image_vec_rep,
vid_vec_rep,
video_hash,
)

all_operators = [
# classify_video_zero_shot,
cluster_embeddings,
detect_lewd_images,
detect_text_in_image_tesseract,
detect_text_in_image,
dimension_reduction,
image_vec_rep_resnet,
vid_vec_rep_clip,
video_hash_tmk,
image_vec_rep,
vid_vec_rep,
video_hash,
]
6 changes: 3 additions & 3 deletions benchmark/operators/classify_video_zero_shot.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
from benchmark.config import BENCHMARK_VIDEOS
from benchmark.profiler import Profiler
from feluda.factory import VideoFactory
from operators.classify_video_zero_shot import VideoClassifier
from operators.classify_video_zero_shot import ClassifyVideoZeroShot


def benchmark() -> list[dict]:
"""Benchmark the VideoClassifier operator."""
"""Benchmark the ClassifyVideoZeroShot operator."""
test_data = [
{"file": VideoFactory.make_from_file_on_disk(video_path)}
for video_path in BENCHMARK_VIDEOS
Expand All @@ -14,7 +14,7 @@ def benchmark() -> list[dict]:
for test_item in test_data:
print(test_item)
result = Profiler.benchmark_operator(
operator_class=VideoClassifier,
operator_class=ClassifyVideoZeroShot,
operator_name="video_classifier",
runtime_kwargs=test_item,
)
Expand Down
6 changes: 3 additions & 3 deletions benchmark/operators/detect_lewd_images.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
from benchmark.config import BENCHMARK_IMAGES
from benchmark.profiler import Profiler
from operators.detect_lewd_images import LewdImageDetector
from operators.detect_lewd_images import DetectLewdImages


def benchmark() -> list[dict]:
"""Benchmark the LewdImageDetector operator."""
"""Benchmark the DetectLewdImages operator."""
test_data = [{"file": image} for image in BENCHMARK_IMAGES]

results = []
for test_item in test_data:
result = Profiler.benchmark_operator(
operator_class=LewdImageDetector,
operator_class=DetectLewdImages,
operator_name="detect_lewd_images",
runtime_kwargs=test_item,
)
Expand Down
8 changes: 4 additions & 4 deletions benchmark/operators/detect_text_in_image_tesseract.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
from benchmark.config import BENCHMARK_IMAGES
from benchmark.profiler import Profiler
from operators.detect_text_in_image_tesseract import ImageTextDetector
from operators.detect_text_in_image import DetectTextInImage


def benchmark() -> list[dict]:
"""Benchmark the ImageTextDetector operator."""
"""Benchmark the DetectTextInImage operator."""
test_data = [{"file": image} for image in BENCHMARK_IMAGES]

results = []
for test_item in test_data:
result = Profiler.benchmark_operator(
operator_class=ImageTextDetector,
operator_name="detect_text_in_image_tesseract",
operator_class=DetectTextInImage,
operator_name="detect_text_in_image",
runtime_kwargs=test_item,
)
results.append(result)
Expand Down
8 changes: 4 additions & 4 deletions benchmark/operators/image_vec_rep_resnet.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
from benchmark.config import BENCHMARK_IMAGES
from benchmark.profiler import Profiler
from feluda.factory import ImageFactory
from operators.image_vec_rep_resnet import ImageVecRepResnet
from operators.image_vec_rep import ImageVecRep


def benchmark() -> list[dict]:
"""Benchmark the ImageVecRepResnet operator."""
"""Benchmark the ImageVecRep operator."""
test_data = [
{"image": ImageFactory.make_from_file_on_disk_to_path(image)}
for image in BENCHMARK_IMAGES
Expand All @@ -14,8 +14,8 @@ def benchmark() -> list[dict]:
results = []
for test_item in test_data:
result = Profiler.benchmark_operator(
operator_class=ImageVecRepResnet,
operator_name="image_vec_rep_resnet",
operator_class=ImageVecRep,
operator_name="image_vec_rep",
runtime_kwargs=test_item,
)
results.append(result)
Expand Down
8 changes: 4 additions & 4 deletions benchmark/operators/vid_vec_rep_clip.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
from benchmark.config import BENCHMARK_VIDEOS
from benchmark.profiler import Profiler
from feluda.factory import VideoFactory
from operators.vid_vec_rep_clip import VidVecRepClip
from operators.vid_vec_rep import VidVecRep


def benchmark() -> list[dict]:
"""Benchmark the VidVecRepClip operator."""
"""Benchmark the VidVecRep operator."""
test_data = [
{"file": VideoFactory.make_from_file_on_disk(video_path)}
for video_path in BENCHMARK_VIDEOS
Expand All @@ -14,8 +14,8 @@ def benchmark() -> list[dict]:

for test_item in test_data:
result = Profiler.benchmark_operator(
operator_class=VidVecRepClip,
operator_name="vid_vec_rep_clip",
operator_class=VidVecRep,
operator_name="vid_vec_rep",
runtime_kwargs=test_item,
)
results.append(result)
Expand Down
8 changes: 4 additions & 4 deletions benchmark/operators/video_hash_tmk.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
from benchmark.config import BENCHMARK_VIDEOS
from benchmark.profiler import Profiler
from operators.video_hash_tmk import VideoHashTmk
from operators.video_hash import VideoHash


def benchmark() -> list[dict]:
"""Benchmark the VideoHashTmk operator."""
"""Benchmark the VideoHash operator."""
test_data = [{"video_path": video_path} for video_path in BENCHMARK_VIDEOS]

results = []
for test_item in test_data:
result = Profiler.benchmark_operator(
operator_class=VideoHashTmk,
operator_name="video_hash_tmk",
operator_class=VideoHash,
operator_name="video_hash",
runtime_kwargs=test_item,
)
results.append(result)
Expand Down
2 changes: 1 addition & 1 deletion benchmark/results/benchmark_results_20250829.json
Original file line number Diff line number Diff line change
Expand Up @@ -1033,4 +1033,4 @@
"success_rate": 1.0
}
}
}
}
24 changes: 12 additions & 12 deletions benchmark/results/benchmark_results_20250829.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ Generated: 2025-08-28T23:55:24.834470
| cluster_embeddings_dbscan | 3 | 100.0 | 6.256 | 6.214 | 6.277 | 0.030 | -10.64 | 851.92 |
| cluster_embeddings_kmeans | 3 | 100.0 | 12.851 | 10.772 | 14.850 | 1.666 | -38.10 | 846.00 |
| detect_lewd_images | 11 | 100.0 | 8.863 | 6.579 | 9.632 | 0.817 | -18.47 | 1905.08 |
| detect_text_in_image_tesseract | 11 | 100.0 | 9.381 | 5.869 | 15.024 | 2.735 | 0.54 | 1615.25 |
| detect_text_in_image | 11 | 100.0 | 9.381 | 5.869 | 15.024 | 2.735 | 0.54 | 1615.25 |
| dimension_reduction_tsne | 4 | 100.0 | 30.234 | 6.242 | 63.125 | 21.633 | 40.23 | 2008.08 |
| dimension_reduction_umap | 4 | 100.0 | 12.339 | 8.190 | 16.055 | 3.177 | 18.89 | 2041.16 |

Expand Down Expand Up @@ -46,17 +46,17 @@ Generated: 2025-08-28T23:55:24.834470
| detect_lewd_images | - | success | 9.087 | -21.03 | 1848.25 | 1.734 | |
| detect_lewd_images | - | success | 6.579 | 53.12 | 1865.45 | 1.780 | |
| detect_lewd_images | - | success | 9.302 | -24.25 | 1877.45 | 1.800 | |
| detect_text_in_image_tesseract | - | success | 12.963 | 5.95 | 1615.22 | 0.068 | |
| detect_text_in_image_tesseract | - | success | 10.756 | 0.03 | 1615.25 | 0.063 | |
| detect_text_in_image_tesseract | - | success | 15.024 | 0.00 | 1615.25 | 0.062 | |
| detect_text_in_image_tesseract | - | success | 11.240 | 0.00 | 1615.25 | 0.080 | |
| detect_text_in_image_tesseract | - | success | 6.466 | 0.00 | 1615.25 | 0.060 | |
| detect_text_in_image_tesseract | - | success | 8.662 | 0.00 | 1615.25 | 0.054 | |
| detect_text_in_image_tesseract | - | success | 8.693 | 0.00 | 1615.25 | 0.053 | |
| detect_text_in_image_tesseract | - | success | 6.450 | 0.00 | 1615.25 | 0.101 | |
| detect_text_in_image_tesseract | - | success | 5.869 | 0.00 | 1615.25 | 0.059 | |
| detect_text_in_image_tesseract | - | success | 8.997 | 0.00 | 1615.25 | 0.093 | |
| detect_text_in_image_tesseract | - | success | 8.067 | 0.00 | 1615.25 | 0.055 | |
| detect_text_in_image | - | success | 12.963 | 5.95 | 1615.22 | 0.068 | |
| detect_text_in_image | - | success | 10.756 | 0.03 | 1615.25 | 0.063 | |
| detect_text_in_image | - | success | 15.024 | 0.00 | 1615.25 | 0.062 | |
| detect_text_in_image | - | success | 11.240 | 0.00 | 1615.25 | 0.080 | |
| detect_text_in_image | - | success | 6.466 | 0.00 | 1615.25 | 0.060 | |
| detect_text_in_image | - | success | 8.662 | 0.00 | 1615.25 | 0.054 | |
| detect_text_in_image | - | success | 8.693 | 0.00 | 1615.25 | 0.053 | |
| detect_text_in_image | - | success | 6.450 | 0.00 | 1615.25 | 0.101 | |
| detect_text_in_image | - | success | 5.869 | 0.00 | 1615.25 | 0.059 | |
| detect_text_in_image | - | success | 8.997 | 0.00 | 1615.25 | 0.093 | |
| detect_text_in_image | - | success | 8.067 | 0.00 | 1615.25 | 0.055 | |
| dimension_reduction_tsne | small_500x512 - tsne | success | 6.242 | 5.47 | 1679.20 | 8.336 | |
| dimension_reduction_umap | small_500x512 - umap | success | 8.190 | 10.77 | 1689.98 | 2.457 | |
| dimension_reduction_tsne | medium_2000x512 - tsne | success | 16.433 | 22.95 | 1712.94 | 85.710 | |
Expand Down
8 changes: 4 additions & 4 deletions docs/api/operators.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@ This section documents all the available operators in Feluda.
:maxdepth: 1
:caption: Operators

operators/image_vec_rep_resnet
operators/image_vec_rep
operators/detect_lewd_images
operators/detect_text_in_image_tesseract
operators/vid_vec_rep_clip
operators/video_hash_tmk
operators/detect_text_in_image
operators/vid_vec_rep
operators/video_hash
operators/classify_video_zero_shot
operators/cluster_embeddings
operators/dimension_reduction
Expand Down
2 changes: 1 addition & 1 deletion docs/api/operators/classify_video_zero_shot.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
```

```{eval-rst}
.. autoclass:: operators.classify_video_zero_shot.classify_video_zero_shot.VideoClassifier
.. autoclass:: operators.classify_video_zero_shot.classify_video_zero_shot.ClassifyVideoZeroShot
:members:
:undoc-members:
:show-inheritance:
Expand Down
2 changes: 1 addition & 1 deletion docs/api/operators/detect_lewd_images.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
```

```{eval-rst}
.. autoclass:: operators.detect_lewd_images.detect_lewd_images.LewdImageDetector
.. autoclass:: operators.detect_lewd_images.detect_lewd_images.DetectLewdImages
:members:
:undoc-members:
:show-inheritance:
Expand Down
4 changes: 2 additions & 2 deletions docs/api/operators/detect_text_in_image_tesseract.md
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
```{include} ../../../operators/detect_text_in_image_tesseract/README.md
```{include} ../../../operators/detect_text_in_image/README.md
:parser: myst_parser.sphinx_
```

```{eval-rst}
.. autoclass:: operators.detect_text_in_image_tesseract.detect_text_in_image_tesseract.ImageTextDetector
.. autoclass:: operators.detect_text_in_image.detect_text_in_image.DetectTextInImage
:members:
:undoc-members:
:show-inheritance:
Expand Down
4 changes: 2 additions & 2 deletions docs/api/operators/image_vec_rep_resnet.md
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
```{include} ../../../operators/image_vec_rep_resnet/README.md
```{include} ../../../operators/image_vec_rep/README.md
:parser: myst_parser.sphinx_
```

```{eval-rst}
.. autoclass:: operators.image_vec_rep_resnet.image_vec_rep_resnet.ImageVecRepResnet
.. autoclass:: operators.image_vec_rep.image_vec_rep.ImageVecRep
:members:
:undoc-members:
:show-inheritance:
Expand Down
4 changes: 2 additions & 2 deletions docs/api/operators/vid_vec_rep_clip.md
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
```{include} ../../../operators/vid_vec_rep_clip/README.md
```{include} ../../../operators/vid_vec_rep/README.md
:parser: myst_parser.sphinx_
```

```{eval-rst}
.. autoclass:: operators.vid_vec_rep_clip.vid_vec_rep_clip.VidVecRepClip
.. autoclass:: operators.vid_vec_rep.vid_vec_rep.VidVecRep
:members:
:undoc-members:
:show-inheritance:
Expand Down
Loading
Loading