python.yml

# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python

name: Python

on:
  push:
  pull_request:
  schedule:
    # Run daily at 00:00 so we get notified if CI is broken before a pull request is submitted. 
    # It also notifies us about new Arrow releases for which we need to release a corresponding version of PalletJack.
    - cron:  '0 0 * * *'

permissions:
  contents: read
  
jobs:

  format:
    if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v6    
    - name: Set up Python 3.12
      uses: actions/setup-python@v6
      with:
        python-version: "3.12"

    - name: Run Black (check only)
      run: |
        python3 -m pip install black
        black --check --diff --color .

    - name: Lint with flake8
      run: |
        python3 -m pip install flake8
        # stop the build if there are Python syntax errors or undefined names
        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics

  test:
    if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
    runs-on: ubuntu-latest
    strategy:
      matrix:
        mode: [None, io_uring, io_uring_odirect]
      fail-fast: false
    steps:
    - uses: actions/checkout@v6
    - name: Set up Python 3.12
      uses: actions/setup-python@v6
      with:
        python-version: "3.12"
        
    - name: Set environment variables
      run: |
        if [[ "${{ matrix.mode }}" != "None" ]]; then
          echo "JJ_READER_BACKEND=${{ matrix.mode }}" >> $GITHUB_ENV
        fi

    - name: Install dependencies
      shell: bash
      run: |
        python3 -m pip install --upgrade pip
        python3 -m pip install torch --index-url https://download.pytorch.org/whl/cpu
        python3 -m pip install -r requirements.txt
        python3 -m pip install pytest

    - name: Test with pytest
      run: |
        sudo apt-get install liburing-dev
        pip install --editable .
        pytest

  build-wheels:
    if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
    runs-on: ${{ matrix.runner }}
    strategy:
      matrix:
        runner: [ubuntu-24.04-arm, ubuntu-latest, windows-latest, macos-latest]
      fail-fast: false

    steps:
    - uses: actions/checkout@v6

    - name: Set up Python 3.12
      uses: actions/setup-python@v6
      with:
        python-version: "3.12"
        
    - name: Install dependencies
      shell: bash
      run: |
        python3 -m pip install --upgrade pip
        python3 -m pip install -r requirements.txt
        python3 -m pip install flake8 pytest cibuildwheel build

    - name: Build sdist (Linux)
      if: runner.os == 'linux'
      run: |
        python3 -m build --sdist
        
    - name: Build wheels
      run: python3 -m cibuildwheel --output-dir dist
      env:
        CIBW_BUILD_VERBOSITY: 1

    - uses: actions/upload-artifact@v7
      with:
        name: dist-${{ matrix.runner }}
        path: ./dist/*

  test-binary:
    needs: build-wheels
    runs-on: ${{ matrix.runner }}
    strategy:
      matrix:
        runner: [ubuntu-24.04-arm, ubuntu-latest, windows-latest, macos-latest]
      fail-fast: false
    steps:
    - uses: actions/checkout@v6
    - name: Set up Python 3.12
      uses: actions/setup-python@v6
      with:
        python-version: "3.12"

    - name: Download artifacts
      uses: actions/download-artifact@v8
      with:
        pattern: dist-*
        path: dist
        merge-multiple: true
 
    - name: Setup torch
      run: |
        python3 -m pip install torch --index-url https://download.pytorch.org/whl/cpu
         
    - name: Test with pytest
      run: |
        python3 -m pip install -r requirements.txt
        python3 -m pip install pytest
        # Keep in mind that if the local and remote versions are the same, the remote version will be installed
        python3 -m pip install JollyJack --pre --find-links ./dist --break-system-packages --only-binary=:all:
        # So now ensure that the local version is installed 
        python3 -m pip install JollyJack --pre --find-links ./dist --break-system-packages --only-binary=:all: --force-reinstall --no-index --no-deps
        python3 test/test_jollyjack.py

  test-binary-manylinux:
    needs: build-wheels
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["cp310-cp310", "cp311-cp311", "cp312-cp312", "cp313-cp313", "cp314-cp314"]
      fail-fast: false

    steps:
      - name: Checkout code
        uses: actions/checkout@v6
  
      - name: Download artifacts
        uses: actions/download-artifact@v8
        with:
          pattern: dist-*
          path: dist
          merge-multiple: true
  
      - name: Build and Run Docker Container for Tests
        run: |
          # Build the Docker image with Python version as build arg
          docker build \
            --build-arg PYTHON_VERSION=${{ matrix.python-version }} \
            -t jollyjack-test-py${{ matrix.python-version }} \
            --progress plain \
            -f Dockerfile.test .
          docker run jollyjack-test-py${{ matrix.python-version }}

  # Virtual job that can be configured as a required check before a PR can be merged.
  # As GitHub considers a check as successful if it is skipped, we need to check its status in
  # another workflow (check-required.yml) and create a check there.
  all-required-checks-done:
    name: All required checks done
    needs:
      - test
      - build-wheels
      - test-binary
      - test-binary-manylinux
    runs-on: ubuntu-latest
    steps:
      - run: echo "All required checks done"

  benchmarks:
    needs: build-wheels
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest]
        mode: [CPU, FILE_SYSTEM]
      fail-fast: false

    runs-on: ${{ matrix.os }}
    steps:
    - uses: actions/checkout@v6
    - name: Set up Python 3.10
      uses: actions/setup-python@v6
      with:
        python-version: "3.10"

    - name: Download artifacts
      uses: actions/download-artifact@v8
      with:
        pattern: dist-*
        path: dist
        merge-multiple: true

    - name: Setup torch
      run: |
        python3 -m pip install torch --index-url https://download.pytorch.org/whl/cpu

    - name: Run benchmarks
      run: |
        python3 -m pip install -r requirements.txt
        # Keep in mind that if the local and remote versions are the same, the remote version will be installed
        pip install JollyJack --pre --find-links ./dist --break-system-packages --only-binary=:all:
        # So now ensure that the local version is installed 
        pip install JollyJack --pre --find-links ./dist --break-system-packages --only-binary=:all: --force-reinstall --no-index --no-deps
        python3 ./benchmarks/benchmark_jollyjack.py
      env:
        JJB_benchmark_mode: ${{ matrix.mode }}

  publish:
    if: ${{ !github.event.repository.fork && startsWith(github.ref, 'refs/tags/v') }}
    needs: [all-required-checks-done]
    runs-on: ubuntu-latest
    environment: release
    permissions:
      id-token: write

    steps:

    - name: Download artifacts
      uses: actions/download-artifact@v8
      with:
        pattern: dist-*
        path: dist
        merge-multiple: true
    
    - name: Display structure of downloaded files
      run: ls -R dist
    
    - name: Validate tag
      shell: pwsh
      run: |
        $tag = "${{ github.ref }}".SubString(11)
        $expectedFile = "dist/jollyjack-$tag.tar.gz"
    
        # Check whether the tag and the package version match together
        if (-not (Test-Path -Path $expectedFile)) {
            echo "::error ::Expected file $expectedFile doesn't exist"
            Get-ChildItem -Path dist
            exit 1
        }
    
    - name: Publish to PyPi
      uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b
      with:
        packages-dir: dist