Release v1.2.0: sql-jobs and summary commands #11
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| jobs: | |
| unit-tests: | |
| name: Unit Tests | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.10", "3.12", "3.13"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| pip install -e . | |
| pip install pytest | |
| - name: Run unit tests | |
| run: | | |
| python -m pytest spark_history_cli/tests/test_core.py -v --tb=short | |
| e2e-tests: | |
| name: E2E Tests (Docker SHS) | |
| runs-on: ubuntu-latest | |
| env: | |
| SPARK_IMAGE: apache/spark:4.0.0 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Cache Docker image | |
| id: docker-cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: /tmp/spark-docker-image.tar | |
| key: spark-docker-${{ env.SPARK_IMAGE }} | |
| - name: Load cached Docker image | |
| if: steps.docker-cache.outputs.cache-hit == 'true' | |
| run: docker load -i /tmp/spark-docker-image.tar | |
| - name: Pull and save Docker image | |
| if: steps.docker-cache.outputs.cache-hit != 'true' | |
| run: | | |
| docker pull ${{ env.SPARK_IMAGE }} | |
| docker save ${{ env.SPARK_IMAGE }} -o /tmp/spark-docker-image.tar | |
| - name: Generate sample event logs | |
| run: | | |
| mkdir -p /tmp/spark-events | |
| python ci/generate_sample_logs.py | |
| env: | |
| SPARK_EVENT_LOG_DIR: /tmp/spark-events | |
| - name: Start Spark History Server (Docker) | |
| run: | | |
| docker run -d --name spark-history-server \ | |
| -p 18080:18080 \ | |
| -v /tmp/spark-events:/tmp/spark-events \ | |
| -e SPARK_HISTORY_OPTS="-Dspark.history.fs.logDirectory=/tmp/spark-events" \ | |
| ${{ env.SPARK_IMAGE }} \ | |
| /opt/spark/bin/spark-class org.apache.spark.deploy.history.HistoryServer | |
| # Wait for SHS to be ready (up to 30s) | |
| echo "Waiting for History Server to start..." | |
| for i in $(seq 1 30); do | |
| if curl -sf http://localhost:18080/api/v1/version > /dev/null 2>&1; then | |
| echo "History Server is ready!" | |
| curl -s http://localhost:18080/api/v1/version | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| # Verify apps are loaded | |
| echo "Waiting for apps to be indexed..." | |
| for i in $(seq 1 30); do | |
| APPS=$(curl -sf http://localhost:18080/api/v1/applications 2>/dev/null | python3 -c "import sys,json; print(len(json.load(sys.stdin)))" 2>/dev/null || echo "0") | |
| if [ "$APPS" -gt "0" ]; then | |
| echo "Found $APPS applications" | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| - name: Install spark-history-cli | |
| run: | | |
| pip install -e . | |
| pip install pytest | |
| - name: Run E2E tests | |
| run: | | |
| python -m pytest spark_history_cli/tests/ -v -s --tb=short | |
| env: | |
| SPARK_HISTORY_SERVER: http://localhost:18080 | |
| - name: Stop History Server | |
| if: always() | |
| run: | | |
| docker stop spark-history-server || true | |
| docker rm spark-history-server || true |