Add sql-jobs command to show jobs for a SQL execution

Fetches all job IDs (succeeded/failed/running) from a SQL execution and
displays their details. Uses bulk list_jobs + client-side filter for
efficiency and gracefully handles missing job IDs.

Co-authored-by: Copilot <[email protected]>

Author: yaooqinn

CHANGELOG.md

@@ -10,7 +10,11 @@
   - `--dot` outputs the plan DAG as a Graphviz DOT file for visualization.
   - `-o <file>` writes output to a file instead of stdout.
   - `--json` returns structured JSON with `isAdaptive`, `sectionCount`, and parsed `sections`.
-- `sql-plan` REPL command with the same options.
+- **`sql-jobs` command** — Show jobs associated with a SQL execution.
+  - Fetches all job IDs (succeeded, failed, running) from the SQL execution.
+  - Displays job details in a table with status, stages, and task counts.
+  - Gracefully handles cases where referenced job IDs are not found.
+- `sql-plan` and `sql-jobs` REPL commands with the same options.
 
 ### Changed
 - **E2E CI switched to Docker-based SHS** — Uses `apache/spark:4.0.0` Docker image with `actions/cache` for faster CI runs (~5s cached load vs ~2min download).

README.md

@@ -79,13 +79,17 @@ spark-history-cli --app-id <id> sql-plan <exec-id> --view final   # post-AQE pla
 spark-history-cli --app-id <id> sql-plan <exec-id> --dot          # Graphviz DOT
 spark-history-cli --app-id <id> sql-plan <exec-id> --dot -o plan.dot  # save to file
 
+# Jobs for a SQL execution
+spark-history-cli --app-id <id> sql-jobs <exec-id>
+
 # Download event logs
 spark-history-cli --app-id <id> logs output.zip
 
 # JSON output for scripting/agents
 spark-history-cli --json apps
 spark-history-cli --json --app-id <id> jobs
 spark-history-cli --json --app-id <id> sql-plan <exec-id>
+spark-history-cli --json --app-id <id> sql-jobs <exec-id>
 ```
 
 ### REPL Commands
@@ -101,6 +105,7 @@ stage <id> [attempt]    Show stage details
 executors [--all]       List executors
 sql [id]                List or show SQL executions
 sql-plan <id> [opts]    Show SQL plan (--view, --dot, -o)
+sql-jobs <id>           Show jobs for a SQL execution
 rdds                    List cached RDDs
 env                     Show environment/config
 logs [path]             Download event logs

spark_history_cli/cli.py

@@ -61,6 +61,25 @@ def output_status_block(skin, info: dict[str, str], title: str = ""):
     skin.status_block(info, title=title)
 
 
+def _collect_sql_job_ids(sql_exec: dict) -> list[int]:
+    """Collect all job IDs from a SQL execution (success + failed + running)."""
+    ids = []
+    ids.extend(sql_exec.get("successJobIds", []))
+    ids.extend(sql_exec.get("failedJobIds", []))
+    ids.extend(sql_exec.get("runningJobIds", []))
+    return sorted(set(ids))
+
+
+def _fetch_sql_jobs(client, app_id: str, sql_exec: dict) -> list[dict]:
+    """Fetch job details for a SQL execution using bulk list + filter."""
+    job_ids = _collect_sql_job_ids(sql_exec)
+    if not job_ids:
+        return []
+    target = set(job_ids)
+    all_jobs = client.list_jobs(app_id)
+    return [j for j in all_jobs if j.get("jobId") in target]
+
+
 # ── Main CLI group ────────────────────────────────────────────────────
 
 @click.group(invoke_without_command=True)
@@ -321,6 +340,25 @@ def repl(state: CliState):
                         else:
                             click.echo(parsed["fullPlan"])
 
+            elif cmd == "sql-jobs":
+                app_id = state.resolve_app_id(None)
+                if not args or not args[0].isdigit():
+                    skin.error("Usage: sql-jobs <execution-id>")
+                else:
+                    exec_id = int(args[0])
+                    ex = client.get_sql(app_id, exec_id)
+                    job_ids = _collect_sql_job_ids(ex)
+                    if not job_ids:
+                        skin.warning(f"No jobs found for SQL execution {exec_id}")
+                    else:
+                        jobs = _fetch_sql_jobs(client, app_id, ex)
+                        if not jobs:
+                            skin.warning(f"SQL execution {exec_id} references jobs {job_ids} but none were found")
+                        else:
+                            skin.section(f"Jobs for SQL Execution {exec_id} ({len(jobs)}/{len(job_ids)} jobs)")
+                            headers, rows = fmt.format_job_list(jobs)
+                            output_table(skin, headers, rows)
+
             elif cmd == "rdds":
                 app_id = state.resolve_app_id(None)
                 rdds = client.list_rdds(app_id)
@@ -598,6 +636,42 @@ def cmd_sql_plan(state: CliState, execution_id: int, view_mode: str, dot_mode: b
             click.echo(text)
 
 
+@cli.command("sql-jobs")
+@click.argument("execution_id", type=int)
+@pass_state
+def cmd_sql_jobs(state: CliState, execution_id: int):
+    """Show jobs associated with a SQL execution.
+
+    Fetches the SQL execution, collects all job IDs (succeeded, failed,
+    running), and displays each job's details.
+
+    Examples:
+
+      spark-history-cli -a <app> sql-jobs 4
+
+      spark-history-cli -a <app> --json sql-jobs 4
+    """
+    client = state.ensure_client()
+    app_id = state.resolve_app_id(None)
+    ex = client.get_sql(app_id, execution_id)
+    job_ids = _collect_sql_job_ids(ex)
+    if not job_ids:
+        click.echo(f"No jobs found for SQL execution {execution_id}.")
+        return
+    jobs = _fetch_sql_jobs(client, app_id, ex)
+    if not jobs:
+        click.echo(f"SQL execution {execution_id} references jobs {job_ids} but none were found.")
+        return
+    if state.json_mode:
+        output_json(jobs)
+    else:
+        from spark_history_cli.utils.repl_skin import ReplSkin
+        skin = ReplSkin("spark_history", version=__version__)
+        skin.section(f"Jobs for SQL Execution {execution_id} ({len(jobs)}/{len(job_ids)} jobs)")
+        headers, rows = fmt.format_job_list(jobs)
+        output_table(skin, headers, rows)
+
+
 @cli.command("rdds")
 @pass_state
 def cmd_rdds(state: CliState):

spark_history_cli/skills/SKILL.md

@@ -37,6 +37,7 @@ spark-history-cli --json --server http://localhost:18080 --app-id <app-id> execu
 spark-history-cli --json --server http://localhost:18080 --app-id <app-id> sql
 spark-history-cli --json --server http://localhost:18080 --app-id <app-id> sql-plan <exec-id> --view final
 spark-history-cli --server http://localhost:18080 --app-id <app-id> sql-plan <exec-id> --dot -o plan.dot
+spark-history-cli --json --server http://localhost:18080 --app-id <app-id> sql-jobs <exec-id>
 spark-history-cli --json --server http://localhost:18080 --app-id <app-id> env
 spark-history-cli --server http://localhost:18080 --app-id <app-id> logs output.zip
 ```
@@ -62,6 +63,7 @@ python -m spark_history_cli --json apps
   - `--dot`: Graphviz DOT output for visualizing the plan DAG
   - `--json` + `--view`: structured JSON with `isAdaptive`, `sectionCount`, `plan`, and `sections`
   - `-o <file>`: write output to file instead of stdout
+- `sql-jobs <id>` for jobs associated with a SQL execution (fetches all linked jobs by ID)
 - `env` for Spark config/runtime context
 - `logs` only when the user explicitly wants the event log archive saved locally