Merge pull request #8 from unicef/increment-3-pipeline

Merge storyboard view into main

Author: amoghbanta

apps/api/src/app.ts

@@ -6,6 +6,7 @@ import { errorHandler } from "./middleware/error-handler.js"
 import { healthRoutes } from "./routes/health.js"
 import { createBookRoutes } from "./routes/books.js"
 import { createPipelineRoutes } from "./routes/pipeline.js"
+import { createPageRoutes } from "./routes/pages.js"
 import { createPipelineService } from "./services/pipeline-service.js"
 import { createPipelineRunner } from "./services/pipeline-runner.js"
 
@@ -29,5 +30,6 @@ app.onError(errorHandler)
 app.route("/api", healthRoutes)
 app.route("/api", createBookRoutes(booksDir))
 app.route("/api", createPipelineRoutes(pipelineService, booksDir, promptsDir))
+app.route("/api", createPageRoutes(booksDir))
 
 export default app

apps/api/src/routes/pages.test.ts

@@ -0,0 +1,188 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest"
+import fs from "node:fs"
+import path from "node:path"
+import os from "node:os"
+import { Hono } from "hono"
+import { createBookStorage } from "@adt/storage"
+import { errorHandler } from "../middleware/error-handler.js"
+import { createPageRoutes } from "./pages.js"
+
+describe("Page routes", () => {
+  let tmpDir: string
+  let app: Hono
+  const label = "test-book"
+
+  beforeEach(() => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "pages-routes-"))
+
+    // Create a book with extracted pages and pipeline data
+    const storage = createBookStorage(label, tmpDir)
+    try {
+      // Simulate extracted pages
+      const fakeImage = {
+        imageId: `${label}_p1_page`,
+        pngBuffer: Buffer.from("fake-png-data"),
+        hash: "abc123",
+        width: 800,
+        height: 600,
+      }
+      storage.putExtractedPage({
+        pageId: `${label}_p1`,
+        pageNumber: 1,
+        text: "Page one text content",
+        pageImage: fakeImage,
+        images: [],
+      })
+
+      const fakeImage2 = {
+        imageId: `${label}_p2_page`,
+        pngBuffer: Buffer.from("fake-png-data-2"),
+        hash: "def456",
+        width: 800,
+        height: 600,
+      }
+      storage.putExtractedPage({
+        pageId: `${label}_p2`,
+        pageNumber: 2,
+        text: "Page two text content",
+        pageImage: fakeImage2,
+        images: [],
+      })
+
+      // Simulate pipeline output for page 1
+      storage.putNodeData("text-classification", `${label}_p1`, {
+        reasoning: "test reasoning",
+        groups: [
+          {
+            groupId: "g1",
+            groupType: "body",
+            texts: [
+              { textType: "paragraph", text: "Hello world", isPruned: false },
+            ],
+          },
+        ],
+      })
+      storage.putNodeData("image-classification", `${label}_p1`, {
+        images: [],
+      })
+      storage.putNodeData("page-sectioning", `${label}_p1`, {
+        reasoning: "sectioned",
+        sections: [
+          {
+            sectionType: "content",
+            partIds: ["g1"],
+            backgroundColor: "#ffffff",
+            textColor: "#000000",
+            pageNumber: 1,
+            isPruned: false,
+          },
+        ],
+      })
+      storage.putNodeData("web-rendering", `${label}_p1`, {
+        sections: [
+          {
+            sectionIndex: 0,
+            sectionType: "content",
+            reasoning: "rendered",
+            html: "<div>Hello world</div>",
+          },
+        ],
+      })
+    } finally {
+      storage.close()
+    }
+
+    const routes = createPageRoutes(tmpDir)
+    app = new Hono()
+    app.onError(errorHandler)
+    app.route("/api", routes)
+  })
+
+  afterEach(() => {
+    fs.rmSync(tmpDir, { recursive: true, force: true })
+  })
+
+  describe("GET /api/books/:label/pages", () => {
+    it("returns list of pages", async () => {
+      const res = await app.request(`/api/books/${label}/pages`)
+
+      expect(res.status).toBe(200)
+      const body = await res.json()
+      expect(body).toHaveLength(2)
+      expect(body[0].pageId).toBe(`${label}_p1`)
+      expect(body[0].pageNumber).toBe(1)
+      expect(body[0].hasRendering).toBe(true)
+      expect(body[1].pageId).toBe(`${label}_p2`)
+      expect(body[1].pageNumber).toBe(2)
+      expect(body[1].hasRendering).toBe(false)
+    })
+
+    it("returns 404 for nonexistent book", async () => {
+      const res = await app.request("/api/books/no-such-book/pages")
+      expect(res.status).toBe(404)
+    })
+  })
+
+  describe("GET /api/books/:label/pages/:pageId", () => {
+    it("returns full page data with pipeline outputs", async () => {
+      const res = await app.request(
+        `/api/books/${label}/pages/${label}_p1`
+      )
+
+      expect(res.status).toBe(200)
+      const body = await res.json()
+      expect(body.pageId).toBe(`${label}_p1`)
+      expect(body.pageNumber).toBe(1)
+      expect(body.text).toBe("Page one text content")
+      expect(body.textClassification).toBeTruthy()
+      expect(body.textClassification.groups).toHaveLength(1)
+      expect(body.imagClassification).toBeFalsy // typo check
+      expect(body.imageClassification).toBeTruthy()
+      expect(body.sectioning).toBeTruthy()
+      expect(body.sectioning.sections).toHaveLength(1)
+      expect(body.rendering).toBeTruthy()
+      expect(body.rendering.sections[0].html).toBe(
+        "<div>Hello world</div>"
+      )
+    })
+
+    it("returns page without pipeline data if not processed", async () => {
+      const res = await app.request(
+        `/api/books/${label}/pages/${label}_p2`
+      )
+
+      expect(res.status).toBe(200)
+      const body = await res.json()
+      expect(body.pageId).toBe(`${label}_p2`)
+      expect(body.textClassification).toBeNull()
+      expect(body.rendering).toBeNull()
+    })
+
+    it("returns 404 for nonexistent page", async () => {
+      const res = await app.request(
+        `/api/books/${label}/pages/fake-page`
+      )
+      expect(res.status).toBe(404)
+    })
+  })
+
+  describe("GET /api/books/:label/pages/:pageId/image", () => {
+    it("returns page image as base64 JSON", async () => {
+      const res = await app.request(
+        `/api/books/${label}/pages/${label}_p1/image`
+      )
+
+      expect(res.status).toBe(200)
+      const body = await res.json()
+      expect(body.imageBase64).toBeTruthy()
+      expect(typeof body.imageBase64).toBe("string")
+    })
+
+    it("returns 404 for nonexistent page image", async () => {
+      const res = await app.request(
+        `/api/books/${label}/pages/fake-page/image`
+      )
+      expect(res.status).toBe(404)
+    })
+  })
+})

apps/api/src/routes/pages.ts

@@ -0,0 +1,170 @@
+import fs from "node:fs"
+import path from "node:path"
+import { Hono } from "hono"
+import { HTTPException } from "hono/http-exception"
+import { parseBookLabel } from "@adt/types"
+import { openBookDb } from "@adt/storage"
+import { createBookStorage } from "@adt/storage"
+
+interface PageSummary {
+  pageId: string
+  pageNumber: number
+  hasRendering: boolean
+}
+
+interface PageDetail {
+  pageId: string
+  pageNumber: number
+  text: string
+  textClassification: unknown | null
+  imageClassification: unknown | null
+  sectioning: unknown | null
+  rendering: unknown | null
+}
+
+function getDbPath(label: string, booksDir: string): string {
+  const safeLabel = parseBookLabel(label)
+  return path.join(path.resolve(booksDir), safeLabel, `${safeLabel}.db`)
+}
+
+export function createPageRoutes(booksDir: string): Hono {
+  const app = new Hono()
+
+  // GET /books/:label/pages — List pages with pipeline status
+  app.get("/books/:label/pages", (c) => {
+    const { label } = c.req.param()
+    const safeLabel = parseBookLabel(label)
+    const dbPath = getDbPath(safeLabel, booksDir)
+
+    if (!fs.existsSync(dbPath)) {
+      throw new HTTPException(404, {
+        message: `Book not found or not yet extracted: ${safeLabel}`,
+      })
+    }
+
+    const db = openBookDb(dbPath)
+    try {
+      const pages = db.all(
+        "SELECT page_id, page_number FROM pages ORDER BY page_number"
+      ) as Array<{ page_id: string; page_number: number }>
+
+      // Check which pages have web-rendering output
+      const rendered = new Set<string>()
+      const renderRows = db.all(
+        "SELECT DISTINCT item_id FROM node_data WHERE node = ?",
+        ["web-rendering"]
+      ) as Array<{ item_id: string }>
+      for (const row of renderRows) {
+        rendered.add(row.item_id)
+      }
+
+      const result: PageSummary[] = pages.map((p) => ({
+        pageId: p.page_id,
+        pageNumber: p.page_number,
+        hasRendering: rendered.has(p.page_id),
+      }))
+
+      return c.json(result)
+    } finally {
+      db.close()
+    }
+  })
+
+  // GET /books/:label/pages/:pageId — Full page data with pipeline outputs
+  app.get("/books/:label/pages/:pageId", (c) => {
+    const { label, pageId } = c.req.param()
+    const safeLabel = parseBookLabel(label)
+    const dbPath = getDbPath(safeLabel, booksDir)
+
+    if (!fs.existsSync(dbPath)) {
+      throw new HTTPException(404, {
+        message: `Book not found: ${safeLabel}`,
+      })
+    }
+
+    const db = openBookDb(dbPath)
+    try {
+      // Get page data
+      const pageRows = db.all(
+        "SELECT page_id, page_number, text FROM pages WHERE page_id = ?",
+        [pageId]
+      ) as Array<{ page_id: string; page_number: number; text: string }>
+
+      if (pageRows.length === 0) {
+        throw new HTTPException(404, {
+          message: `Page not found: ${pageId}`,
+        })
+      }
+
+      const page = pageRows[0]
+
+      // Get pipeline outputs
+      const getNodeData = (node: string): unknown | null => {
+        const rows = db.all(
+          "SELECT data FROM node_data WHERE node = ? AND item_id = ? ORDER BY version DESC LIMIT 1",
+          [node, pageId]
+        ) as Array<{ data: string }>
+        if (rows.length === 0) return null
+        return JSON.parse(rows[0].data)
+      }
+
+      const result: PageDetail = {
+        pageId: page.page_id,
+        pageNumber: page.page_number,
+        text: page.text,
+        textClassification: getNodeData("text-classification"),
+        imageClassification: getNodeData("image-classification"),
+        sectioning: getNodeData("page-sectioning"),
+        rendering: getNodeData("web-rendering"),
+      }
+
+      return c.json(result)
+    } finally {
+      db.close()
+    }
+  })
+
+  // GET /books/:label/pages/:pageId/image — Page image as base64
+  app.get("/books/:label/pages/:pageId/image", (c) => {
+    const { label, pageId } = c.req.param()
+    const safeLabel = parseBookLabel(label)
+    const resolvedDir = path.resolve(booksDir)
+    const bookDir = path.join(resolvedDir, safeLabel)
+    const dbPath = path.join(bookDir, `${safeLabel}.db`)
+
+    if (!fs.existsSync(dbPath)) {
+      throw new HTTPException(404, {
+        message: `Book not found: ${safeLabel}`,
+      })
+    }
+
+    const db = openBookDb(dbPath)
+    try {
+      // Look up the page image path
+      const imageId = `${pageId}_page`
+      const rows = db.all(
+        "SELECT path FROM images WHERE image_id = ?",
+        [imageId]
+      ) as Array<{ path: string }>
+
+      if (rows.length === 0) {
+        throw new HTTPException(404, {
+          message: `Page image not found: ${pageId}`,
+        })
+      }
+
+      const imagePath = path.resolve(bookDir, rows[0].path)
+      // Verify path doesn't escape book directory
+      if (!imagePath.startsWith(bookDir + path.sep) && imagePath !== bookDir) {
+        throw new HTTPException(400, { message: "Invalid image path" })
+      }
+
+      const imageBase64 = fs.readFileSync(imagePath).toString("base64")
+      return c.json({ imageBase64 })
+    } finally {
+      db.close()
+    }
+  })
+
+  return app
+}