Merge pull request #4 from unicef/nicpottier/section-render

Add page sectioning, web rendering and replace Listr2 with cli-progress

Author: nicpottier

config.yaml

@@ -27,20 +27,57 @@ text_group_types:
   list: A list of items (ordered or unordered)
   other: Anything that doesn't fit the above
 
+section_types:
+  front_cover: Front cover page (first page only)
+  inside_cover: Inside cover page
+  back_cover: Back cover page (last page only)
+  separator: Separator page between logical sections
+  credits: Credits and acknowledgements
+  foreword: Introduction, overview, or author's note
+  table_of_contents: Table of contents
+  boxed_text: Text in a box or callout
+  text_only: Reading section with only text
+  text_and_single_image: Section with text and a single image
+  text_and_images: Reading section with text and multiple images
+  images_only: Section with only images
+  activity_matching: Matching activity
+  activity_fill_in_a_table: Table fill-in activity
+  activity_multiple_choice: Multiple choice activity
+  activity_true_false: True or false activity
+  activity_open_ended_answer: Open-ended text response activity
+  activity_fill_in_the_blank: Fill in the blank activity
+  activity_sorting: Sorting activity
+  other: Any other section type
+
 metadata:
   prompt: metadata_extraction
   model: openai:gpt-4o
 
 text_classification:
   prompt: text_classification
   model: openai:gpt-4o
-  concurrency: 5
+  concurrency: 16
+
+page_sectioning:
+  prompt: page_sectioning
+  model: openai:gpt-4o
+
+web_rendering:
+  prompt: web_generation_html
+  model: openai:gpt-4o
+  concurrency: 16
+  max_retries: 8
 
 pruned_text_types:
   - header_text
   - footer_text
   - page_number
 
+pruned_section_types:
+  - back_cover
+  - credits
+  - inside_cover
+
 image_filters:
   min_side: 100
   max_side: 5000

packages/pipeline/package.json

@@ -11,12 +11,16 @@
     "pipeline": "./dist/cli.js"
   },
   "dependencies": {
+    "@adt/llm": "workspace:*",
     "@adt/pdf": "workspace:*",
-    "@adt/types": "workspace:*",
     "@adt/storage": "workspace:*",
-    "@adt/llm": "workspace:*",
+    "@adt/types": "workspace:*",
+    "cli-progress": "^3.12.0",
+    "htmlparser2": "^10.1.0",
     "js-yaml": "^4.1.0",
-    "listr2": "^9.0.0",
     "zod": "^3.24.0"
+  },
+  "devDependencies": {
+    "@types/cli-progress": "^3.11.6"
   }
 }