test

pearson_json/final_subtopics.json

@@ -0,0 +1,142 @@
+[
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [],
+    "children": []
+  }
+]

topic_extr.py

@@ -169,36 +169,149 @@ async def classify_image_async(image_data: bytes, api_key: str, max_retries: int
     return await loop.run_in_executor(None, call_gemini_for_table_classification, preprocessed, api_key, max_retries)
 
 
-def call_gemini_for_subtopic_identification(text: str, api_key: str, max_retries: int = 1) -> dict:
-    """
-    Sends the recognized text from a specification table to Gemini,
-    asking it to identify the main topic (like '2 Algebra and functions')
-    and subtopics (like '2.5', '3.4', etc.).
-
-    Returns a dict of the form:
-    {
-      "title": "2 Algebra and functions",
-      "subtopics": ["2.5", "2.6", ...]
-    }
-
-    If Gemini can't find anything, it might return empty strings or lists.
-    """
+# def call_gemini_for_subtopic_identification_image(image_data: bytes, api_key: str, max_retries: int = 1) -> dict:
+#     for attempt in range(max_retries + 1):
+#         try:
+#             prompt = """
+# You are given an image from an educational curriculum specification. The image may contain either:
+# 1) A main topic heading in the format: "<number> <Topic Name>", for example "2 Algebra and functions continued".
+# 2) A subtopic heading in the format "<number>.<number>", for example "2.5", "2.6", or "3.4".
+# 3) Possibly no relevant text at all.
+
+# Your task:
+# 1. If the cell shows a main topic, extract the topic name (e.g. "2 Algebra and functions") and place it in the JSON key "title".
+# 2. If the cell shows one or more subtopic numbers (e.g. "2.5", "2.6"), collect them in the JSON key "subtopics" as an array of strings.
+# 3. If neither a main topic nor subtopic is detected, return empty values.
+
+# Output only valid JSON in this exact structure, with no extra text or explanation:
+
+# {
+#   "title": "...",
+#   "subtopics": [...]
+# }
+
+# Where:
+# - "title" is the recognized main topic (if any). Otherwise, an empty string.
+# - "subtopics" is an array of recognized subtopic numbers (e.g. ["2.5", "2.6"]). Otherwise, an empty array.
+
+# Examples:
+# 1. If the image text is "2 Algebra and functions continued", return:
+# {
+#   "title": "2 Algebra and functions continued",
+#   "subtopics": []
+# }
+
+# 2. If the image text is "2.5 Solve linear and quadratic inequalities ...", return:
+# {
+#   "title": "",
+#   "subtopics": ["2.5"]
+# }
+
+# 3. If the image text is "2.6 Manipulate polynomials algebraically ...", return:
+# {
+#   "title": "",
+#   "subtopics": ["2.6"]
+# }
+
+# If you cannot recognize any text matching these patterns, or if nothing is found, return:
+# {
+#   "title": "",
+#   "subtopics": []
+# }
+# """
+#             global _GEMINI_CLIENT
+#             if _GEMINI_CLIENT is None:
+#                 _GEMINI_CLIENT = genai.Client(api_key=api_key)
+#             client = _GEMINI_CLIENT
+
+#             resp = client.models.generate_content(
+#                 model="gemini-2.0-flash",
+#                 contents=[
+#                     {
+#                         "parts": [
+#                             {"text": prompt},
+#                             {
+#                                 "inline_data": {
+#                                     "mime_type": "image/jpeg",
+#                                     "data": base64.b64encode(image_data).decode("utf-8")
+#                                 }
+#                             }
+#                         ]
+#                     }
+#                 ],
+#                 config=types.GenerateContentConfig(temperature=0.0)
+#             )
+#             if not resp or not resp.text:
+#                 return {"title": "", "subtopics": []}
+
+#             raw = resp.text.strip()
+
+#             data = json.loads(raw)
+#             title = data.get("title", "")
+#             subtopics = data.get("subtopics", [])
+#             if not isinstance(subtopics, list):
+#                 subtopics = []
+#             return {"title": title, "subtopics": subtopics}
+
+#         except Exception as e:
+#             if attempt < max_retries:
+#                 time.sleep(0.5)
+#             else:
+#                 return {"title": "", "subtopics": []}
+
+    # return {"title": "", "subtopics": []}
+
+def call_gemini_for_subtopic_identification_image(image_data: bytes, api_key: str, max_retries: int = 1) -> dict:
     for attempt in range(max_retries + 1):
         try:
-            prompt = f"""
-You are given text extracted from a table that represents topics and subtopics from an educational curriculum.
-The text may include a main topic heading in the format: "<number> <Topic Name>", for example, "2 Algebra and functions".
-It may also include subtopics in the format of "<number>.<number>", such as "2.5", "3.4", etc.
-Extract and output a valid JSON object with exactly two keys:
-  - "title": the main topic heading (if found). If not found, use an empty string.
-  - "subtopics": an array of strings representing each subtopic number extracted from the text.
-Output exactly in this JSON format with no additional text. For example:
+            prompt = """
+            You are given an image from an educational curriculum specification. The image may contain either:
+1) A main topic heading in the format: "<number> <Topic Name>", for example "2 Algebra and functions continued".
+2) A subtopic heading in the format "<number>.<number>", for example "2.5", "2.6", or "3.4".
+3) Possibly no relevant text at all.
+
+Your task:
+1. If the cell shows a main topic, extract the topic name (e.g. "2 Algebra and functions") and place it in the JSON key "title".
+2. If the cell shows one or more subtopic numbers (e.g. "2.5", "2.6"), collect them in the JSON key "subtopics" as an array of strings.
+3. If neither a main topic nor subtopic is detected, return empty values.
+
+Output only valid JSON in this exact structure, with no extra text or explanation:
+
+Output only valid JSON in this exact structure, with no extra text or explanation:
+
+{
+  "title": "...",
+  "subtopics": [...]
+}
+
+Where:
+- "title" is the recognized main topic (if any). Otherwise, an empty string.
+- "subtopics" is an array of recognized subtopic numbers (e.g. ["2.5", "2.6"]). Otherwise, an empty array.
+
+Examples:
+1. If the image text is "2 Algebra and functions continued", return:
+{
+  "title": "2 Algebra and functions continued",
+  "subtopics": []
+}
+
+2. If the image text is "2.5 Solve linear and quadratic inequalities ...", return:
+{
+  "title": "",
+  "subtopics": ["2.5"]
+}
+
+3. If the image text is "2.6 Manipulate polynomials algebraically ...", return:
+{
+  "title": "",
+  "subtopics": ["2.6"]
+}
+
+If you cannot recognize any text matching these patterns, or if nothing is found, return:
 {
-  "title": "2 Algebra and functions",
-  "subtopics": ["2.5", "2.6"]
+  "title": "",
+  "subtopics": []
 }
-Text:
-{text}
 """
             global _GEMINI_CLIENT
             if _GEMINI_CLIENT is None:
@@ -207,36 +320,45 @@ Text:
 
             resp = client.models.generate_content(
                 model="gemini-2.0-flash",
-                contents=[prompt],
+                contents=[
+                    {
+                        "parts": [
+                            {"text": prompt},
+                            {
+                                "inline_data": {
+                                    "mime_type": "image/jpeg",
+                                    "data": base64.b64encode(image_data).decode("utf-8")
+                                }
+                            }
+                        ]
+                    }
+                ],
                 config=types.GenerateContentConfig(temperature=0.0)
             )
-
+            # Log the raw response
+            logger.info(f"Gemini subtopic extraction raw response: {resp.text if resp and resp.text else 'None'}")
+            
             if not resp or not resp.text:
-                # If Gemini gives no response, fallback to empty.
+                logger.warning("Gemini returned an empty response for subtopic extraction.")
                 return {"title": "", "subtopics": []}
 
             raw = resp.text.strip()
-
-            # Attempt to parse raw as JSON
-            try:
-                data = json.loads(raw)
-                # Guarantee the structure we want
-                title = data.get("title", "")
-                subs = data.get("subtopics", [])
-                if not isinstance(subs, list):
-                    subs = []
-                return {"title": title, "subtopics": subs}
-            except Exception:
-                # If JSON parse fails, return empty
-                return {"title": "", "subtopics": []}
+            data = json.loads(raw)
+            title = data.get("title", "")
+            subtopics = data.get("subtopics", [])
+            if not isinstance(subtopics, list):
+                subtopics = []
+            return {"title": title, "subtopics": subtopics}
 
         except Exception as e:
-            # If there's an error or a 503, we can retry or bail out
+            logger.error(f"Gemini subtopic identification error on attempt {attempt}: {e}")
             if attempt < max_retries:
                 time.sleep(0.5)
             else:
                 return {"title": "", "subtopics": []}
 
+    return {"title": "", "subtopics": []}
+
 
 
 class S3ImageWriter(DataWriter):
@@ -314,11 +436,13 @@ class S3ImageWriter(DataWriter):
                 logger.warning(f"No image data found for S3 key {s3_key}. Skipping.")
                 continue
 
+            # Write temporary file for processing.
             with tempfile.NamedTemporaryFile(delete=False, suffix=".jpg") as temp_file:
                 temp_file.write(img_data)
                 temp_path = temp_file.name
 
             try:
+                # 1) Extract row bounding boxes.
                 if col_type.lower() == 'two':
                     extractor = TableExtractor(
                         skip_header=True,
@@ -334,42 +458,77 @@ class S3ImageWriter(DataWriter):
                         subtopic_threshold=0.2
                     )
                 row_boxes = extractor.process_image(temp_path)
-
-                snippet = ["**Extracted table cells:**"]
-                cell_texts = []
+                logger.info(f"Extracted {len(row_boxes)} rows from {temp_path}")
                 for i, row in enumerate(row_boxes):
-                    for j, box in enumerate(row):
-                        cell_key = f"{self.base_path}cells/table_s3_{os.path.basename(s3_key)}_r{i}_c{j}.jpg"
-                        self.s3_writer.write(cell_key, img_data)  # or cell_data if you truly cropped
+                    logger.info(f"Row {i} has {len(row)} cells")
 
-                        text = "..."  # placeholder
-                        cell_texts.append(text)
-
-                        snippet.append(f"![Row {i} Col {j}]({cell_key})")
+                # out_folder = temp_path + "_rows"
+                # os.makedirs(out_folder, exist_ok=True)
+                out_folder = os.path.join(os.path.dirname(temp_path), os.path.basename(temp_path) + "_rows")
+                os.makedirs(out_folder, exist_ok=True)
 
-                combined_text = "\n".join(cell_texts)
+                extractor.save_extracted_cells(temp_path, row_boxes, out_folder)
+                logger.info(f"Files in {out_folder}:")
+                for root, dirs, files in os.walk(out_folder):
+                    logger.info(f"{root}: {files}")
 
-                subtopic_info = call_gemini_for_subtopic_identification(combined_text, self.gemini_api_key)
+                recognized_main_topic = ""
+                main_topic_image_key = None
+                recognized_subtopics = []
 
-                # subtopic_info might be: {"title": "2 Algebra and functions", "subtopics": ["2.5"]}
+                # 2) Loop over each cell image.
+                for i, row in enumerate(row_boxes):
+                    row_dir = os.path.join(out_folder, f"row_{i}")
+                    for j, _ in enumerate(row):
+                        cell_path = os.path.join(row_dir, f"col_{j}.png")
+                        if not os.path.isfile(cell_path):
+                            alternative_path = os.path.join(row_dir, f"col_{j}.jpg")
+                            if os.path.isfile(alternative_path):
+                                cell_path = alternative_path
+                            else:
+                                logger.warning(f"Cell image not found: {cell_path}")
+                                continue
+
+                        with open(cell_path, "rb") as cf:
+                            cell_image_data = cf.read()
+
+                        # Save cell image to S3.
+                        cell_key = f"{self.base_path}cells/{os.path.basename(s3_key)}_r{i}_c{j}.png"
+                        self.s3_writer.write(cell_key, cell_image_data)
+                        
+                        # Log before calling Gemini.
+                        logger.debug(f"About to call Gemini for cell image: {cell_path}")
+                        info = call_gemini_for_subtopic_identification_image(cell_image_data, self.gemini_api_key)
+                        logger.info(f"Gemini subtopic extraction result for cell {cell_path}: {info}")
+
+                        if info["title"] and not recognized_main_topic:
+                            recognized_main_topic = info["title"]
+                            main_topic_image_key = cell_key
+
+                        for st in info["subtopics"]:
+                            recognized_subtopics.append({
+                                "title": st,
+                                "contents": [{"type": "image", "key": cell_key}],
+                                "children": []
+                            })
+
+                # 3) Build final JSON for this table.
                 final_json = {
-                    "title": subtopic_info.get("title", ""),
-                    "contents": [
-                        {"type": "image", "key": s3_key}
-                    ],
-                    "children": []
+                    "title": recognized_main_topic,
+                    "contents": [],
+                    "children": recognized_subtopics
                 }
-                for st in subtopic_info.get("subtopics", []):
-                    final_json["children"].append({
-                        "title": st,
-                        "contents": [
-                            {"type": "image", "key": f"subtopic_{st}_example.jpg"}
-                        ]
-                    })
+                if main_topic_image_key:
+                    final_json["contents"].append({"type": "image", "key": main_topic_image_key})
 
+                # Save the final JSON.
                 self.extracted_subtopics[s3_key] = final_json
 
-                # Replace the original table image line in the markdown with the snippet
+                # Optionally, create a snippet to replace the markdown line.
+                snippet = ["**Extracted table cells:**"]
+                for i, row in enumerate(row_boxes):
+                    for j, _ in enumerate(row):
+                        snippet.append(f"![Row {i} Col {j}]({self.base_path}cells/{os.path.basename(s3_key)}_r{i}_c{j}.jpg)")
                 new_snip = "\n".join(snippet)
                 old_line = f"![HAS TO BE PROCESSED - {col_type} column table]({s3_key})"
                 md_content = md_content.replace(old_line, new_snip)
@@ -381,6 +540,8 @@ class S3ImageWriter(DataWriter):
 
         return md_content
 
+
+
     def post_process(self, key: str, md_content: str) -> str:
         return asyncio.run(self.post_process_async(key, md_content))
 
@@ -457,7 +618,7 @@ class LocalImageWriter(DataWriter):
                 with open(temp_path, "wb") as f:
                     f.write(desc_item["data"])
             try:
-                if col_type.lower() == 'two':
+                if col_type.lower() == 'two':       #check for table_row_extr script for more details
                     extractor = TableExtractor(
                         skip_header=True,
                         merge_two_col_rows=True,
@@ -822,7 +983,7 @@ class MineruNoTextProcessor:
 
 if __name__ == "__main__":
     input_pdf = "/home/user/app/input_output/a-level-pearson-mathematics-specification.pdf"
-    output_dir = "/home/user/app/we"
+    output_dir = "/home/user/app/pearson_json"
     gemini_key = os.getenv("GEMINI_API_KEY", "AIzaSyDtoakpXa2pjJwcQB6TJ5QaXHNSA5JxcrU")
     try:
         processor = MineruNoTextProcessor(output_folder=output_dir, gemini_api_key=gemini_key)
@@ -830,4 +991,4 @@ if __name__ == "__main__":
         logger.info("Processing completed successfully.")
         # The result includes final_markdown and subtopics_extracted
     except Exception as e:
-        logger.error(f"Processing failed: {e}")
+        logger.error(f"Processing failed: {e}")

topic_extract_arsenii.py

@@ -0,0 +1,883 @@
+#!/usr/bin/env python3
+import os
+import re
+import gc
+import json
+import logging
+import fitz
+import boto3
+import base64
+import time
+import asyncio
+import tempfile
+import requests
+from io import BytesIO
+from typing import List, Dict, Any
+
+import torch
+import cv2
+import numpy as np
+
+from google import genai
+from google.genai import types
+
+from magic_pdf.data.dataset import PymuDocDataset
+from magic_pdf.model.doc_analyze_by_custom_model import doc_analyze
+from magic_pdf.data.data_reader_writer.base import DataWriter
+from table_row_extraction import TableExtractor
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+file_handler = logging.FileHandler("topic_extraction_ars.log")
+file_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s - %(message)s"))
+logger.addHandler(file_handler)
+
+_GEMINI_CLIENT = None
+
+def unify_whitespace(text: str) -> str:
+    return re.sub(r"\s+", " ", text).strip()
+
+def find_all_occurrences(pdf_bytes: bytes, search_text: str) -> List[int]:
+    doc = fitz.open(stream=pdf_bytes, filetype="pdf")
+    st_norm = unify_whitespace(search_text)
+    found = []
+    for i in range(doc.page_count):
+        raw = doc[i].get_text("raw")
+        norm = unify_whitespace(raw)
+        if st_norm in norm:
+            found.append(i)
+    doc.close()
+    return sorted(found)
+
+def create_subset_pdf(original_pdf_bytes: bytes, page_indices: List[int]) -> bytes:
+    if not page_indices:
+        raise ValueError("No page indices provided for subset creation.")
+    doc = fitz.open(stream=original_pdf_bytes, filetype="pdf")
+    new_doc = fitz.open()
+    for p in sorted(set(page_indices)):
+        if 0 <= p < doc.page_count:
+            new_doc.insert_pdf(doc, from_page=p, to_page=p)
+        else:
+            logger.error(f"Page index {p} out of range (0..{doc.page_count - 1}).")
+            raise ValueError(f"Page index {p} out of range.")
+    subset_bytes = new_doc.tobytes()
+    new_doc.close()
+    doc.close()
+    return subset_bytes
+
+class s3Writer:
+    def __init__(self, ak: str, sk: str, bucket: str, endpoint_url: str):
+        self.bucket = bucket
+        self.client = boto3.client(
+            's3',
+            aws_access_key_id=ak,
+            aws_secret_access_key=sk,
+            endpoint_url=endpoint_url
+        )
+
+    def write(self, path: str, data: bytes) -> None:
+        try:
+            file_obj = BytesIO(data)
+            self.client.upload_fileobj(
+                file_obj,
+                self.bucket,
+                path
+            )
+            logger.info(f"Uploaded to S3: {path}")
+        except Exception as e:
+            logger.error(f"Failed to upload to S3: {str(e)}")
+            raise
+
+def preprocess_image(image_data: bytes, max_dim: int = 600, quality: int = 60) -> bytes:
+    arr = np.frombuffer(image_data, np.uint8)
+    img = cv2.imdecode(arr, cv2.IMREAD_COLOR)
+    if img is not None:
+        h, w, _ = img.shape
+        if max(h, w) > max_dim:
+            scale = max_dim / float(max(h, w))
+            new_w = int(w * scale)
+            new_h = int(h * scale)
+            img = cv2.resize(img, (new_w, new_h), interpolation=cv2.INTER_AREA)
+        encode_params = [int(cv2.IMWRITE_JPEG_QUALITY), quality]
+        success, enc = cv2.imencode(".jpg", img, encode_params)
+        if success:
+            return enc.tobytes()
+    return image_data
+
+def call_gemini_for_table_classification(image_data: bytes, api_key: str, max_retries: int = 1) -> str:
+    """
+    Existing Gemini call to classify an image as TWO_COLUMN, THREE_COLUMN, or NO_TABLE.
+    """
+    for attempt in range(max_retries + 1):
+        try:
+            prompt = """You are given an image. Determine if it shows a table that has exactly 2 or 3 columns. 
+The three-column 'table' image includes such key features:
+    - Three columns header
+    - Headers like 'Topics', 'Content', 'Guidelines'
+    - Possibly sections (e.g. 8.4, 9.1)
+The two-column 'table' image includes such key features:
+    - Two columns
+    - Headers like 'Subject content' and 'Additional information'
+    - Possibly sections (e.g. 2.1, 3.4)
+If the image is a relevant table with 2 columns, respond with 'TWO_COLUMN'.
+If the image is a relevant table with 3 columns, respond with 'THREE_COLUMN'.
+If the image does not show a table at all, respond with 'NO_TABLE'.
+Return only one of these exact labels.
+"""
+            global _GEMINI_CLIENT
+            if _GEMINI_CLIENT is None:
+                _GEMINI_CLIENT = genai.Client(api_key=api_key)
+            client = _GEMINI_CLIENT
+
+            resp = client.models.generate_content(
+                model="gemini-2.0-flash",
+                contents=[
+                    {
+                        "parts": [
+                            {"text": prompt},
+                            {
+                                "inline_data": {
+                                    "mime_type": "image/jpeg",
+                                    "data": base64.b64encode(image_data).decode('utf-8')
+                                }
+                            }
+                        ]
+                    }
+                ],
+                config=types.GenerateContentConfig(temperature=0.0)
+            )
+            if resp and resp.text:
+                classification = resp.text.strip().upper()
+                if "THREE" in classification:
+                    return "THREE_COLUMN"
+                elif "TWO" in classification:
+                    return "TWO_COLUMN"
+            return "NO_TABLE"
+        except Exception as e:
+            logger.error(f"Gemini table classification error: {e}")
+            if "503" in str(e):
+                return "NO_TABLE"
+            if attempt < max_retries:
+                time.sleep(0.5)
+            else:
+                return "NO_TABLE"
+
+async def classify_image_async(image_data: bytes, api_key: str, max_retries: int = 1) -> str:
+    loop = asyncio.get_event_loop()
+    preprocessed = preprocess_image(image_data)
+    return await loop.run_in_executor(None, call_gemini_for_table_classification, preprocessed, api_key, max_retries)
+
+
+def call_gemini_for_subtopic_identification_image(image_data: bytes, api_key: str, max_retries: int = 1) -> dict:
+    """
+    Sends the *image* (not text) of a table cell to Gemini to identify:
+      - A main topic heading in the format: "<number> <Topic Name>", e.g. "2 Algebra and functions"
+      - A subtopic heading in the format: "<number>.<number>", e.g. "2.5", "3.4"
+    Returns a dict of the form:
+      {
+        "title": "<the recognized main topic or empty if not found>",
+        "subtopics": ["2.5", "2.6", ...]
+      }
+    """
+    for attempt in range(max_retries + 1):
+        try:
+            # Prompt specifically instructs Gemini to read the image’s text and extract
+            # either a main topic or subtopic heading if present:
+            prompt = """
+                You are given an image of a table cell from an educational curriculum specification. 
+                The text in this cell may contain:
+                1) A main topic heading in the format "<number> <Topic Name>", for example: "2 Algebra and functions"
+                2) A subtopic heading in the format "<number>.<number>", for example: "2.5" or "3.4"
+                Identify if the cell contains exactly one main topic or subtopic. 
+                Return a valid JSON object with the keys "title" and "subtopics" of the form:
+                {{
+                "title": "2 Algebra and functions",
+                "subtopics": ["2.5", "2.6"]
+                }}
+                If you find a main topic (like '2 Algebra and functions'), put it in "title". 
+                If you find subtopic numbers (like '2.5', '3.4'), put them in the "subtopics" array. 
+                """
+
+            # Re-use or initialize your global Gemini client:
+            global _GEMINI_CLIENT
+            if _GEMINI_CLIENT is None:
+                _GEMINI_CLIENT = genai.Client(api_key=api_key)
+            client = _GEMINI_CLIENT
+
+            # Send the prompt + image to Gemini:
+            resp = client.models.generate_content(
+                model="gemini-2.0-flash",
+                contents=[
+                    {
+                        "parts": [
+                            {"text": prompt},
+                            {
+                                "inline_data": {
+                                    "mime_type": "image/jpeg",
+                                    "data": base64.b64encode(image_data).decode("utf-8")
+                                }
+                            }
+                        ]
+                    }
+                ],
+                config=types.GenerateContentConfig(temperature=0.0)
+            )
+            # if not resp or not resp.text:
+            #     return {"title": "", "subtopics": []}
+
+            raw = resp.text.strip().replace("```json", "").replace("```", "")
+            logger.info(f"== RAW == {raw}")
+
+            # Attempt to parse JSON from Gemini’s response:
+            data = json.loads(raw)
+            title = data.get("title", "")
+            subtopics = data.get("subtopics", [])
+            if not isinstance(subtopics, list):
+                subtopics = []
+            return {"title": title, "subtopics": subtopics}
+
+        except Exception as e:
+            # Retry logic if you like:
+            if attempt < max_retries:
+                time.sleep(0.5)
+            else:
+                return {"title": "", "subtopics": []}
+    # fallback:
+    return {"title": "", "subtopics": []}
+
+
+
+
+class S3ImageWriter(DataWriter):
+    def __init__(self, s3_writer: s3Writer, base_path: str, gemini_api_key: str):
+        self.s3_writer = s3_writer
+        self.base_path = base_path if base_path.endswith("/") else base_path + "/"
+        self.gemini_api_key = gemini_api_key
+        self.descriptions = {}
+        self._img_count = 0
+        self.extracted_tables = {}
+        # New attribute to store final subtopic JSON
+        self.extracted_subtopics = {}
+
+    def write(self, path: str, data: bytes) -> None:
+        self._img_count += 1
+        unique_id = f"img_{self._img_count}.jpg"
+        s3_key = f"{self.base_path}{unique_id}"
+        self.s3_writer.write(s3_key, data)
+        self.descriptions[path] = {
+            "data": data,
+            "s3_path": s3_key,
+            "table_classification": "NO_TABLE",
+            "final_alt": ""
+        }
+
+    async def post_process_async(self, key: str, md_content: str) -> str:
+        logger.info("Classifying images to detect tables.")
+        tasks = {
+            p: asyncio.create_task(classify_image_async(info["data"], self.gemini_api_key))
+            for p, info in self.descriptions.items()
+        }
+        results = await asyncio.gather(*tasks.values(), return_exceptions=True)
+        for p, result in zip(tasks.keys(), results):
+            if isinstance(result, Exception):
+                logger.error(f"Table classification error for {p}: {result}")
+                self.descriptions[p]['table_classification'] = "NO_TABLE"
+            else:
+                self.descriptions[p]['table_classification'] = result
+
+        # 2) Replace the original markdown references with alt text
+        for p, info in self.descriptions.items():
+            cls = info['table_classification']
+            if cls == "TWO_COLUMN":
+                info['final_alt'] = "HAS TO BE PROCESSED - two column table"
+            elif cls == "THREE_COLUMN":
+                info['final_alt'] = "HAS TO BE PROCESSED - three column table"
+            else:
+                info['final_alt'] = "NO_TABLE image"
+            md_content = md_content.replace(f"![]({key}{p})", f"![{info['final_alt']}]({info['s3_path']})")
+
+        md_content = await self._process_table_images_in_markdown(key, md_content)
+
+        # Filter final lines to keep only lines with images
+        final_lines = [
+            line.strip() for line in md_content.split("\n")
+            if re.match(r"^\!\[.*\]\(.*\)", line.strip())
+        ]
+        return "\n".join(final_lines)
+
+
+    async def _process_table_images_in_markdown(self, key: str, md_content: str) -> str:
+        pat = r"!\[HAS TO BE PROCESSED - (two|three) column table\]\(([^)]+)\)"
+        matches = re.findall(pat, md_content, flags=re.IGNORECASE)
+        if not matches:
+            return md_content
+
+        for (col_type, s3_key) in matches:
+            logger.info(f"Processing table image: {s3_key}, columns={col_type}")
+            img_data = None
+            for desc in self.descriptions.values():
+                if desc.get("s3_path") == s3_key:
+                    img_data = desc.get("data")
+                    break
+            if img_data is None:
+                logger.warning(f"No image data found for S3 key {s3_key}. Skipping.")
+                continue
+
+            with tempfile.NamedTemporaryFile(delete=False, suffix=".jpg") as temp_file:
+                temp_file.write(img_data)
+                temp_path = temp_file.name
+
+            try:
+                if col_type.lower() == 'two':
+                    extractor = TableExtractor(
+                        skip_header=True,
+                        merge_two_col_rows=True,
+                        enable_subtopic_merge=True,
+                        subtopic_threshold=0.2
+                    )
+                else:
+                    extractor = TableExtractor(
+                        skip_header=True,
+                        merge_two_col_rows=False,
+                        enable_subtopic_merge=False,
+                        subtopic_threshold=0.2
+                    )
+                row_boxes = extractor.process_image(temp_path)
+
+                #save cell images to S3 or local
+                out_folder = temp_path + "_rows"
+                os.makedirs(out_folder, exist_ok=True)
+                extractor.save_extracted_cells(temp_path, row_boxes, out_folder)
+
+                recognized_main_topic = None
+                recognized_subtopics = []
+
+                for i, row in enumerate(row_boxes):
+                    row_dir = os.path.join(out_folder, f"row_{i}")
+                    for j, _ in enumerate(row):
+                        cell_path = os.path.join(row_dir, f"col_{j}.jpg")
+                        if not os.path.isfile(cell_path):
+                            continue
+
+                        with open(cell_path, "rb") as cf:
+                            cell_image_data = cf.read()
+
+                        # store that cell image to S3
+                        cell_key = f"{self.base_path}cells/{os.path.basename(s3_key)}_r{i}_c{j}.jpg"
+                        self.s3_writer.write(cell_key, cell_image_data)
+
+                        # Call Gemini with the cell image
+                        info = call_gemini_for_subtopic_identification_image(cell_image_data, self.gemini_api_key)
+                        logger.info(f"== INFO == {info}")
+                        # e.g. info = {"title": "2 Algebra and functions", "subtopics": ["2.5"]}
+
+                        # 3d) Merge the recognized topic/subtopics
+                        if info["title"]:
+                            recognized_main_topic = info["title"]
+                        if info["subtopics"]:
+                            recognized_subtopics.extend(info["subtopics"])
+
+                snippet = ["**Extracted table cells:**"]
+                cell_texts = []
+                for i, row in enumerate(row_boxes):
+                    for j, box in enumerate(row):
+                        cell_key = f"{self.base_path}cells/table_s3_{os.path.basename(s3_key)}_r{i}_c{j}.jpg"
+                        self.s3_writer.write(cell_key, img_data)  # or cell_data if you truly cropped
+
+                        text = "..."  # placeholder
+                        cell_texts.append(text)
+
+                        snippet.append(f"![Row {i} Col {j}]({cell_key})")
+
+                final_json = {
+                "title": recognized_main_topic,
+                "contents": [
+                    {
+                        "type": "image",
+                        "key": s3_key
+                    }
+                ],
+                "children": []
+                }
+                for st in recognized_subtopics:
+                    final_json["children"].append({
+                        "title": st,
+                        "contents": [
+                            {"type": "image", "key": f"subtopic_{st}_example.jpg"}
+                        ]
+                    })
+
+                self.extracted_subtopics[s3_key] = final_json
+
+
+                # Replace the original table image line in the markdown with the snippet
+                new_snip = "\n".join(snippet)
+                old_line = f"![HAS TO BE PROCESSED - {col_type} column table]({s3_key})"
+                md_content = md_content.replace(old_line, new_snip)
+
+                snippet = ["**Extracted table cells:**"]
+                for i, row in enumerate(row_boxes):
+                    for j, _ in enumerate(row):
+                        snippet.append(f"![Row {i} Col {j}]({self.base_path}cells/{os.path.basename(s3_key)}_r{i}_c{j}.jpg)")
+                new_snip = "\n".join(snippet)
+                old_line = f"![HAS TO BE PROCESSED - {col_type} column table]({s3_key})"
+                md_content = md_content.replace(old_line, new_snip)
+
+            except Exception as e:
+                logger.error(f"Error processing table image {s3_key}: {e}")
+            finally:
+                os.remove(temp_path)
+
+        return md_content
+
+    def post_process(self, key: str, md_content: str) -> str:
+        return asyncio.run(self.post_process_async(key, md_content))
+
+
+class LocalImageWriter(DataWriter):
+    def __init__(self, output_folder: str, gemini_api_key: str):
+        self.output_folder = output_folder
+        os.makedirs(self.output_folder, exist_ok=True)
+        self.descriptions = {}
+        self._img_count = 0
+        self.gemini_api_key = gemini_api_key
+
+        self.extracted_tables = {}
+
+    def write(self, path: str, data: bytes) -> None:
+        self._img_count += 1
+        unique_id = f"img_{self._img_count}.jpg"
+        self.descriptions[path] = {
+            "data": data,
+            "relative_path": unique_id,
+            "table_classification": "NO_TABLE",
+            "final_alt": ""
+        }
+        # Also save the original image locally for testing.
+        image_path = os.path.join(self.output_folder, unique_id)
+        with open(image_path, "wb") as f:
+            f.write(data)
+
+    async def post_process_async(self, key: str, md_content: str) -> str:
+        logger.info("Classifying images to detect tables.")
+        tasks = []
+        for p, info in self.descriptions.items():
+            tasks.append((p, classify_image_async(info["data"], self.gemini_api_key)))
+        for p, task in tasks:
+            try:
+                classification = await task
+                self.descriptions[p]['table_classification'] = classification
+            except Exception as e:
+                logger.error(f"Table classification error: {e}")
+                self.descriptions[p]['table_classification'] = "NO_TABLE"
+        for p, info in self.descriptions.items():
+            cls = info['table_classification']
+            if cls == "TWO_COLUMN":
+                info['final_alt'] = "HAS TO BE PROCESSED - two column table"
+            elif cls == "THREE_COLUMN":
+                info['final_alt'] = "HAS TO BE PROCESSED - three column table"
+            else:
+                info['final_alt'] = "NO_TABLE image"
+            md_content = md_content.replace(f"![]({key}{p})", f"![{info['final_alt']}]({info['relative_path']})")
+        md_content = self._process_table_images_in_markdown(md_content)
+        final_lines = []
+        for line in md_content.split("\n"):
+            if re.match(r"^\!\[.*\]\(.*\)", line.strip()):
+                final_lines.append(line.strip())
+        return "\n".join(final_lines)
+
+    def _process_table_images_in_markdown(self, md_content: str) -> str:
+        pat = r"!\[HAS TO BE PROCESSED - (two|three) column table\]\(([^)]+)\)"
+        matches = re.findall(pat, md_content, flags=re.IGNORECASE)
+        if not matches:
+            return md_content
+        for (col_type, image_id) in matches:
+            logger.info(f"Processing table image => {image_id}, columns={col_type}")
+            temp_path = os.path.join(self.output_folder, image_id)
+            desc_item = None
+            for k, val in self.descriptions.items():
+                if val["relative_path"] == image_id:
+                    desc_item = val
+                    break
+            if not desc_item:
+                logger.warning(f"No matching image data for {image_id}, skipping extraction.")
+                continue
+            if not os.path.exists(temp_path):
+                with open(temp_path, "wb") as f:
+                    f.write(desc_item["data"])
+            try:
+                if col_type.lower() == 'two':       #check for table_row_extr script for more details
+                    extractor = TableExtractor(
+                        skip_header=True,
+                        merge_two_col_rows=True,
+                        enable_subtopic_merge=True,
+                        subtopic_threshold=0.2
+                    )
+                else:
+                    extractor = TableExtractor(
+                        skip_header=True,
+                        merge_two_col_rows=False,
+                        enable_subtopic_merge=False,
+                        subtopic_threshold=0.2
+                    )
+                row_boxes = extractor.process_image(temp_path)
+                out_folder = temp_path + "_rows"
+                os.makedirs(out_folder, exist_ok=True)
+                extractor.save_extracted_cells(temp_path, row_boxes, out_folder)
+                # List all extracted cell images relative to the output folder.
+                extracted_cells = []
+                for root, dirs, files in os.walk(out_folder):
+                    for file in files:
+                        rel_path = os.path.relpath(os.path.join(root, file), self.output_folder)
+                        extracted_cells.append(rel_path)
+                # Save mapping for testing.
+                self.extracted_tables[image_id] = extracted_cells
+                snippet = ["**Extracted table cells:**"]
+                for i, row in enumerate(row_boxes):
+                    row_dir = os.path.join(out_folder, f"row_{i}")
+                    for j, _ in enumerate(row):
+                        cell_file = f"col_{j}.jpg"
+                        cell_path = os.path.join(row_dir, cell_file)
+                        relp = os.path.relpath(cell_path, self.output_folder)
+                        snippet.append(f"![Row {i} Col {j}]({relp})")
+                new_snip = "\n".join(snippet)
+                old_line = f"![HAS TO BE PROCESSED - {col_type} column table]({image_id})"
+                md_content = md_content.replace(old_line, new_snip)
+            except Exception as e:
+                logger.error(f"Error processing table image {image_id}: {e}")
+            finally:
+                if os.path.exists(temp_path):
+                    os.remove(temp_path)
+        return md_content
+
+    def post_process(self, key: str, md_content: str) -> str:
+        return asyncio.run(self.post_process_async(key, md_content))
+
+class GeminiTopicExtractor:
+    def __init__(self, api_key: str = None, num_pages: int = 14):
+        self.api_key = api_key or os.getenv("GEMINI_API_KEY", "")
+        self.num_pages = num_pages
+
+    def extract_subtopics(self, pdf_path: str) -> Dict[str, List[int]]:
+        first_pages_text = self._read_first_pages_raw(pdf_path, self.num_pages)
+        if not first_pages_text.strip():
+            logger.error("No text from first pages => cannot extract subtopics.")
+            return {}
+        prompt = f"""
+You have the first pages of a PDF specification, including a table of contents.
+Instructions:
+1. Identify the 'Contents' section listing all topics, subtopics, and their corresponding pages.
+2. Identify the major academic subtopics (common desired topic names "Paper X", "Theme X", "Content of X", "AS Unit X", "A2 Unit X", or similar headings).
+3. For each subtopic, give the range of pages [start_page, end_page] (1-based) from the table of contents.
+4. Output only valid JSON of the form:
+    {{
+    "Subtopic A": [start_page, end_page],
+    "Subtopic B": [start_page, end_page]
+    }}
+5. If you can't find any subtopics, return an empty JSON.
+Important notes:
+- The correct "end_page" must be the page number of the next topic or subtopic minus 1.
+- The final output must be valid JSON only, with no extra text or code blocks.
+Examples:
+1. Given this table of contents:
+1 Introduction – 2
+    Why choose Edexcel A Level Mathematics? - 2
+    Supporting you in planning and implementing this qualification - 3
+    Qualification at a glance - 5
+2 Subject content and assessment information – 7
+    Paper 1 and Paper 2: Pure Mathematics - 11
+    Paper 3: Statistics and Mechanics - 30
+    Assessment Objectives - 40
+3 Administration and general information – 42
+    Entries - 42
+    Access arrangements, reasonable adjustments, special consideration and malpractice - 42
+    Student recruitment and progression - 45
+Appendix 1: Formulae – 49
+Appendix 2: Notation – 53
+Appendix 3: Use of calculators – 59
+Appendix 4: Assessment Objectives – 60
+Appendix 5: The context for the development of this qualification – 62
+Appendix 6: Transferable skills – 64
+Appendix 7: Level 3 Extended Project qualification – 65
+Appendix 8: Codes – 67
+The correct output should be:
+{{
+    "Paper 1 and Paper 2: Pure Mathematics": [11, 29],
+    "Paper 3: Statistics and Mechanics": [30, 42]
+}}
+2. Given this table of contents:
+Qualification at a glance – 1
+    Assessment Objectives and weightings - 4
+Knowledge, skills and understanding – 5
+    Theme 1: Introduction to markets and market failure - 5
+    Theme 2: The UK economy – performance and policies - 11
+    Theme 3: Business behaviour and the labour market - 21
+    Theme 4: A global perspective - 29
+Assessment – 39
+    Assessment summary - 39
+    Assessment objectives - 41
+    Assessment overview - 42
+    Breakdown of assessment objectives - 42
+        Synoptic assessment - 43
+        Discount code and performance tables - 43
+        Access arrangements, reasonable adjustments and special consideration - 44
+        Malpractice - 45
+        Equality Act 2010 and Pearson equality policy - 45
+        Synoptic assessment - 46
+        Awarding and reporting - 47
+Other information – 49
+    Student recruitment -49
+    Prior learning and other requirements -49
+    Progression - 49
+Appendix 1: Transferable skills – 53
+Appendix 2: Level 3 Extended Project qualification – 55
+Appendix 3: Quantitative skills – 59
+Appendix 4: Codes – 61
+Appendix 5: Index – 63
+The correct output should be:
+{{
+    "Theme 1: Introduction to markets and market failure": [5, 10],
+    "Theme 2: The UK economy – performance and policies": [11, 20],
+    "Theme 3: Business behaviour and the labour market": [21, 28],
+    "Theme 4: A global perspective": [29, 38]
+}}
+3. You might also see sections like:
+2.1 AS Unit 1 11
+2.2 AS Unit 2 18
+2.3 A2 Unit 3 24
+2.4 A2 Unit 4 31
+In that scenario, your output might look like:
+{{
+    "2.1 AS Unit 1": [11, 17],
+    "2.2 AS Unit 2": [18, 23],
+    "2.3 A2 Unit 3": [24, 30],
+    "2.4 A2 Unit 4": [31, 35]
+}}
+4. Another example might list subtopics:
+3.1 Overarching themes 11
+3.2 A: Proof 12
+3.3 B: Algebra and functions 13
+3.4 C: Coordinate geometry in the ( x , y ) plane 14
+3.5 D: Sequences and series 15
+3.6 E: Trigonometry 16
+3.7 F: Exponentials and logarithms 17
+3.8 G: Differentiation 18
+3.9 H: Integration 19
+3.10 I: Numerical methods 20
+3.11 J: Vectors 20
+3.12 K: Statistical sampling 21
+3.13 L: Data presentation and interpretation 21
+3.14 M: Probability 22
+3.15 N: Statistical distributions 23
+3.16 O: Statistical hypothesis testing 23
+3.17 P: Quantities and units in mechanics 24
+3.18 Q: Kinematics 24
+3.19 R: Forces and Newton’s laws 24
+3.20 S: Moments 25
+3.21 Use of data in statistics 26
+Here the correct output might look like:
+{{
+    "A: Proof": [12, 12],
+    "B: Algebra and functions": [13, 13],
+    ...
+}}
+Now, extract topics from this text:
+{first_pages_text}
+"""
+        global _GEMINI_CLIENT
+        if _GEMINI_CLIENT is None:
+            _GEMINI_CLIENT = genai.Client(api_key=self.api_key)
+        client = _GEMINI_CLIENT
+        try:
+            response = client.models.generate_content(
+                model="gemini-2.0-flash",
+                contents=[prompt],
+                config=types.GenerateContentConfig(temperature=0.0)
+            )
+            if not response or not response.text:
+                logger.warning("No text from LLM => returning empty subtopics.")
+                return {}
+            raw_json = response.text.strip()
+            cleaned = raw_json.replace("```json", "").replace("```", "")
+            try:
+                data = json.loads(cleaned)
+            except Exception as json_err:
+                logger.error(f"JSON parsing error: {json_err}")
+                return {}
+            final_dict = {}
+            found_sub_dict = None
+            for k, v in data.items():
+                if isinstance(v, dict):
+                    found_sub_dict = v
+                    break
+            if found_sub_dict is not None:
+                for subk, rng in found_sub_dict.items():
+                    if isinstance(rng, list) and len(rng) == 2:
+                        final_dict[subk] = rng
+            else:
+                for subk, rng in data.items():
+                    if isinstance(rng, list) and len(rng) == 2:
+                        final_dict[subk] = rng
+            return final_dict
+        except Exception as e:
+            logger.error(f"Gemini subtopic extraction error: {e}")
+            return {}
+
+    def _read_first_pages_raw(self, pdf_path: str, num_pages: int) -> str:
+        text_parts = []
+        try:
+            if pdf_path.startswith("http://") or pdf_path.startswith("https://"):
+                response = requests.get(pdf_path)
+                if response.status_code != 200:
+                    logger.error("Failed to download PDF from %s. Status code: %d", pdf_path, response.status_code)
+                    return ""
+                pdf_bytes = response.content
+            else:
+                with open(pdf_path, "rb") as f:
+                    pdf_bytes = f.read()
+            doc = fitz.open(stream=pdf_bytes, filetype="pdf")
+            pages_to_read = min(num_pages, doc.page_count)
+            for i in range(pages_to_read):
+                raw_text = doc[i].get_text("raw")
+                text_parts.append(raw_text)
+            doc.close()
+        except Exception as e:
+            logger.error(f"Could not open PDF: {e}")
+        return "\n".join(text_parts)
+
+
+class MineruNoTextProcessor:
+    def __init__(self, output_folder: str, gemini_api_key: str):
+        self.output_folder = output_folder
+        os.makedirs(self.output_folder, exist_ok=True)
+        self.layout_model = "doclayout_yolo"
+        self.formula_enable = True
+        self.table_enable = False
+        self.language = "en"
+
+        self.subtopic_extractor = GeminiTopicExtractor(api_key=gemini_api_key, num_pages=20)
+        self.gemini_api_key = gemini_api_key or os.getenv("GEMINI_API_KEY", "")
+
+        self.use_s3 = True
+        self.s3_writer = s3Writer(
+            ak=os.getenv("S3_ACCESS_KEY"),
+            sk=os.getenv("S3_SECRET_KEY"),
+            bucket="quextro-resources",
+            endpoint_url=os.getenv("S3_ENDPOINT")
+        )
+
+    def cleanup_gpu(self):
+        try:
+            gc.collect()
+            torch.cuda.empty_cache()
+            logger.info("GPU memory cleaned up.")
+        except Exception as e:
+            logger.error(f"Error during GPU cleanup: {e}")
+
+    def process(self, pdf_path: str) -> Dict[str, Any]:
+        logger.info(f"Processing PDF: {pdf_path}")
+        try:
+            # 1) Possibly call subtopic_extractor on first pages to find subtopics in the PDF as a whole
+            subtopics = self.subtopic_extractor.extract_subtopics(pdf_path)
+            logger.info(f"Gemini returned subtopics: {subtopics}")
+
+            if pdf_path.startswith("http://") or pdf_path.startswith("https://"):
+                response = requests.get(pdf_path)
+                if response.status_code != 200:
+                    logger.error("Failed to download PDF from %s. Status code: %d", pdf_path, response.status_code)
+                    raise Exception(f"Failed to download PDF: {pdf_path}")
+                pdf_bytes = response.content
+                logger.info("Downloaded %d bytes for pdf_url='%s'", len(pdf_bytes), pdf_path)
+            else:
+                with open(pdf_path, "rb") as f:
+                    pdf_bytes = f.read()
+                logger.info("Loaded %d bytes from local file '%s'", len(pdf_bytes), pdf_path)
+
+            doc = fitz.open(stream=pdf_bytes, filetype="pdf")
+            total_pages = doc.page_count
+            doc.close()
+
+            # 3) Decide which pages to process
+            final_pages = set()
+            if not subtopics:
+                # fallback
+                final_pages = set(range(total_pages))
+            else:
+                offset_candidates = []
+                for subname, rng in subtopics.items():
+                    start_p, _ = rng
+                    occs = find_all_occurrences(pdf_bytes, subname)
+                    for p in occs:
+                        candidate = p - (start_p - 1)
+                        if candidate > 0:
+                            offset_candidates.append(candidate)
+                if offset_candidates:
+                    try:
+                        from statistics import mode
+                        global_offset = mode(offset_candidates)
+                    except:
+                        from statistics import median
+                        global_offset = int(median(offset_candidates))
+                else:
+                    global_offset = 0
+
+                logger.info(f"Computed global offset: {global_offset}")
+                for subname, rng in subtopics.items():
+                    if not (isinstance(rng, list) and len(rng) == 2):
+                        continue
+                    start_p, end_p = rng
+                    if start_p > end_p:
+                        continue
+                    s0 = (start_p - 1) + global_offset
+                    e0 = (end_p - 1) + global_offset
+                    for pp in range(s0, e0 + 1):
+                        final_pages.add(pp)
+
+            if not final_pages:
+                final_pages = set(range(total_pages))
+
+            logger.info(f"Processing pages (0-based): {sorted(final_pages)}")
+            subset_pdf_bytes = create_subset_pdf(pdf_bytes, sorted(final_pages))
+
+            # 4) Analyze and produce markdown
+            dataset = PymuDocDataset(subset_pdf_bytes)
+            inference = doc_analyze(
+                dataset,
+                ocr=True,
+                lang=self.language,
+                layout_model=self.layout_model,
+                formula_enable=self.formula_enable,
+                table_enable=self.table_enable
+            )
+            writer = S3ImageWriter(self.s3_writer, "/topic-extraction", self.gemini_api_key)
+            md_prefix = "/topic-extraction/"
+            pipe_result = inference.pipe_ocr_mode(writer, lang=self.language)
+            md_content = pipe_result.get_markdown(md_prefix)
+            final_markdown = writer.post_process(md_prefix, md_content)
+
+            subtopic_list = list(writer.extracted_subtopics.values())
+
+            out_path = os.path.join(self.output_folder, "final_subtopics.json")
+            with open(out_path, "w", encoding="utf-8") as f:
+                json.dump(subtopic_list, f, indent=2)
+            logger.info(f"Final subtopics JSON saved locally at {out_path}")
+
+            return {
+                "final_markdown": final_markdown,
+                "subtopics_extracted": subtopic_list
+            }
+        finally:
+            self.cleanup_gpu()
+
+if __name__ == "__main__":
+    input_pdf = "/home/user/app/input_output/a-level-pearson-mathematics-specification.pdf"
+    output_dir = "/home/user/app/we/we_ars"
+    gemini_key = os.getenv("GEMINI_API_KEY", "AIzaSyDtoakpXa2pjJwcQB6TJ5QaXHNSA5JxcrU")
+    try:
+        processor = MineruNoTextProcessor(output_folder=output_dir, gemini_api_key=gemini_key)
+        result = processor.process(input_pdf)
+        logger.info("Processing completed successfully.")
+        # The result includes final_markdown and subtopics_extracted
+    except Exception as e:
+        logger.error(f"Processing failed: {e}")

topic_extraction.py

@@ -721,7 +721,7 @@ class MineruNoTextProcessor:
             self.cleanup_gpu()
 
 if __name__ == "__main__":
-    input_pdf = "/home/user/app/input_output/wjec-gce-maths-spec-from-2017-e.pdf"
+    input_pdf = "/home/user/app/input_output/a-level-pearson-mathematics-specification.pdf"
     output_dir = "/home/user/app/wje"
     gemini_key = os.getenv("GEMINI_API_KEY", "AIzaSyDtoakpXa2pjJwcQB6TJ5QaXHNSA5JxcrU")
     try:

topic_extraction_ars.log

@@ -0,0 +1,460 @@
+2025-03-03 15:45:38,171 [INFO] __main__ - Processing PDF: /home/user/app/input_output/a-level-pearson-mathematics-specification.pdf
+2025-03-03 15:45:38,974 [INFO] __main__ - Gemini returned subtopics: {'Paper 1 and Paper 2: Pure Mathematics': [11, 29], 'Paper 3: Statistics and Mechanics': [30, 40]}
+2025-03-03 15:45:38,975 [INFO] __main__ - Loaded 1135473 bytes from local file '/home/user/app/input_output/a-level-pearson-mathematics-specification.pdf'
+2025-03-03 15:45:39,261 [INFO] __main__ - Computed global offset: 4
+2025-03-03 15:45:39,261 [INFO] __main__ - Processing pages (0-based): [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43]
+2025-03-03 15:46:34,912 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_1.jpg
+2025-03-03 15:46:36,964 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_2.jpg
+2025-03-03 15:46:37,539 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_3.jpg
+2025-03-03 15:46:38,161 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_4.jpg
+2025-03-03 15:46:38,703 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_5.jpg
+2025-03-03 15:46:39,330 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_6.jpg
+2025-03-03 15:46:39,805 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_7.jpg
+2025-03-03 15:46:40,281 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_8.jpg
+2025-03-03 15:46:40,751 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_9.jpg
+2025-03-03 15:46:41,336 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_10.jpg
+2025-03-03 15:46:41,773 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_11.jpg
+2025-03-03 15:46:42,431 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_12.jpg
+2025-03-03 15:46:42,903 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_13.jpg
+2025-03-03 15:46:43,490 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_14.jpg
+2025-03-03 15:46:43,962 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_15.jpg
+2025-03-03 15:46:44,566 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_16.jpg
+2025-03-03 15:46:45,155 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_17.jpg
+2025-03-03 15:46:45,448 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_18.jpg
+2025-03-03 15:46:45,896 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_19.jpg
+2025-03-03 15:46:46,485 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_20.jpg
+2025-03-03 15:46:47,081 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_21.jpg
+2025-03-03 15:46:47,652 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_22.jpg
+2025-03-03 15:46:48,109 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_23.jpg
+2025-03-03 15:46:48,593 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_24.jpg
+2025-03-03 15:46:49,101 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_25.jpg
+2025-03-03 15:46:49,644 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_26.jpg
+2025-03-03 15:46:50,274 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_27.jpg
+2025-03-03 15:46:50,891 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_28.jpg
+2025-03-03 15:46:51,327 [INFO] __main__ - Classifying images to detect tables.
+2025-03-03 15:46:55,176 [INFO] __main__ - Processing table image: /topic-extraction/img_1.jpg, columns=three
+2025-03-03 15:46:58,654 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_1.jpg_r0_c0.jpg
+2025-03-03 15:46:58,952 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_1.jpg_r0_c1.jpg
+2025-03-03 15:46:59,179 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_1.jpg_r1_c0.jpg
+2025-03-03 15:46:59,433 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_1.jpg_r1_c1.jpg
+2025-03-03 15:46:59,434 [INFO] __main__ - Processing table image: /topic-extraction/img_2.jpg, columns=three
+2025-03-03 15:47:02,885 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_2.jpg_r0_c0.jpg
+2025-03-03 15:47:03,187 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_2.jpg_r0_c1.jpg
+2025-03-03 15:47:03,419 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_2.jpg_r1_c0.jpg
+2025-03-03 15:47:03,657 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_2.jpg_r2_c0.jpg
+2025-03-03 15:47:03,872 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_2.jpg_r3_c0.jpg
+2025-03-03 15:47:03,873 [INFO] __main__ - Processing table image: /topic-extraction/img_3.jpg, columns=three
+2025-03-03 15:47:07,421 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_3.jpg_r0_c0.jpg
+2025-03-03 15:47:07,712 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_3.jpg_r0_c1.jpg
+2025-03-03 15:47:07,918 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_3.jpg_r1_c0.jpg
+2025-03-03 15:47:07,918 [INFO] __main__ - Processing table image: /topic-extraction/img_4.jpg, columns=three
+2025-03-03 15:47:11,395 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_4.jpg_r0_c0.jpg
+2025-03-03 15:47:11,689 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_4.jpg_r0_c1.jpg
+2025-03-03 15:47:11,904 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_4.jpg_r1_c0.jpg
+2025-03-03 15:47:12,137 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_4.jpg_r1_c1.jpg
+2025-03-03 15:47:12,138 [INFO] __main__ - Processing table image: /topic-extraction/img_5.jpg, columns=three
+2025-03-03 15:47:15,853 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_5.jpg_r0_c0.jpg
+2025-03-03 15:47:16,176 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_5.jpg_r0_c1.jpg
+2025-03-03 15:47:16,379 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_5.jpg_r1_c0.jpg
+2025-03-03 15:47:16,611 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_5.jpg_r1_c1.jpg
+2025-03-03 15:47:16,850 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_5.jpg_r2_c0.jpg
+2025-03-03 15:47:16,850 [INFO] __main__ - Processing table image: /topic-extraction/img_6.jpg, columns=three
+2025-03-03 15:47:20,810 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_6.jpg_r0_c0.jpg
+2025-03-03 15:47:21,101 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_6.jpg_r0_c1.jpg
+2025-03-03 15:47:21,322 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_6.jpg_r1_c0.jpg
+2025-03-03 15:47:21,549 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_6.jpg_r1_c1.jpg
+2025-03-03 15:47:21,549 [INFO] __main__ - Processing table image: /topic-extraction/img_7.jpg, columns=three
+2025-03-03 15:47:25,075 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_7.jpg_r0_c0.jpg
+2025-03-03 15:47:25,405 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_7.jpg_r0_c1.jpg
+2025-03-03 15:47:25,599 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_7.jpg_r1_c0.jpg
+2025-03-03 15:47:25,823 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_7.jpg_r2_c0.jpg
+2025-03-03 15:47:26,054 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_7.jpg_r2_c1.jpg
+2025-03-03 15:47:26,054 [INFO] __main__ - Processing table image: /topic-extraction/img_8.jpg, columns=three
+2025-03-03 15:47:29,662 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r0_c0.jpg
+2025-03-03 15:47:29,944 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r0_c1.jpg
+2025-03-03 15:47:30,160 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r0_c2.jpg
+2025-03-03 15:47:30,354 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r1_c0.jpg
+2025-03-03 15:47:30,586 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r1_c1.jpg
+2025-03-03 15:47:30,801 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r1_c2.jpg
+2025-03-03 15:47:31,028 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r2_c0.jpg
+2025-03-03 15:47:31,232 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r2_c1.jpg
+2025-03-03 15:47:31,461 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r3_c0.jpg
+2025-03-03 15:47:31,654 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r3_c1.jpg
+2025-03-03 15:47:31,912 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r4_c0.jpg
+2025-03-03 15:47:32,139 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r4_c1.jpg
+2025-03-03 15:47:32,345 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r5_c0.jpg
+2025-03-03 15:47:32,586 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r5_c1.jpg
+2025-03-03 15:47:32,587 [INFO] __main__ - Processing table image: /topic-extraction/img_9.jpg, columns=three
+2025-03-03 15:47:36,350 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_9.jpg_r0_c0.jpg
+2025-03-03 15:47:36,676 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_9.jpg_r0_c1.jpg
+2025-03-03 15:47:36,893 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_9.jpg_r0_c2.jpg
+2025-03-03 15:47:37,141 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_9.jpg_r1_c0.jpg
+2025-03-03 15:47:37,374 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_9.jpg_r1_c1.jpg
+2025-03-03 15:47:37,565 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_9.jpg_r2_c0.jpg
+2025-03-03 15:47:37,760 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_9.jpg_r2_c1.jpg
+2025-03-03 15:47:38,012 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_9.jpg_r3_c0.jpg
+2025-03-03 15:47:38,226 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_9.jpg_r3_c1.jpg
+2025-03-03 15:47:38,226 [INFO] __main__ - Processing table image: /topic-extraction/img_10.jpg, columns=three
+2025-03-03 15:47:42,402 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_10.jpg_r0_c0.jpg
+2025-03-03 15:47:42,675 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_10.jpg_r0_c1.jpg
+2025-03-03 15:47:42,917 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_10.jpg_r1_c0.jpg
+2025-03-03 15:47:43,133 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_10.jpg_r2_c0.jpg
+2025-03-03 15:47:43,355 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_10.jpg_r3_c0.jpg
+2025-03-03 15:47:43,355 [INFO] __main__ - Processing table image: /topic-extraction/img_11.jpg, columns=two
+2025-03-03 15:47:48,037 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_11.jpg_r0_c0.jpg
+2025-03-03 15:47:48,332 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_11.jpg_r1_c0.jpg
+2025-03-03 15:47:48,540 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_11.jpg_r2_c0.jpg
+2025-03-03 15:47:48,786 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_11.jpg_r3_c0.jpg
+2025-03-03 15:47:49,037 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_11.jpg_r4_c0.jpg
+2025-03-03 15:47:49,264 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_11.jpg_r5_c0.jpg
+2025-03-03 15:47:49,264 [INFO] __main__ - Processing table image: /topic-extraction/img_12.jpg, columns=three
+2025-03-03 15:47:53,266 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_12.jpg_r0_c0.jpg
+2025-03-03 15:47:53,598 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_12.jpg_r0_c1.jpg
+2025-03-03 15:47:53,819 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_12.jpg_r1_c0.jpg
+2025-03-03 15:47:54,034 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_12.jpg_r1_c1.jpg
+2025-03-03 15:47:54,250 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_12.jpg_r2_c0.jpg
+2025-03-03 15:47:54,474 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_12.jpg_r2_c1.jpg
+2025-03-03 15:47:54,474 [INFO] __main__ - Processing table image: /topic-extraction/img_13.jpg, columns=three
+2025-03-03 15:47:57,779 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_13.jpg_r0_c0.jpg
+2025-03-03 15:47:58,103 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_13.jpg_r0_c1.jpg
+2025-03-03 15:47:58,326 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_13.jpg_r1_c0.jpg
+2025-03-03 15:47:58,545 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_13.jpg_r1_c1.jpg
+2025-03-03 15:47:58,738 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_13.jpg_r2_c0.jpg
+2025-03-03 15:47:58,994 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_13.jpg_r3_c0.jpg
+2025-03-03 15:47:58,994 [INFO] __main__ - Processing table image: /topic-extraction/img_14.jpg, columns=three
+2025-03-03 15:48:03,866 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_14.jpg_r0_c0.jpg
+2025-03-03 15:48:04,164 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_14.jpg_r0_c1.jpg
+2025-03-03 15:48:04,382 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_14.jpg_r1_c0.jpg
+2025-03-03 15:48:04,605 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_14.jpg_r1_c1.jpg
+2025-03-03 15:48:04,799 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_14.jpg_r2_c0.jpg
+2025-03-03 15:48:05,032 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_14.jpg_r3_c0.jpg
+2025-03-03 15:48:05,247 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_14.jpg_r4_c0.jpg
+2025-03-03 15:48:05,493 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_14.jpg_r4_c1.jpg
+2025-03-03 15:48:05,710 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_14.jpg_r5_c0.jpg
+2025-03-03 15:48:05,711 [INFO] __main__ - Processing table image: /topic-extraction/img_15.jpg, columns=three
+2025-03-03 15:48:09,411 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_15.jpg_r0_c0.jpg
+2025-03-03 15:48:09,698 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_15.jpg_r0_c1.jpg
+2025-03-03 15:48:09,923 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_15.jpg_r1_c0.jpg
+2025-03-03 15:48:10,113 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_15.jpg_r1_c1.jpg
+2025-03-03 15:48:10,361 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_15.jpg_r2_c0.jpg
+2025-03-03 15:48:10,587 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_15.jpg_r3_c0.jpg
+2025-03-03 15:48:10,799 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_15.jpg_r4_c0.jpg
+2025-03-03 15:48:10,800 [INFO] __main__ - Processing table image: /topic-extraction/img_16.jpg, columns=three
+2025-03-03 15:48:14,668 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_16.jpg_r0_c0.jpg
+2025-03-03 15:48:14,969 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_16.jpg_r0_c1.jpg
+2025-03-03 15:48:15,207 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_16.jpg_r1_c0.jpg
+2025-03-03 15:48:15,414 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_16.jpg_r1_c1.jpg
+2025-03-03 15:48:15,634 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_16.jpg_r2_c0.jpg
+2025-03-03 15:48:15,893 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_16.jpg_r3_c0.jpg
+2025-03-03 15:48:16,111 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_16.jpg_r3_c1.jpg
+2025-03-03 15:48:16,343 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_16.jpg_r4_c0.jpg
+2025-03-03 15:48:17,176 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_16.jpg_r5_c0.jpg
+2025-03-03 15:48:17,176 [INFO] __main__ - Processing table image: /topic-extraction/img_17.jpg, columns=three
+2025-03-03 15:48:20,954 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_17.jpg_r0_c0.jpg
+2025-03-03 15:48:21,213 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_17.jpg_r0_c1.jpg
+2025-03-03 15:48:21,423 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_17.jpg_r1_c0.jpg
+2025-03-03 15:48:21,634 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_17.jpg_r2_c0.jpg
+2025-03-03 15:48:21,832 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_17.jpg_r2_c1.jpg
+2025-03-03 15:48:22,056 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_17.jpg_r3_c0.jpg
+2025-03-03 15:48:22,261 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_17.jpg_r4_c0.jpg
+2025-03-03 15:48:22,481 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_17.jpg_r5_c0.jpg
+2025-03-03 15:48:22,482 [INFO] __main__ - Processing table image: /topic-extraction/img_18.jpg, columns=three
+2025-03-03 15:48:23,665 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_18.jpg_r0_c0.jpg
+2025-03-03 15:48:23,852 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_18.jpg_r0_c1.jpg
+2025-03-03 15:48:24,035 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_18.jpg_r1_c0.jpg
+2025-03-03 15:48:24,219 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_18.jpg_r1_c1.jpg
+2025-03-03 15:48:24,219 [INFO] __main__ - Processing table image: /topic-extraction/img_19.jpg, columns=three
+2025-03-03 15:48:27,206 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_19.jpg_r0_c0.jpg
+2025-03-03 15:48:27,482 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_19.jpg_r0_c1.jpg
+2025-03-03 15:48:27,693 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_19.jpg_r1_c0.jpg
+2025-03-03 15:48:27,924 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_19.jpg_r1_c1.jpg
+2025-03-03 15:48:28,131 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_19.jpg_r2_c0.jpg
+2025-03-03 15:48:28,337 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_19.jpg_r2_c1.jpg
+2025-03-03 15:48:28,338 [INFO] __main__ - Processing table image: /topic-extraction/img_20.jpg, columns=three
+2025-03-03 15:48:32,733 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_20.jpg_r0_c0.jpg
+2025-03-03 15:48:32,995 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_20.jpg_r0_c1.jpg
+2025-03-03 15:48:33,221 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_20.jpg_r1_c0.jpg
+2025-03-03 15:48:33,449 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_20.jpg_r1_c1.jpg
+2025-03-03 15:48:33,449 [INFO] __main__ - Processing table image: /topic-extraction/img_21.jpg, columns=three
+2025-03-03 15:48:37,495 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_21.jpg_r0_c0.jpg
+2025-03-03 15:48:37,802 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_21.jpg_r0_c1.jpg
+2025-03-03 15:48:38,060 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_21.jpg_r1_c0.jpg
+2025-03-03 15:48:38,267 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_21.jpg_r1_c1.jpg
+2025-03-03 15:48:38,267 [INFO] __main__ - Processing table image: /topic-extraction/img_22.jpg, columns=three
+2025-03-03 15:48:42,539 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_22.jpg_r0_c0.jpg
+2025-03-03 15:48:42,847 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_22.jpg_r0_c1.jpg
+2025-03-03 15:48:43,064 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_22.jpg_r1_c0.jpg
+2025-03-03 15:48:43,280 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_22.jpg_r1_c1.jpg
+2025-03-03 15:48:43,487 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_22.jpg_r2_c0.jpg
+2025-03-03 15:48:43,716 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_22.jpg_r2_c1.jpg
+2025-03-03 15:48:43,918 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_22.jpg_r3_c0.jpg
+2025-03-03 15:48:43,918 [INFO] __main__ - Processing table image: /topic-extraction/img_23.jpg, columns=three
+2025-03-03 15:48:47,600 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_23.jpg_r0_c0.jpg
+2025-03-03 15:48:47,900 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_23.jpg_r0_c1.jpg
+2025-03-03 15:48:48,125 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_23.jpg_r1_c0.jpg
+2025-03-03 15:48:48,343 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_23.jpg_r1_c1.jpg
+2025-03-03 15:48:48,343 [INFO] __main__ - Processing table image: /topic-extraction/img_24.jpg, columns=three
+2025-03-03 15:48:52,065 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_24.jpg_r0_c0.jpg
+2025-03-03 15:48:52,376 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_24.jpg_r0_c1.jpg
+2025-03-03 15:48:52,614 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_24.jpg_r1_c0.jpg
+2025-03-03 15:48:52,870 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_24.jpg_r1_c1.jpg
+2025-03-03 15:48:53,066 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_24.jpg_r2_c0.jpg
+2025-03-03 15:48:53,066 [INFO] __main__ - Processing table image: /topic-extraction/img_25.jpg, columns=two
+2025-03-03 15:48:56,548 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_25.jpg_r0_c0.jpg
+2025-03-03 15:48:56,863 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_25.jpg_r1_c0.jpg
+2025-03-03 15:48:57,087 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_25.jpg_r2_c0.jpg
+2025-03-03 15:48:57,301 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_25.jpg_r3_c0.jpg
+2025-03-03 15:48:57,526 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_25.jpg_r4_c0.jpg
+2025-03-03 15:48:57,759 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_25.jpg_r5_c0.jpg
+2025-03-03 15:48:57,759 [INFO] __main__ - Processing table image: /topic-extraction/img_26.jpg, columns=three
+2025-03-03 15:49:01,116 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_26.jpg_r0_c0.jpg
+2025-03-03 15:49:01,407 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_26.jpg_r0_c1.jpg
+2025-03-03 15:49:01,618 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_26.jpg_r1_c0.jpg
+2025-03-03 15:49:01,847 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_26.jpg_r2_c0.jpg
+2025-03-03 15:49:01,847 [INFO] __main__ - Processing table image: /topic-extraction/img_27.jpg, columns=three
+2025-03-03 15:49:04,977 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_27.jpg_r0_c0.jpg
+2025-03-03 15:49:05,258 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_27.jpg_r0_c1.jpg
+2025-03-03 15:49:05,498 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_27.jpg_r1_c0.jpg
+2025-03-03 15:49:05,712 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_27.jpg_r1_c1.jpg
+2025-03-03 15:49:05,934 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_27.jpg_r2_c0.jpg
+2025-03-03 15:49:06,162 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_27.jpg_r3_c0.jpg
+2025-03-03 15:49:06,385 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_27.jpg_r4_c0.jpg
+2025-03-03 15:49:06,612 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_27.jpg_r4_c1.jpg
+2025-03-03 15:49:06,613 [INFO] __main__ - Processing table image: /topic-extraction/img_28.jpg, columns=two
+2025-03-03 15:49:10,036 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_28.jpg_r0_c0.jpg
+2025-03-03 15:49:10,328 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_28.jpg_r1_c0.jpg
+2025-03-03 15:49:10,548 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_28.jpg_r2_c0.jpg
+2025-03-03 15:49:10,777 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_28.jpg_r3_c0.jpg
+2025-03-03 15:49:10,780 [INFO] __main__ - Final subtopics JSON saved locally at /home/user/app/we/we_ars/final_subtopics.json
+2025-03-03 15:49:11,098 [INFO] __main__ - GPU memory cleaned up.
+2025-03-03 15:49:11,106 [INFO] __main__ - Processing completed successfully.
+2025-03-03 15:53:27,401 [INFO] __main__ - Processing PDF: /home/user/app/input_output/a-level-pearson-mathematics-specification.pdf
+2025-03-03 15:53:28,230 [INFO] __main__ - Gemini returned subtopics: {'Paper 1 and Paper 2: Pure Mathematics': [11, 29], 'Paper 3: Statistics and Mechanics': [30, 40]}
+2025-03-03 15:53:28,231 [INFO] __main__ - Loaded 1135473 bytes from local file '/home/user/app/input_output/a-level-pearson-mathematics-specification.pdf'
+2025-03-03 15:53:28,557 [INFO] __main__ - Computed global offset: 4
+2025-03-03 15:53:28,557 [INFO] __main__ - Processing pages (0-based): [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43]
+2025-03-03 15:54:23,423 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_1.jpg
+2025-03-03 15:54:25,210 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_2.jpg
+2025-03-03 15:54:25,742 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_3.jpg
+2025-03-03 15:54:26,250 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_4.jpg
+2025-03-03 15:54:26,794 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_5.jpg
+2025-03-03 15:54:27,347 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_6.jpg
+2025-03-03 15:54:27,803 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_7.jpg
+2025-03-03 15:54:28,391 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_8.jpg
+2025-03-03 15:54:28,891 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_9.jpg
+2025-03-03 15:54:29,437 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_10.jpg
+2025-03-03 15:54:29,870 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_11.jpg
+2025-03-03 15:54:30,421 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_12.jpg
+2025-03-03 15:54:30,852 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_13.jpg
+2025-03-03 15:54:31,438 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_14.jpg
+2025-03-03 15:54:32,029 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_15.jpg
+2025-03-03 15:54:32,600 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_16.jpg
+2025-03-03 15:54:33,157 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_17.jpg
+2025-03-03 15:54:33,444 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_18.jpg
+2025-03-03 15:54:33,920 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_19.jpg
+2025-03-03 15:54:34,554 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_20.jpg
+2025-03-03 15:54:35,147 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_21.jpg
+2025-03-03 15:54:35,680 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_22.jpg
+2025-03-03 15:54:36,094 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_23.jpg
+2025-03-03 15:54:36,554 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_24.jpg
+2025-03-03 15:54:37,089 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_25.jpg
+2025-03-03 15:54:37,502 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_26.jpg
+2025-03-03 15:54:38,008 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_27.jpg
+2025-03-03 15:54:38,585 [INFO] __main__ - Uploaded to S3: /topic-extraction/img_28.jpg
+2025-03-03 15:54:39,068 [INFO] __main__ - Classifying images to detect tables.
+2025-03-03 15:54:42,753 [INFO] __main__ - Processing table image: /topic-extraction/img_1.jpg, columns=three
+2025-03-03 15:54:46,419 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_1.jpg_r0_c0.jpg
+2025-03-03 15:54:46,711 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_1.jpg_r0_c1.jpg
+2025-03-03 15:54:46,896 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_1.jpg_r1_c0.jpg
+2025-03-03 15:54:47,110 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_1.jpg_r1_c1.jpg
+2025-03-03 15:54:47,110 [INFO] __main__ - Processing table image: /topic-extraction/img_2.jpg, columns=three
+2025-03-03 15:54:50,464 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_2.jpg_r0_c0.jpg
+2025-03-03 15:54:50,784 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_2.jpg_r0_c1.jpg
+2025-03-03 15:54:50,976 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_2.jpg_r1_c0.jpg
+2025-03-03 15:54:51,228 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_2.jpg_r2_c0.jpg
+2025-03-03 15:54:51,462 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_2.jpg_r3_c0.jpg
+2025-03-03 15:54:51,463 [INFO] __main__ - Processing table image: /topic-extraction/img_3.jpg, columns=three
+2025-03-03 15:54:55,079 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_3.jpg_r0_c0.jpg
+2025-03-03 15:54:55,364 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_3.jpg_r0_c1.jpg
+2025-03-03 15:54:55,570 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_3.jpg_r1_c0.jpg
+2025-03-03 15:54:55,571 [INFO] __main__ - Processing table image: /topic-extraction/img_4.jpg, columns=three
+2025-03-03 15:54:58,838 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_4.jpg_r0_c0.jpg
+2025-03-03 15:54:59,144 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_4.jpg_r0_c1.jpg
+2025-03-03 15:54:59,326 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_4.jpg_r1_c0.jpg
+2025-03-03 15:54:59,577 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_4.jpg_r1_c1.jpg
+2025-03-03 15:54:59,578 [INFO] __main__ - Processing table image: /topic-extraction/img_5.jpg, columns=three
+2025-03-03 15:55:03,518 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_5.jpg_r0_c0.jpg
+2025-03-03 15:55:03,801 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_5.jpg_r0_c1.jpg
+2025-03-03 15:55:03,983 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_5.jpg_r1_c0.jpg
+2025-03-03 15:55:04,202 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_5.jpg_r1_c1.jpg
+2025-03-03 15:55:04,417 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_5.jpg_r2_c0.jpg
+2025-03-03 15:55:04,417 [INFO] __main__ - Processing table image: /topic-extraction/img_6.jpg, columns=three
+2025-03-03 15:55:08,109 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_6.jpg_r0_c0.jpg
+2025-03-03 15:55:08,423 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_6.jpg_r0_c1.jpg
+2025-03-03 15:55:08,629 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_6.jpg_r1_c0.jpg
+2025-03-03 15:55:08,816 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_6.jpg_r1_c1.jpg
+2025-03-03 15:55:08,816 [INFO] __main__ - Processing table image: /topic-extraction/img_7.jpg, columns=three
+2025-03-03 15:55:12,344 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_7.jpg_r0_c0.jpg
+2025-03-03 15:55:12,644 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_7.jpg_r0_c1.jpg
+2025-03-03 15:55:12,867 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_7.jpg_r1_c0.jpg
+2025-03-03 15:55:13,114 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_7.jpg_r2_c0.jpg
+2025-03-03 15:55:13,343 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_7.jpg_r2_c1.jpg
+2025-03-03 15:55:13,344 [INFO] __main__ - Processing table image: /topic-extraction/img_8.jpg, columns=three
+2025-03-03 15:55:16,823 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r0_c0.jpg
+2025-03-03 15:55:17,140 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r0_c1.jpg
+2025-03-03 15:55:17,422 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r0_c2.jpg
+2025-03-03 15:55:17,706 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r1_c0.jpg
+2025-03-03 15:55:18,019 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r1_c1.jpg
+2025-03-03 15:55:18,320 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r1_c2.jpg
+2025-03-03 15:55:18,619 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r2_c0.jpg
+2025-03-03 15:55:18,911 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r2_c1.jpg
+2025-03-03 15:55:19,208 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r3_c0.jpg
+2025-03-03 15:55:19,491 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r3_c1.jpg
+2025-03-03 15:55:19,806 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r4_c0.jpg
+2025-03-03 15:55:20,093 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r4_c1.jpg
+2025-03-03 15:55:20,406 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r5_c0.jpg
+2025-03-03 15:55:20,689 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_8.jpg_r5_c1.jpg
+2025-03-03 15:55:20,690 [INFO] __main__ - Processing table image: /topic-extraction/img_9.jpg, columns=three
+2025-03-03 15:55:24,558 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_9.jpg_r0_c0.jpg
+2025-03-03 15:55:24,859 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_9.jpg_r0_c1.jpg
+2025-03-03 15:55:25,142 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_9.jpg_r0_c2.jpg
+2025-03-03 15:55:25,422 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_9.jpg_r1_c0.jpg
+2025-03-03 15:55:25,738 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_9.jpg_r1_c1.jpg
+2025-03-03 15:55:26,031 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_9.jpg_r2_c0.jpg
+2025-03-03 15:55:26,335 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_9.jpg_r2_c1.jpg
+2025-03-03 15:55:26,616 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_9.jpg_r3_c0.jpg
+2025-03-03 15:55:26,908 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_9.jpg_r3_c1.jpg
+2025-03-03 15:55:26,909 [INFO] __main__ - Processing table image: /topic-extraction/img_10.jpg, columns=three
+2025-03-03 15:55:30,379 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_10.jpg_r0_c0.jpg
+2025-03-03 15:55:30,667 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_10.jpg_r0_c1.jpg
+2025-03-03 15:55:30,961 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_10.jpg_r1_c0.jpg
+2025-03-03 15:55:31,248 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_10.jpg_r2_c0.jpg
+2025-03-03 15:55:31,547 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_10.jpg_r3_c0.jpg
+2025-03-03 15:55:31,549 [INFO] __main__ - Processing table image: /topic-extraction/img_11.jpg, columns=two
+2025-03-03 15:55:34,706 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_11.jpg_r0_c0.jpg
+2025-03-03 15:55:34,994 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_11.jpg_r1_c0.jpg
+2025-03-03 15:55:35,254 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_11.jpg_r2_c0.jpg
+2025-03-03 15:55:35,558 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_11.jpg_r3_c0.jpg
+2025-03-03 15:55:35,852 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_11.jpg_r4_c0.jpg
+2025-03-03 15:55:36,137 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_11.jpg_r5_c0.jpg
+2025-03-03 15:55:36,137 [INFO] __main__ - Processing table image: /topic-extraction/img_12.jpg, columns=three
+2025-03-03 15:55:39,497 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_12.jpg_r0_c0.jpg
+2025-03-03 15:55:39,757 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_12.jpg_r0_c1.jpg
+2025-03-03 15:55:40,062 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_12.jpg_r1_c0.jpg
+2025-03-03 15:55:40,345 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_12.jpg_r1_c1.jpg
+2025-03-03 15:55:40,666 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_12.jpg_r2_c0.jpg
+2025-03-03 15:55:40,976 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_12.jpg_r2_c1.jpg
+2025-03-03 15:55:40,977 [INFO] __main__ - Processing table image: /topic-extraction/img_13.jpg, columns=three
+2025-03-03 15:55:44,159 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_13.jpg_r0_c0.jpg
+2025-03-03 15:55:44,436 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_13.jpg_r0_c1.jpg
+2025-03-03 15:55:44,643 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_13.jpg_r1_c0.jpg
+2025-03-03 15:55:44,853 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_13.jpg_r1_c1.jpg
+2025-03-03 15:55:45,041 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_13.jpg_r2_c0.jpg
+2025-03-03 15:55:45,254 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_13.jpg_r3_c0.jpg
+2025-03-03 15:55:45,255 [INFO] __main__ - Processing table image: /topic-extraction/img_14.jpg, columns=three
+2025-03-03 15:55:49,508 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_14.jpg_r0_c0.jpg
+2025-03-03 15:55:49,786 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_14.jpg_r0_c1.jpg
+2025-03-03 15:55:50,075 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_14.jpg_r1_c0.jpg
+2025-03-03 15:55:50,355 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_14.jpg_r1_c1.jpg
+2025-03-03 15:55:50,647 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_14.jpg_r2_c0.jpg
+2025-03-03 15:55:50,978 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_14.jpg_r3_c0.jpg
+2025-03-03 15:55:51,295 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_14.jpg_r4_c0.jpg
+2025-03-03 15:55:51,582 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_14.jpg_r4_c1.jpg
+2025-03-03 15:55:51,855 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_14.jpg_r5_c0.jpg
+2025-03-03 15:55:51,856 [INFO] __main__ - Processing table image: /topic-extraction/img_15.jpg, columns=three
+2025-03-03 15:55:55,882 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_15.jpg_r0_c0.jpg
+2025-03-03 15:55:56,182 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_15.jpg_r0_c1.jpg
+2025-03-03 15:55:56,463 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_15.jpg_r1_c0.jpg
+2025-03-03 15:55:56,727 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_15.jpg_r1_c1.jpg
+2025-03-03 15:55:57,005 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_15.jpg_r2_c0.jpg
+2025-03-03 15:55:57,301 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_15.jpg_r3_c0.jpg
+2025-03-03 15:55:57,584 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_15.jpg_r4_c0.jpg
+2025-03-03 15:55:57,584 [INFO] __main__ - Processing table image: /topic-extraction/img_16.jpg, columns=three
+2025-03-03 15:56:01,615 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_16.jpg_r0_c0.jpg
+2025-03-03 15:56:01,906 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_16.jpg_r0_c1.jpg
+2025-03-03 15:56:02,222 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_16.jpg_r1_c0.jpg
+2025-03-03 15:56:02,513 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_16.jpg_r1_c1.jpg
+2025-03-03 15:56:02,801 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_16.jpg_r2_c0.jpg
+2025-03-03 15:56:03,083 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_16.jpg_r3_c0.jpg
+2025-03-03 15:56:03,393 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_16.jpg_r3_c1.jpg
+2025-03-03 15:56:03,676 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_16.jpg_r4_c0.jpg
+2025-03-03 15:56:04,667 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_16.jpg_r5_c0.jpg
+2025-03-03 15:56:04,667 [INFO] __main__ - Processing table image: /topic-extraction/img_17.jpg, columns=three
+2025-03-03 15:56:09,007 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_17.jpg_r0_c0.jpg
+2025-03-03 15:56:09,286 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_17.jpg_r0_c1.jpg
+2025-03-03 15:56:09,520 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_17.jpg_r1_c0.jpg
+2025-03-03 15:56:09,740 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_17.jpg_r2_c0.jpg
+2025-03-03 15:56:09,947 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_17.jpg_r2_c1.jpg
+2025-03-03 15:56:10,171 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_17.jpg_r3_c0.jpg
+2025-03-03 15:56:10,389 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_17.jpg_r4_c0.jpg
+2025-03-03 15:56:10,610 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_17.jpg_r5_c0.jpg
+2025-03-03 15:56:10,610 [INFO] __main__ - Processing table image: /topic-extraction/img_18.jpg, columns=three
+2025-03-03 15:56:11,718 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_18.jpg_r0_c0.jpg
+2025-03-03 15:56:11,899 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_18.jpg_r0_c1.jpg
+2025-03-03 15:56:12,081 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_18.jpg_r1_c0.jpg
+2025-03-03 15:56:12,266 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_18.jpg_r1_c1.jpg
+2025-03-03 15:56:12,266 [INFO] __main__ - Processing table image: /topic-extraction/img_19.jpg, columns=three
+2025-03-03 15:56:15,231 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_19.jpg_r0_c0.jpg
+2025-03-03 15:56:15,582 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_19.jpg_r0_c1.jpg
+2025-03-03 15:56:15,802 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_19.jpg_r1_c0.jpg
+2025-03-03 15:56:16,018 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_19.jpg_r1_c1.jpg
+2025-03-03 15:56:16,234 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_19.jpg_r2_c0.jpg
+2025-03-03 15:56:16,451 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_19.jpg_r2_c1.jpg
+2025-03-03 15:56:16,452 [INFO] __main__ - Processing table image: /topic-extraction/img_20.jpg, columns=three
+2025-03-03 15:56:20,970 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_20.jpg_r0_c0.jpg
+2025-03-03 15:56:21,300 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_20.jpg_r0_c1.jpg
+2025-03-03 15:56:21,518 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_20.jpg_r1_c0.jpg
+2025-03-03 15:56:21,742 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_20.jpg_r1_c1.jpg
+2025-03-03 15:56:21,742 [INFO] __main__ - Processing table image: /topic-extraction/img_21.jpg, columns=three
+2025-03-03 15:56:25,577 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_21.jpg_r0_c0.jpg
+2025-03-03 15:56:25,883 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_21.jpg_r0_c1.jpg
+2025-03-03 15:56:26,108 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_21.jpg_r1_c0.jpg
+2025-03-03 15:56:26,319 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_21.jpg_r1_c1.jpg
+2025-03-03 15:56:26,320 [INFO] __main__ - Processing table image: /topic-extraction/img_22.jpg, columns=three
+2025-03-03 15:56:30,722 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_22.jpg_r0_c0.jpg
+2025-03-03 15:56:31,018 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_22.jpg_r0_c1.jpg
+2025-03-03 15:56:31,267 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_22.jpg_r1_c0.jpg
+2025-03-03 15:56:31,455 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_22.jpg_r1_c1.jpg
+2025-03-03 15:56:31,684 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_22.jpg_r2_c0.jpg
+2025-03-03 15:56:31,904 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_22.jpg_r2_c1.jpg
+2025-03-03 15:56:32,136 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_22.jpg_r3_c0.jpg
+2025-03-03 15:56:32,136 [INFO] __main__ - Processing table image: /topic-extraction/img_23.jpg, columns=three
+2025-03-03 15:56:35,410 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_23.jpg_r0_c0.jpg
+2025-03-03 15:56:35,689 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_23.jpg_r0_c1.jpg
+2025-03-03 15:56:35,917 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_23.jpg_r1_c0.jpg
+2025-03-03 15:56:36,143 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_23.jpg_r1_c1.jpg
+2025-03-03 15:56:36,144 [INFO] __main__ - Processing table image: /topic-extraction/img_24.jpg, columns=three
+2025-03-03 15:56:39,869 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_24.jpg_r0_c0.jpg
+2025-03-03 15:56:40,150 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_24.jpg_r0_c1.jpg
+2025-03-03 15:56:40,387 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_24.jpg_r1_c0.jpg
+2025-03-03 15:56:40,608 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_24.jpg_r1_c1.jpg
+2025-03-03 15:56:40,828 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_24.jpg_r2_c0.jpg
+2025-03-03 15:56:40,829 [INFO] __main__ - Processing table image: /topic-extraction/img_25.jpg, columns=two
+2025-03-03 15:56:44,221 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_25.jpg_r0_c0.jpg
+2025-03-03 15:56:44,522 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_25.jpg_r1_c0.jpg
+2025-03-03 15:56:44,728 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_25.jpg_r2_c0.jpg
+2025-03-03 15:56:44,929 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_25.jpg_r3_c0.jpg
+2025-03-03 15:56:45,153 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_25.jpg_r4_c0.jpg
+2025-03-03 15:56:45,372 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_25.jpg_r5_c0.jpg
+2025-03-03 15:56:45,372 [INFO] __main__ - Processing table image: /topic-extraction/img_26.jpg, columns=three
+2025-03-03 15:56:48,485 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_26.jpg_r0_c0.jpg
+2025-03-03 15:56:48,806 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_26.jpg_r0_c1.jpg
+2025-03-03 15:56:49,036 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_26.jpg_r1_c0.jpg
+2025-03-03 15:56:49,282 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_26.jpg_r2_c0.jpg
+2025-03-03 15:56:49,282 [INFO] __main__ - Processing table image: /topic-extraction/img_27.jpg, columns=three
+2025-03-03 15:56:52,374 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_27.jpg_r0_c0.jpg
+2025-03-03 15:56:52,664 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_27.jpg_r0_c1.jpg
+2025-03-03 15:56:52,887 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_27.jpg_r1_c0.jpg
+2025-03-03 15:56:53,103 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_27.jpg_r1_c1.jpg
+2025-03-03 15:56:53,329 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_27.jpg_r2_c0.jpg
+2025-03-03 15:56:53,543 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_27.jpg_r3_c0.jpg
+2025-03-03 15:56:53,759 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_27.jpg_r4_c0.jpg
+2025-03-03 15:56:53,978 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_27.jpg_r4_c1.jpg
+2025-03-03 15:56:53,979 [INFO] __main__ - Processing table image: /topic-extraction/img_28.jpg, columns=two
+2025-03-03 15:56:57,389 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_28.jpg_r0_c0.jpg
+2025-03-03 15:56:57,690 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_28.jpg_r1_c0.jpg
+2025-03-03 15:56:57,897 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_28.jpg_r2_c0.jpg
+2025-03-03 15:56:58,126 [INFO] __main__ - Uploaded to S3: /topic-extraction/cells/table_s3_img_28.jpg_r3_c0.jpg
+2025-03-03 15:56:58,131 [INFO] __main__ - Final subtopics JSON saved locally at /home/user/app/we/we_ars/final_subtopics.json
+2025-03-03 15:56:58,438 [INFO] __main__ - GPU memory cleaned up.
+2025-03-03 15:56:58,445 [INFO] __main__ - Processing completed successfully.

we/final_subtopics.json

@@ -1 +1,282 @@
-[]
+[
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_1.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_2.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_3.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_4.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_5.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_6.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_7.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_8.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_9.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_10.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_11.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_12.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_13.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_14.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_15.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_16.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_17.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_18.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_19.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_20.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_21.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_22.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_23.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_24.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_25.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_26.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_27.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_28.jpg"
+      }
+    ],
+    "children": []
+  }
+]

we/we_ars/final_subtopics.json

@@ -0,0 +1,282 @@
+[
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_1.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_2.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_3.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_4.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_5.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_6.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_7.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_8.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_9.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_10.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_11.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_12.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_13.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_14.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_15.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_16.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_17.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_18.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_19.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_20.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_21.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_22.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_23.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_24.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_25.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_26.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_27.jpg"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": null,
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/img_28.jpg"
+      }
+    ],
+    "children": []
+  }
+]

wje/final_output.json

@@ -0,0 +1,265 @@
+{
+  "subtopics": {
+    "Paper 1 and Paper 2: Pure Mathematics": [
+      11,
+      29
+    ],
+    "Paper 3: Statistics and Mechanics": [
+      30,
+      40
+    ]
+  },
+  "local_images": {
+    "e7e5c8f3c0a6316c2b50698c45ebe05b49bfd8bbe47a07b7b1929dd3cfd3e609.jpg": "img_1.jpg",
+    "b243ef738ec2465b1cc00f4dd8dd0e5f5e10a91debf7762903ac6c023dd238c4.jpg": "img_2.jpg",
+    "5e22a8a8c5bc23ee4d16bda9cce4a6ab4bb53854074fd4d691531d5adb9f3ebe.jpg": "img_3.jpg",
+    "0e70645e72eadab75c88846b7947fc1216cf31d325febf02fbdf4898b430465d.jpg": "img_4.jpg",
+    "52484f429af5d74ef75e96bf132b15fdc4acd2ed46accb981d670592dcc57ff2.jpg": "img_5.jpg",
+    "5a153708e7a128d8f6477cb294d2f902d3a9bc57af709c81ccc3937b96580137.jpg": "img_6.jpg",
+    "fd3a52607bef204e6998e09db82d195de76d929399c2cb1a63e26f87054eec6f.jpg": "img_7.jpg",
+    "93885318f77c148b9fb1cd162cb9938d6f6cd795d000d5b997f2297198462fcf.jpg": "img_8.jpg",
+    "133a659582f49fb71dc5fcae918278e6659a257026e35741ba8e6b94fcdb9de6.jpg": "img_9.jpg",
+    "860d10a56a7e892c674f74fd030592339e629fb80d6e6dbfc343f95ec65a7c16.jpg": "img_10.jpg",
+    "ae5ee4479ae736ff433ca9b2a1c3f753bbc8cc11a384e27cb710b426757c31e9.jpg": "img_11.jpg",
+    "39ac9ccc8cd681e552fb1ae08341b4a2dcb33ea8fe6c787daf99fb993d29e57f.jpg": "img_12.jpg",
+    "6d67beb5c0bf2168a87ad6b7c179ff9c7de8bbd7e720f77f7bf206080cceb589.jpg": "img_13.jpg",
+    "b89d31200bc06fda181bd2538b5f3274de3e52b0adc7dd023ca676e168e6d487.jpg": "img_14.jpg",
+    "78907967ba7a56221a0987e6e696e361c82fcf057f41659e4aa77943a62b6763.jpg": "img_15.jpg",
+    "bd3eb31469dd7b72e9773564915dc768e2e152878d887dcab34e83875e0625bb.jpg": "img_16.jpg",
+    "f1f1acb21df3d785fa3120fbae5fc74f7064769d9b38524bb991cfaa110177f6.jpg": "img_17.jpg",
+    "b8b803d008ec9053c40f4a9c2c265a8a0b15742059331dc7997336c94ab74dc4.jpg": "img_18.jpg",
+    "9cbb4e3b89d75d1d5da2fe8c6ccc4c1d3f612779abaccf3322f8b78b2db8a161.jpg": "img_19.jpg",
+    "c6c4dfd8d7d1b83ef05d0ad30d4d09e75fe1d1152099b976eef7aededb872873.jpg": "img_20.jpg",
+    "7eaeb5261341b3dbe0554989b2681f87c4b7a418e21445f3e88aa873e16db0df.jpg": "img_21.jpg",
+    "22cbebb54b25ccf620ab043fc977fcc709fd5692d1e74b02267b8f689284225d.jpg": "img_22.jpg",
+    "7a3f07a668cfc19e26c35fb1421908638d5a233723942301eda2764a1e81374d.jpg": "img_23.jpg",
+    "42b9e068a3fddcc2adaa6736e0ccee448c0302349547c8eaed8a07c870d29b17.jpg": "img_24.jpg",
+    "2efcd74e6c9447686d3e08d2dca6998ffd44f5cf0323d7d93b4213a2337b32ab.jpg": "img_25.jpg",
+    "6ba16781c7909a8a47a6a51e520e739320c22791147ad6bbd482473cf5c96717.jpg": "img_26.jpg",
+    "3d3cdfbca59671749e9d93714510a36441a10769f6b43720f9f3e733d893ea3a.jpg": "img_27.jpg",
+    "35394d307566e17440ab0322a3c915a4537db1db85628b38f2fe7827d19d719d.jpg": "img_28.jpg"
+  },
+  "tables_extracted": {
+    "img_1.jpg": [
+      "img_1.jpg_rows/row_0/col_0.png",
+      "img_1.jpg_rows/row_0/col_1.png",
+      "img_1.jpg_rows/row_1/col_0.png",
+      "img_1.jpg_rows/row_1/col_1.png"
+    ],
+    "img_2.jpg": [
+      "img_2.jpg_rows/row_0/col_0.png",
+      "img_2.jpg_rows/row_0/col_1.png",
+      "img_2.jpg_rows/row_1/col_0.png",
+      "img_2.jpg_rows/row_2/col_0.png",
+      "img_2.jpg_rows/row_3/col_0.png"
+    ],
+    "img_3.jpg": [
+      "img_3.jpg_rows/row_0/col_0.png",
+      "img_3.jpg_rows/row_0/col_1.png",
+      "img_3.jpg_rows/row_1/col_0.png"
+    ],
+    "img_4.jpg": [
+      "img_4.jpg_rows/row_0/col_0.png",
+      "img_4.jpg_rows/row_0/col_1.png",
+      "img_4.jpg_rows/row_1/col_0.png",
+      "img_4.jpg_rows/row_1/col_1.png"
+    ],
+    "img_5.jpg": [
+      "img_5.jpg_rows/row_0/col_0.png",
+      "img_5.jpg_rows/row_0/col_1.png",
+      "img_5.jpg_rows/row_1/col_0.png",
+      "img_5.jpg_rows/row_1/col_1.png",
+      "img_5.jpg_rows/row_2/col_0.png"
+    ],
+    "img_6.jpg": [
+      "img_6.jpg_rows/row_0/col_0.png",
+      "img_6.jpg_rows/row_0/col_1.png",
+      "img_6.jpg_rows/row_1/col_0.png",
+      "img_6.jpg_rows/row_1/col_1.png"
+    ],
+    "img_7.jpg": [
+      "img_7.jpg_rows/row_0/col_0.png",
+      "img_7.jpg_rows/row_0/col_1.png",
+      "img_7.jpg_rows/row_1/col_0.png",
+      "img_7.jpg_rows/row_2/col_0.png",
+      "img_7.jpg_rows/row_2/col_1.png"
+    ],
+    "img_8.jpg": [
+      "img_8.jpg_rows/row_0/col_0.png",
+      "img_8.jpg_rows/row_0/col_1.png",
+      "img_8.jpg_rows/row_0/col_2.png",
+      "img_8.jpg_rows/row_1/col_0.png",
+      "img_8.jpg_rows/row_1/col_1.png",
+      "img_8.jpg_rows/row_1/col_2.png",
+      "img_8.jpg_rows/row_2/col_0.png",
+      "img_8.jpg_rows/row_2/col_1.png",
+      "img_8.jpg_rows/row_3/col_0.png",
+      "img_8.jpg_rows/row_3/col_1.png",
+      "img_8.jpg_rows/row_4/col_0.png",
+      "img_8.jpg_rows/row_4/col_1.png",
+      "img_8.jpg_rows/row_5/col_0.png",
+      "img_8.jpg_rows/row_5/col_1.png"
+    ],
+    "img_9.jpg": [
+      "img_9.jpg_rows/row_0/col_0.png",
+      "img_9.jpg_rows/row_0/col_1.png",
+      "img_9.jpg_rows/row_0/col_2.png",
+      "img_9.jpg_rows/row_1/col_0.png",
+      "img_9.jpg_rows/row_1/col_1.png",
+      "img_9.jpg_rows/row_2/col_0.png",
+      "img_9.jpg_rows/row_2/col_1.png",
+      "img_9.jpg_rows/row_3/col_0.png",
+      "img_9.jpg_rows/row_3/col_1.png"
+    ],
+    "img_10.jpg": [
+      "img_10.jpg_rows/row_0/col_0.png",
+      "img_10.jpg_rows/row_0/col_1.png",
+      "img_10.jpg_rows/row_1/col_0.png",
+      "img_10.jpg_rows/row_2/col_0.png",
+      "img_10.jpg_rows/row_3/col_0.png"
+    ],
+    "img_11.jpg": [
+      "img_11.jpg_rows/row_0/col_0.png",
+      "img_11.jpg_rows/row_1/col_0.png",
+      "img_11.jpg_rows/row_2/col_0.png",
+      "img_11.jpg_rows/row_3/col_0.png",
+      "img_11.jpg_rows/row_4/col_0.png",
+      "img_11.jpg_rows/row_5/col_0.png"
+    ],
+    "img_12.jpg": [
+      "img_12.jpg_rows/row_0/col_0.png",
+      "img_12.jpg_rows/row_0/col_1.png",
+      "img_12.jpg_rows/row_1/col_0.png",
+      "img_12.jpg_rows/row_1/col_1.png",
+      "img_12.jpg_rows/row_2/col_0.png",
+      "img_12.jpg_rows/row_2/col_1.png"
+    ],
+    "img_13.jpg": [
+      "img_13.jpg_rows/row_0/col_0.png",
+      "img_13.jpg_rows/row_0/col_1.png",
+      "img_13.jpg_rows/row_1/col_0.png",
+      "img_13.jpg_rows/row_1/col_1.png",
+      "img_13.jpg_rows/row_2/col_0.png",
+      "img_13.jpg_rows/row_3/col_0.png"
+    ],
+    "img_14.jpg": [
+      "img_14.jpg_rows/row_0/col_0.png",
+      "img_14.jpg_rows/row_0/col_1.png",
+      "img_14.jpg_rows/row_1/col_0.png",
+      "img_14.jpg_rows/row_1/col_1.png",
+      "img_14.jpg_rows/row_2/col_0.png",
+      "img_14.jpg_rows/row_3/col_0.png",
+      "img_14.jpg_rows/row_4/col_0.png",
+      "img_14.jpg_rows/row_4/col_1.png",
+      "img_14.jpg_rows/row_5/col_0.png"
+    ],
+    "img_15.jpg": [
+      "img_15.jpg_rows/row_0/col_0.png",
+      "img_15.jpg_rows/row_0/col_1.png",
+      "img_15.jpg_rows/row_1/col_0.png",
+      "img_15.jpg_rows/row_1/col_1.png",
+      "img_15.jpg_rows/row_2/col_0.png",
+      "img_15.jpg_rows/row_3/col_0.png",
+      "img_15.jpg_rows/row_4/col_0.png"
+    ],
+    "img_16.jpg": [
+      "img_16.jpg_rows/row_0/col_0.png",
+      "img_16.jpg_rows/row_0/col_1.png",
+      "img_16.jpg_rows/row_1/col_0.png",
+      "img_16.jpg_rows/row_1/col_1.png",
+      "img_16.jpg_rows/row_2/col_0.png",
+      "img_16.jpg_rows/row_3/col_0.png",
+      "img_16.jpg_rows/row_3/col_1.png",
+      "img_16.jpg_rows/row_4/col_0.png",
+      "img_16.jpg_rows/row_5/col_0.png"
+    ],
+    "img_17.jpg": [
+      "img_17.jpg_rows/row_0/col_0.png",
+      "img_17.jpg_rows/row_0/col_1.png",
+      "img_17.jpg_rows/row_1/col_0.png",
+      "img_17.jpg_rows/row_2/col_0.png",
+      "img_17.jpg_rows/row_2/col_1.png",
+      "img_17.jpg_rows/row_3/col_0.png",
+      "img_17.jpg_rows/row_4/col_0.png",
+      "img_17.jpg_rows/row_5/col_0.png"
+    ],
+    "img_18.jpg": [
+      "img_18.jpg_rows/row_0/col_0.png",
+      "img_18.jpg_rows/row_0/col_1.png",
+      "img_18.jpg_rows/row_1/col_0.png",
+      "img_18.jpg_rows/row_1/col_1.png"
+    ],
+    "img_19.jpg": [
+      "img_19.jpg_rows/row_0/col_0.png",
+      "img_19.jpg_rows/row_0/col_1.png",
+      "img_19.jpg_rows/row_1/col_0.png",
+      "img_19.jpg_rows/row_1/col_1.png",
+      "img_19.jpg_rows/row_2/col_0.png",
+      "img_19.jpg_rows/row_2/col_1.png"
+    ],
+    "img_20.jpg": [
+      "img_20.jpg_rows/row_0/col_0.png",
+      "img_20.jpg_rows/row_0/col_1.png",
+      "img_20.jpg_rows/row_1/col_0.png",
+      "img_20.jpg_rows/row_1/col_1.png"
+    ],
+    "img_21.jpg": [
+      "img_21.jpg_rows/row_0/col_0.png",
+      "img_21.jpg_rows/row_0/col_1.png",
+      "img_21.jpg_rows/row_1/col_0.png",
+      "img_21.jpg_rows/row_1/col_1.png"
+    ],
+    "img_22.jpg": [
+      "img_22.jpg_rows/row_0/col_0.png",
+      "img_22.jpg_rows/row_0/col_1.png",
+      "img_22.jpg_rows/row_1/col_0.png",
+      "img_22.jpg_rows/row_1/col_1.png",
+      "img_22.jpg_rows/row_2/col_0.png",
+      "img_22.jpg_rows/row_2/col_1.png",
+      "img_22.jpg_rows/row_3/col_0.png"
+    ],
+    "img_23.jpg": [
+      "img_23.jpg_rows/row_0/col_0.png",
+      "img_23.jpg_rows/row_0/col_1.png",
+      "img_23.jpg_rows/row_1/col_0.png",
+      "img_23.jpg_rows/row_1/col_1.png"
+    ],
+    "img_24.jpg": [
+      "img_24.jpg_rows/row_0/col_0.png",
+      "img_24.jpg_rows/row_0/col_1.png",
+      "img_24.jpg_rows/row_1/col_0.png",
+      "img_24.jpg_rows/row_1/col_1.png",
+      "img_24.jpg_rows/row_2/col_0.png"
+    ],
+    "img_25.jpg": [
+      "img_25.jpg_rows/row_0/col_0.png",
+      "img_25.jpg_rows/row_1/col_0.png",
+      "img_25.jpg_rows/row_2/col_0.png",
+      "img_25.jpg_rows/row_3/col_0.png",
+      "img_25.jpg_rows/row_4/col_0.png",
+      "img_25.jpg_rows/row_5/col_0.png"
+    ],
+    "img_26.jpg": [
+      "img_26.jpg_rows/row_0/col_0.png",
+      "img_26.jpg_rows/row_0/col_1.png",
+      "img_26.jpg_rows/row_1/col_0.png",
+      "img_26.jpg_rows/row_2/col_0.png"
+    ],
+    "img_27.jpg": [
+      "img_27.jpg_rows/row_0/col_0.png",
+      "img_27.jpg_rows/row_0/col_1.png",
+      "img_27.jpg_rows/row_1/col_0.png",
+      "img_27.jpg_rows/row_1/col_1.png",
+      "img_27.jpg_rows/row_2/col_0.png",
+      "img_27.jpg_rows/row_3/col_0.png",
+      "img_27.jpg_rows/row_4/col_0.png",
+      "img_27.jpg_rows/row_4/col_1.png"
+    ],
+    "img_28.jpg": [
+      "img_28.jpg_rows/row_0/col_0.png",
+      "img_28.jpg_rows/row_1/col_0.png",
+      "img_28.jpg_rows/row_2/col_0.png",
+      "img_28.jpg_rows/row_3/col_0.png"
+    ]
+  }
+}

wje/final_output_local.json

@@ -0,0 +1,265 @@
+{
+  "subtopics": {
+    "Paper 1 and Paper 2: Pure Mathematics": [
+      11,
+      29
+    ],
+    "Paper 3: Statistics and Mechanics": [
+      30,
+      40
+    ]
+  },
+  "local_images": {
+    "e7e5c8f3c0a6316c2b50698c45ebe05b49bfd8bbe47a07b7b1929dd3cfd3e609.jpg": "img_1.jpg",
+    "b243ef738ec2465b1cc00f4dd8dd0e5f5e10a91debf7762903ac6c023dd238c4.jpg": "img_2.jpg",
+    "5e22a8a8c5bc23ee4d16bda9cce4a6ab4bb53854074fd4d691531d5adb9f3ebe.jpg": "img_3.jpg",
+    "0e70645e72eadab75c88846b7947fc1216cf31d325febf02fbdf4898b430465d.jpg": "img_4.jpg",
+    "52484f429af5d74ef75e96bf132b15fdc4acd2ed46accb981d670592dcc57ff2.jpg": "img_5.jpg",
+    "5a153708e7a128d8f6477cb294d2f902d3a9bc57af709c81ccc3937b96580137.jpg": "img_6.jpg",
+    "fd3a52607bef204e6998e09db82d195de76d929399c2cb1a63e26f87054eec6f.jpg": "img_7.jpg",
+    "93885318f77c148b9fb1cd162cb9938d6f6cd795d000d5b997f2297198462fcf.jpg": "img_8.jpg",
+    "133a659582f49fb71dc5fcae918278e6659a257026e35741ba8e6b94fcdb9de6.jpg": "img_9.jpg",
+    "860d10a56a7e892c674f74fd030592339e629fb80d6e6dbfc343f95ec65a7c16.jpg": "img_10.jpg",
+    "ae5ee4479ae736ff433ca9b2a1c3f753bbc8cc11a384e27cb710b426757c31e9.jpg": "img_11.jpg",
+    "39ac9ccc8cd681e552fb1ae08341b4a2dcb33ea8fe6c787daf99fb993d29e57f.jpg": "img_12.jpg",
+    "6d67beb5c0bf2168a87ad6b7c179ff9c7de8bbd7e720f77f7bf206080cceb589.jpg": "img_13.jpg",
+    "b89d31200bc06fda181bd2538b5f3274de3e52b0adc7dd023ca676e168e6d487.jpg": "img_14.jpg",
+    "78907967ba7a56221a0987e6e696e361c82fcf057f41659e4aa77943a62b6763.jpg": "img_15.jpg",
+    "bd3eb31469dd7b72e9773564915dc768e2e152878d887dcab34e83875e0625bb.jpg": "img_16.jpg",
+    "f1f1acb21df3d785fa3120fbae5fc74f7064769d9b38524bb991cfaa110177f6.jpg": "img_17.jpg",
+    "b8b803d008ec9053c40f4a9c2c265a8a0b15742059331dc7997336c94ab74dc4.jpg": "img_18.jpg",
+    "9cbb4e3b89d75d1d5da2fe8c6ccc4c1d3f612779abaccf3322f8b78b2db8a161.jpg": "img_19.jpg",
+    "c6c4dfd8d7d1b83ef05d0ad30d4d09e75fe1d1152099b976eef7aededb872873.jpg": "img_20.jpg",
+    "7eaeb5261341b3dbe0554989b2681f87c4b7a418e21445f3e88aa873e16db0df.jpg": "img_21.jpg",
+    "22cbebb54b25ccf620ab043fc977fcc709fd5692d1e74b02267b8f689284225d.jpg": "img_22.jpg",
+    "7a3f07a668cfc19e26c35fb1421908638d5a233723942301eda2764a1e81374d.jpg": "img_23.jpg",
+    "42b9e068a3fddcc2adaa6736e0ccee448c0302349547c8eaed8a07c870d29b17.jpg": "img_24.jpg",
+    "2efcd74e6c9447686d3e08d2dca6998ffd44f5cf0323d7d93b4213a2337b32ab.jpg": "img_25.jpg",
+    "6ba16781c7909a8a47a6a51e520e739320c22791147ad6bbd482473cf5c96717.jpg": "img_26.jpg",
+    "3d3cdfbca59671749e9d93714510a36441a10769f6b43720f9f3e733d893ea3a.jpg": "img_27.jpg",
+    "35394d307566e17440ab0322a3c915a4537db1db85628b38f2fe7827d19d719d.jpg": "img_28.jpg"
+  },
+  "tables_extracted": {
+    "img_1.jpg": [
+      "img_1.jpg_rows/row_0/col_0.png",
+      "img_1.jpg_rows/row_0/col_1.png",
+      "img_1.jpg_rows/row_1/col_0.png",
+      "img_1.jpg_rows/row_1/col_1.png"
+    ],
+    "img_2.jpg": [
+      "img_2.jpg_rows/row_0/col_0.png",
+      "img_2.jpg_rows/row_0/col_1.png",
+      "img_2.jpg_rows/row_1/col_0.png",
+      "img_2.jpg_rows/row_2/col_0.png",
+      "img_2.jpg_rows/row_3/col_0.png"
+    ],
+    "img_3.jpg": [
+      "img_3.jpg_rows/row_0/col_0.png",
+      "img_3.jpg_rows/row_0/col_1.png",
+      "img_3.jpg_rows/row_1/col_0.png"
+    ],
+    "img_4.jpg": [
+      "img_4.jpg_rows/row_0/col_0.png",
+      "img_4.jpg_rows/row_0/col_1.png",
+      "img_4.jpg_rows/row_1/col_0.png",
+      "img_4.jpg_rows/row_1/col_1.png"
+    ],
+    "img_5.jpg": [
+      "img_5.jpg_rows/row_0/col_0.png",
+      "img_5.jpg_rows/row_0/col_1.png",
+      "img_5.jpg_rows/row_1/col_0.png",
+      "img_5.jpg_rows/row_1/col_1.png",
+      "img_5.jpg_rows/row_2/col_0.png"
+    ],
+    "img_6.jpg": [
+      "img_6.jpg_rows/row_0/col_0.png",
+      "img_6.jpg_rows/row_0/col_1.png",
+      "img_6.jpg_rows/row_1/col_0.png",
+      "img_6.jpg_rows/row_1/col_1.png"
+    ],
+    "img_7.jpg": [
+      "img_7.jpg_rows/row_0/col_0.png",
+      "img_7.jpg_rows/row_0/col_1.png",
+      "img_7.jpg_rows/row_1/col_0.png",
+      "img_7.jpg_rows/row_2/col_0.png",
+      "img_7.jpg_rows/row_2/col_1.png"
+    ],
+    "img_8.jpg": [
+      "img_8.jpg_rows/row_0/col_0.png",
+      "img_8.jpg_rows/row_0/col_1.png",
+      "img_8.jpg_rows/row_0/col_2.png",
+      "img_8.jpg_rows/row_1/col_0.png",
+      "img_8.jpg_rows/row_1/col_1.png",
+      "img_8.jpg_rows/row_1/col_2.png",
+      "img_8.jpg_rows/row_2/col_0.png",
+      "img_8.jpg_rows/row_2/col_1.png",
+      "img_8.jpg_rows/row_3/col_0.png",
+      "img_8.jpg_rows/row_3/col_1.png",
+      "img_8.jpg_rows/row_4/col_0.png",
+      "img_8.jpg_rows/row_4/col_1.png",
+      "img_8.jpg_rows/row_5/col_0.png",
+      "img_8.jpg_rows/row_5/col_1.png"
+    ],
+    "img_9.jpg": [
+      "img_9.jpg_rows/row_0/col_0.png",
+      "img_9.jpg_rows/row_0/col_1.png",
+      "img_9.jpg_rows/row_0/col_2.png",
+      "img_9.jpg_rows/row_1/col_0.png",
+      "img_9.jpg_rows/row_1/col_1.png",
+      "img_9.jpg_rows/row_2/col_0.png",
+      "img_9.jpg_rows/row_2/col_1.png",
+      "img_9.jpg_rows/row_3/col_0.png",
+      "img_9.jpg_rows/row_3/col_1.png"
+    ],
+    "img_10.jpg": [
+      "img_10.jpg_rows/row_0/col_0.png",
+      "img_10.jpg_rows/row_0/col_1.png",
+      "img_10.jpg_rows/row_1/col_0.png",
+      "img_10.jpg_rows/row_2/col_0.png",
+      "img_10.jpg_rows/row_3/col_0.png"
+    ],
+    "img_11.jpg": [
+      "img_11.jpg_rows/row_0/col_0.png",
+      "img_11.jpg_rows/row_1/col_0.png",
+      "img_11.jpg_rows/row_2/col_0.png",
+      "img_11.jpg_rows/row_3/col_0.png",
+      "img_11.jpg_rows/row_4/col_0.png",
+      "img_11.jpg_rows/row_5/col_0.png"
+    ],
+    "img_12.jpg": [
+      "img_12.jpg_rows/row_0/col_0.png",
+      "img_12.jpg_rows/row_0/col_1.png",
+      "img_12.jpg_rows/row_1/col_0.png",
+      "img_12.jpg_rows/row_1/col_1.png",
+      "img_12.jpg_rows/row_2/col_0.png",
+      "img_12.jpg_rows/row_2/col_1.png"
+    ],
+    "img_13.jpg": [
+      "img_13.jpg_rows/row_0/col_0.png",
+      "img_13.jpg_rows/row_0/col_1.png",
+      "img_13.jpg_rows/row_1/col_0.png",
+      "img_13.jpg_rows/row_1/col_1.png",
+      "img_13.jpg_rows/row_2/col_0.png",
+      "img_13.jpg_rows/row_3/col_0.png"
+    ],
+    "img_14.jpg": [
+      "img_14.jpg_rows/row_0/col_0.png",
+      "img_14.jpg_rows/row_0/col_1.png",
+      "img_14.jpg_rows/row_1/col_0.png",
+      "img_14.jpg_rows/row_1/col_1.png",
+      "img_14.jpg_rows/row_2/col_0.png",
+      "img_14.jpg_rows/row_3/col_0.png",
+      "img_14.jpg_rows/row_4/col_0.png",
+      "img_14.jpg_rows/row_4/col_1.png",
+      "img_14.jpg_rows/row_5/col_0.png"
+    ],
+    "img_15.jpg": [
+      "img_15.jpg_rows/row_0/col_0.png",
+      "img_15.jpg_rows/row_0/col_1.png",
+      "img_15.jpg_rows/row_1/col_0.png",
+      "img_15.jpg_rows/row_1/col_1.png",
+      "img_15.jpg_rows/row_2/col_0.png",
+      "img_15.jpg_rows/row_3/col_0.png",
+      "img_15.jpg_rows/row_4/col_0.png"
+    ],
+    "img_16.jpg": [
+      "img_16.jpg_rows/row_0/col_0.png",
+      "img_16.jpg_rows/row_0/col_1.png",
+      "img_16.jpg_rows/row_1/col_0.png",
+      "img_16.jpg_rows/row_1/col_1.png",
+      "img_16.jpg_rows/row_2/col_0.png",
+      "img_16.jpg_rows/row_3/col_0.png",
+      "img_16.jpg_rows/row_3/col_1.png",
+      "img_16.jpg_rows/row_4/col_0.png",
+      "img_16.jpg_rows/row_5/col_0.png"
+    ],
+    "img_17.jpg": [
+      "img_17.jpg_rows/row_0/col_0.png",
+      "img_17.jpg_rows/row_0/col_1.png",
+      "img_17.jpg_rows/row_1/col_0.png",
+      "img_17.jpg_rows/row_2/col_0.png",
+      "img_17.jpg_rows/row_2/col_1.png",
+      "img_17.jpg_rows/row_3/col_0.png",
+      "img_17.jpg_rows/row_4/col_0.png",
+      "img_17.jpg_rows/row_5/col_0.png"
+    ],
+    "img_18.jpg": [
+      "img_18.jpg_rows/row_0/col_0.png",
+      "img_18.jpg_rows/row_0/col_1.png",
+      "img_18.jpg_rows/row_1/col_0.png",
+      "img_18.jpg_rows/row_1/col_1.png"
+    ],
+    "img_19.jpg": [
+      "img_19.jpg_rows/row_0/col_0.png",
+      "img_19.jpg_rows/row_0/col_1.png",
+      "img_19.jpg_rows/row_1/col_0.png",
+      "img_19.jpg_rows/row_1/col_1.png",
+      "img_19.jpg_rows/row_2/col_0.png",
+      "img_19.jpg_rows/row_2/col_1.png"
+    ],
+    "img_20.jpg": [
+      "img_20.jpg_rows/row_0/col_0.png",
+      "img_20.jpg_rows/row_0/col_1.png",
+      "img_20.jpg_rows/row_1/col_0.png",
+      "img_20.jpg_rows/row_1/col_1.png"
+    ],
+    "img_21.jpg": [
+      "img_21.jpg_rows/row_0/col_0.png",
+      "img_21.jpg_rows/row_0/col_1.png",
+      "img_21.jpg_rows/row_1/col_0.png",
+      "img_21.jpg_rows/row_1/col_1.png"
+    ],
+    "img_22.jpg": [
+      "img_22.jpg_rows/row_0/col_0.png",
+      "img_22.jpg_rows/row_0/col_1.png",
+      "img_22.jpg_rows/row_1/col_0.png",
+      "img_22.jpg_rows/row_1/col_1.png",
+      "img_22.jpg_rows/row_2/col_0.png",
+      "img_22.jpg_rows/row_2/col_1.png",
+      "img_22.jpg_rows/row_3/col_0.png"
+    ],
+    "img_23.jpg": [
+      "img_23.jpg_rows/row_0/col_0.png",
+      "img_23.jpg_rows/row_0/col_1.png",
+      "img_23.jpg_rows/row_1/col_0.png",
+      "img_23.jpg_rows/row_1/col_1.png"
+    ],
+    "img_24.jpg": [
+      "img_24.jpg_rows/row_0/col_0.png",
+      "img_24.jpg_rows/row_0/col_1.png",
+      "img_24.jpg_rows/row_1/col_0.png",
+      "img_24.jpg_rows/row_1/col_1.png",
+      "img_24.jpg_rows/row_2/col_0.png"
+    ],
+    "img_25.jpg": [
+      "img_25.jpg_rows/row_0/col_0.png",
+      "img_25.jpg_rows/row_1/col_0.png",
+      "img_25.jpg_rows/row_2/col_0.png",
+      "img_25.jpg_rows/row_3/col_0.png",
+      "img_25.jpg_rows/row_4/col_0.png",
+      "img_25.jpg_rows/row_5/col_0.png"
+    ],
+    "img_26.jpg": [
+      "img_26.jpg_rows/row_0/col_0.png",
+      "img_26.jpg_rows/row_0/col_1.png",
+      "img_26.jpg_rows/row_1/col_0.png",
+      "img_26.jpg_rows/row_2/col_0.png"
+    ],
+    "img_27.jpg": [
+      "img_27.jpg_rows/row_0/col_0.png",
+      "img_27.jpg_rows/row_0/col_1.png",
+      "img_27.jpg_rows/row_1/col_0.png",
+      "img_27.jpg_rows/row_1/col_1.png",
+      "img_27.jpg_rows/row_2/col_0.png",
+      "img_27.jpg_rows/row_3/col_0.png",
+      "img_27.jpg_rows/row_4/col_0.png",
+      "img_27.jpg_rows/row_4/col_1.png"
+    ],
+    "img_28.jpg": [
+      "img_28.jpg_rows/row_0/col_0.png",
+      "img_28.jpg_rows/row_1/col_0.png",
+      "img_28.jpg_rows/row_2/col_0.png",
+      "img_28.jpg_rows/row_3/col_0.png"
+    ]
+  }
+}