#!/bin/bash
# Walks the corpus tree, probes every PDF, and writes manifest.csv + manifest.json.
# Columns: board_dir, class_dir, file, size_bytes, pages, pdf_producer, text_extractable(yes/partial/no)
# text_extractable heuristic: pdftotext of pages 2-6; >=400 chars of non-whitespace = yes,
# 50-399 = partial, <50 = no (likely scanned; needs OCR).
set -u
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
CSV="$ROOT/analysis/manifest.csv"
JSONL="$ROOT/analysis/manifest.jsonl"
echo "board,class,file,size_bytes,pages,producer,text_extractable" > "$CSV"
: > "$JSONL"
find "$ROOT" -name "*.pdf" -not -path "*/extracted-text/*" -not -path "*/site/*" | sort | while read -r f; do
  rel=${f#"$ROOT"/}
  board=$(echo "$rel" | cut -d/ -f1)
  cdir=$(echo "$rel" | cut -d/ -f2)
  base=$(basename "$f")
  size=$(stat -f%z "$f")
  pages=$(pdfinfo "$f" 2>/dev/null | awk '/^Pages:/{print $2}')
  prod=$(pdfinfo "$f" 2>/dev/null | awk -F': *' '/^Producer:/{print $2}' | tr ',' ';')
  mid=$(( ${pages:-10} / 2 )); [ "$mid" -lt 2 ] && mid=2
  chars=$(pdftotext -f "$mid" -l $((mid+4)) "$f" - 2>/dev/null | tr -d '[:space:]' | wc -c | tr -d ' ')
  if [ "${chars:-0}" -ge 400 ]; then ext="yes"; elif [ "${chars:-0}" -ge 50 ]; then ext="partial"; else ext="no"; fi
  echo "$board,$cdir,$base,$size,${pages:-?},\"${prod:-?}\",$ext" >> "$CSV"
  jq -cn --arg b "$board" --arg c "$cdir" --arg f "$base" --arg s "$size" --arg p "${pages:-}" --arg pr "${prod:-}" --arg e "$ext" \
    '{board:$b,class:$c,file:$f,size_bytes:($s|tonumber),pages:$p,producer:$pr,text_extractable:$e}' >> "$JSONL"
done
column -s, -t "$CSV"