#!/bin/bash
# Extracts plain text from every machine-readable PDF in the corpus into
# extracted-text/<board>/<class>/<name>.txt (layout mode, UTF-8).
# Skips files whose extraction yields almost nothing (scanned books -> OCR later).
set -u
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
OUT="$ROOT/extracted-text"
find "$ROOT" -name "*.pdf" -not -path "*/extracted-text/*" | sort | while read -r f; do
  rel=${f#"$ROOT"/}
  board=$(echo "$rel" | cut -d/ -f1)
  cdir=$(echo "$rel" | cut -d/ -f2)
  base=$(basename "$f" .pdf)
  mkdir -p "$OUT/$board/$cdir"
  txt="$OUT/$board/$cdir/$base.txt"
  [ -s "$txt" ] && { echo "SKIP $txt"; continue; }
  [ -s "$txt.scanned-stub" ] && { echo "SKIP (stub) $txt"; continue; }
  pdftotext -layout -enc UTF-8 "$f" "$txt" 2>/dev/null
  chars=$(tr -d '[:space:]' < "$txt" | wc -c | tr -d ' ')
  pages=$(pdfinfo "$f" 2>/dev/null | awk '/^Pages:/{print $2}')
  if [ "${chars:-0}" -lt $(( ${pages:-1} * 250 )) ]; then
    echo "SCANNED (kept stub, ${chars}c/${pages}p): $rel"
    mv "$txt" "$txt.scanned-stub"
  else
    echo "OK  ${chars} chars: $txt"
  fi
done