#!/bin/bash
# Downloads textbooks listed in a JSON array file into the corpus tree and
# verifies each file is a real PDF.
#
# Usage: ./download.sh sources.json
# sources.json: [{"board":"PCTB","class":"9","subject":"Biology","medium":"English","title":"...","url":"..."}]
#
# Board -> directory mapping: PCTB -> pctb-punjab, STBB -> stbb-sindh, KPTBB -> kptbb-kp
set -u
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
UA="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36"
SRC="${1:?usage: download.sh sources.json}"
LOG="$ROOT/analysis/download-log.txt"
: > "$LOG"

dir_for_board() {
  case "$1" in
    PCTB) echo "pctb-punjab" ;;
    STBB) echo "stbb-sindh" ;;
    KPTBB) echo "kptbb-kp" ;;
    BTBB) echo "btbb-balochistan" ;;
    PTB-historic) echo "ptb-historic" ;;
    FBISE) echo "fbise-federal" ;;
    *) echo "other" ;;
  esac
}

slug() { echo "$1" | tr '[:upper:]' '[:lower:]' | tr -cs 'a-z0-9' '-' | sed 's/^-//;s/-$//'; }

n=$(jq length "$SRC")
ok=0; fail=0
for i in $(seq 0 $((n-1))); do
  board=$(jq -r ".[$i].board" "$SRC")
  klass=$(jq -r ".[$i].class" "$SRC")
  subject=$(jq -r ".[$i].subject" "$SRC")
  medium=$(jq -r ".[$i].medium // \"\"" "$SRC")
  url=$(jq -r ".[$i].url" "$SRC")
  bdir=$(dir_for_board "$board")
  # combined grades like "9-10" go in class-9 dir
  cdir="class-$(echo "$klass" | cut -d- -f1)"
  mkdir -p "$ROOT/$bdir/$cdir"
  msuffix=""
  [ -n "$medium" ] && msuffix="-$(slug "$medium" | cut -c1-2)"  # -en / -ur / -si
  out="$ROOT/$bdir/$cdir/$(slug "$subject")-$klass$msuffix.pdf"
  if [ -s "$out" ] && head -c 4 "$out" | grep -q "%PDF"; then
    echo "SKIP (exists): $out" | tee -a "$LOG"; ok=$((ok+1)); continue
  fi
  echo "GET  [$board cl.$klass $subject] $url" | tee -a "$LOG"
  curl -sSL --retry 3 --retry-delay 2 --max-time 600 -A "$UA" -o "$out.part" "$url" 2>>"$LOG"
  # Google Drive fallback: large files return an HTML confirm page from the uc?export URL
  if ! head -c 4 "$out.part" 2>/dev/null | grep -q "%PDF"; then
    fid=$(echo "$url" | sed -n 's/.*[?&]id=\([A-Za-z0-9_-]*\).*/\1/p')
    if [ -n "$fid" ] && echo "$url" | grep -q "drive.google.com"; then
      alt="https://drive.usercontent.google.com/download?id=$fid&export=download&confirm=t"
      echo "  RETRY via usercontent: $alt" | tee -a "$LOG"
      curl -sSL --retry 3 --retry-delay 2 --max-time 600 -A "$UA" -o "$out.part" "$alt" 2>>"$LOG"
    fi
  fi
  # Mirror /download/ routes redirect to Google Drive, which for large files answers with a
  # virus-scan interstitial instead of the file. The form on that page carries the real id and
  # a uuid; posting them to drive.usercontent returns the bytes. Detect and follow it.
  if head -c 512 "$out.part" 2>/dev/null | grep -qi "<!DOC\|<html"; then
    fid=$(grep -oE 'name="id" value="[^"]*"' "$out.part" 2>/dev/null | sed 's/.*value="//;s/"//' | head -1)
    uuid=$(grep -oE 'name="uuid" value="[^"]*"' "$out.part" 2>/dev/null | sed 's/.*value="//;s/"//' | head -1)
    if [ -n "$fid" ]; then
      echo "  RETRY via Drive confirm (id=$fid)" | tee -a "$LOG"
      curl -sSL --retry 3 --retry-delay 2 --max-time 900 -A "$UA" \
        "https://drive.usercontent.google.com/download?id=$fid&export=download&confirm=t&uuid=$uuid" \
        -o "$out.part2" && mv "$out.part2" "$out.part"
    fi
  fi
  # A failed transfer can still look like a PDF: one origin in this project died mid-stream
  # and glued an Apache "503 Service Unavailable" HTML page onto the end of the bytes, so the
  # file began with %PDF, had a plausible size, and had no trailer. Check the TAIL too.
  tail_ok=0
  if [ -s "$out.part" ]; then
    if tail -c 4096 "$out.part" | grep -qi "<html\|503 Service\|Service Unavailable"; then
      echo "  FAIL truncated — HTML error page appended mid-stream" | tee -a "$LOG"
      rm -f "$out.part"
    elif tail -c 4096 "$out.part" | grep -q "%%EOF"; then
      tail_ok=1
    else
      # No trailer in the tail. Some Acrobat output genuinely lacks one, so fall back to the
      # authoritative test: does the page tree parse and does the LAST page render?
      pages=$(pdfinfo "$out.part" 2>/dev/null | awk '/^Pages:/{print $2}')
      if [ -n "$pages" ] && pdftoppm -f "$pages" -l "$pages" -r 20 "$out.part" /tmp/dlchk >/dev/null 2>&1; then
        echo "  OK no %%EOF but page tree parses and page $pages renders" | tee -a "$LOG"
        rm -f /tmp/dlchk-*; tail_ok=1
      else
        echo "  FAIL no %%EOF and last page will not render — truncated" | tee -a "$LOG"
        rm -f "$out.part" /tmp/dlchk-*
      fi
    fi
  fi
  if [ "$tail_ok" = "1" ] && head -c 4 "$out.part" | grep -q "%PDF"; then
    # Mirrors mislabel medium: six KP books listed as "English medium" turned out to be the
    # byte-identical Urdu editions already held, and OCRing them with the English model
    # produced 18,000 words of confident noise. Reject an exact duplicate before it lands.
    newsum=$(shasum -a 256 "$out.part" | cut -d' ' -f1)
    dupe=$(find "$ROOT" -name "*.pdf" -not -path "*/site/*" -not -path "*/other/*" -size "$(stat -f%z "$out.part")c" 2>/dev/null \
           | while read -r ex; do [ "$ex" = "$out" ] && continue; [ "$(shasum -a 256 "$ex" | cut -d' ' -f1)" = "$newsum" ] && { echo "$ex"; break; }; done)
    if [ -n "$dupe" ]; then
      echo "  DUPLICATE of ${dupe#"$ROOT"/} — same bytes, not a separate edition. Discarded." | tee -a "$LOG"
      rm -f "$out.part"; fail=$((fail+1)); continue
    fi
    mv "$out.part" "$out"
    echo "  OK   $(du -h "$out" | cut -f1) -> $out" | tee -a "$LOG"
    ok=$((ok+1))
  else
    got=$(head -c 200 "$out.part" 2>/dev/null | tr -d '\0' | head -c 120)
    echo "  FAIL not a PDF (starts: ${got:-empty})" | tee -a "$LOG"
    rm -f "$out.part"
    fail=$((fail+1))
  fi
done
echo "---"; echo "Downloaded/present: $ok  Failed: $fail" | tee -a "$LOG"