#!/usr/bin/env python3 """Assemble the per-book research dataset (analysis/books.json) from: - analysis/manifest.csv (the spine: one row per PDF actually held) - analysis/sources-*.json (subject, medium, title, source URL) - a curated edition/vintage table (from analysis/EDITION-AUDIT.md findings) - analysis/lens-*.md (per-book scores, where the tables expose them) Scores are only recorded where a lens table states them. Missing is left null and reported — the site must never imply a score that was not assigned. """ import csv, json, glob, os, re, collections, datetime, hashlib ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) A = os.path.join(ROOT, "analysis") # Evidence tiers recomputed from whole-book text density (chars/page), not a 5-page # sample. A 5-page sample is defeated by mirror watermark text layers, which repeat a # constant string on every page: four KP class-8 books scored an identical 86 chars/page # and were previously misclassified as having native text. See scripts/compute_tiers.py. import subprocess def _density_tiers(rows_path): """Whole-book chars/page. Cached on (path, size) — pdftotext over the full corpus takes minutes, and the answer only changes when a file does.""" import csv as _csv, os as _os cache_path = _os.path.join(A, ".tier-cache.json") try: cache = json.load(open(cache_path)) except Exception: cache = {} out, dirty = {}, False for r in _csv.DictReader(open(rows_path)): pages = int(r["pages"]) if r["pages"].isdigit() else 0 if not pages: continue key = f"{r['board']}/{r['class']}/{r['file']}" path = _os.path.join(ROOT, r["board"], r["class"], r["file"]) if not _os.path.exists(path): continue stamp = f"{_os.path.getsize(path)}:{pages}" hit = cache.get(key) if hit and hit.get("stamp") == stamp: out[key] = hit["tier"]; continue txt = subprocess.run(["pdftotext", path, "-"], capture_output=True).stdout d = len(b"".join(txt.split())) // pages tier = "native" if d >= 250 else "partial" if d >= 100 else "scanned" out[key] = tier cache[key] = {"stamp": stamp, "tier": tier, "density": d} dirty = True if dirty: json.dump(cache, open(cache_path, "w"), indent=1) return out BOARD_DIR = {"pctb-punjab": "PCTB", "stbb-sindh": "STBB", "kptbb-kp": "KPTBB", "btbb-balochistan": "BTBB", "ptb-historic": "PTB-H", "fbise-federal": "FBISE"} BOARD_NAME = {"PCTB": "Punjab", "STBB": "Sindh", "KPTBB": "Khyber Pakhtunkhwa", "BTBB": "Balochistan", "PTB-H": "Punjab (historical)", "FBISE": "Federal"} BOARD_FULL = { "PCTB": "Punjab Curriculum and Textbook Board, Lahore", "STBB": "Sindh Textbook Board, Jamshoro", "KPTBB": "Khyber Pakhtunkhwa Textbook Board, Peshawar", "BTBB": "Balochistan Textbook Board, Quetta", "PTB-H": "Punjab Textbook Board, Lahore — historical editions", "FBISE": "Federal Board / National Book Foundation, Islamabad", } SUBJ = { # filename stem -> canonical subject "english": "English", "urdu": "Urdu", "islamiyat": "Islamiyat", "pakistan-studies": "Pakistan Studies", "general-science": "General Science", "social-studies": "Social Studies", "history": "History", "geography": "Geography", "biology": "Biology", "chemistry": "Chemistry", "physics": "Physics", "mutalea-quran": "Mutalea Quran", } # Curated vintage table. Sources: imprint pages, DCTE/board approval letters and the # STBB portal's own printed edition year, as recorded in analysis/EDITION-AUDIT.md. VINTAGE = { "PCTB": { # Punjab is SNC-2022 throughout EXCEPT its class-9 Urdu reader, whose front matter cites the # 2006 National Curriculum, federal approval ref. dated 21-10-2009 and a 2016 printing # (track-urdu.md p.2 evidence). Wildcarding this to SNC 2022 was wrong and is corrected here. "Urdu|9": ("National Curriculum 2006", "2016 printing", "Federal approval ref. dated 21-10-2009; front matter cites NC 2006"), # Punjab reissued class 10 under the 2026 PECTAA syllabus — a further generation beyond SNC 2022 "English|10": ("PECTAA 2026", "2026-27", "Publisher edition string '2026-27 PECTAA'; supersedes the NC-2006 class-10 book"), "Urdu|10": ("PECTAA 2026", "2026", "Publisher edition string 'PECTAA 2026'"), "Biology|10": ("PECTAA 2026", "2026", "Publisher edition string 'PECTAA 2026'"), "Chemistry|10": ("PECTAA 2026", "2026", "Publisher edition string 'PECTAA 2026'"), "Physics|10": ("PECTAA 2026", "2026", "Publisher edition string 'PECTAA 2026'"), "Islamiyat|10": ("SNC 2023-24", "2023-24", "Filename 'Islamiat 10 SNC 2023-24'; no 2026 edition exists on any mirror"), "*": ("SNC 2022", "2022-23", "Single National Curriculum 2022, Experimental Edition"), }, "STBB": { "Social Studies|6": ("Sindh Curriculum 2015", "2016", "Approved 7 Dec 2016, letter SO(G-1) E&L/CURRICULUM-2014"), "Social Studies|7": ("Sindh Curriculum 2015", "2017", "Approved 6 Dec 2017, letter SO(G.J) E&L Curriculum 2014"), "Social Studies|8": ("Federal legacy (Middle School Project)", "~2020-21 reprint", "Approved by Ministry of Education Curriculum Wing, Islamabad; translated from Urdu"), "Pakistan Studies|9": ("NC 2006 / 2004 approval", "2020-21", "Combined IX-X volume; approved 2004"), "Pakistan Studies|10": ("NC 2006 / 2004 approval", "2023-24", "Combined IX-X volume; approved 2004"), "*": ("National Curriculum 2006", "portal-listed", "STBB portal edition year"), }, # Balochistan publishes the federal NCC "Inclusive Scheme of Studies, Grade Pre I-XII, 2024" # as its own governing document — a generation PAST the SNC-2022/NCP-2023 label. Evidence: the # board's own catalogue hosts that scheme (id 464); its title page names the NCC wing of the # Ministry of Federal Education, and it contains zero occurrences of "2006", "SNC" or "NCP". # Caveat: this is the scheme the board hosts, not an imprint page read from each textbook. # Historical Punjab Textbook Board editions. Vintage is the printed year where the filename # carries one; otherwise the imprint has not been read and it is recorded as undated. "PTB-H": {"*": ("Historical PTB edition", "see year", "Pre-2006 curriculum era; imprint year from the scan where recorded")}, # Corrected from the books' own front matter. The board hosts the federal "NCC Inclusive Scheme # of Studies 2024" on its website and the catalogue previously recorded that as the vintage for # every volume — but the phrase appears in none of the 28 books. Balochistan runs two generations # side by side, which makes it the only board supplying vintage-matched partners for both. # The federal board is the baseline the provinces are meant to follow, which makes every # provincial divergence measurable rather than assumed. Its books are National Book Foundation # printings under the Single National Curriculum; per-book imprints are not yet read. "FBISE": {"*": ("Single National Curriculum (federal)", "NBF printing", "Federal Board / National Book Foundation; SNC-era. Per-book imprint pages not yet verified")}, "BTBB": {"|10": ("National Curriculum 2006", "NC 2006", "Class-10 front matter cites National Curriculum 2006 (Chemistry 10, English 10, Pakistan Studies 10)"), "*": ("National Curriculum 2022-23", "NC 2022-23 pilot", "Front matter: 'developed ... based on National Curriculum of Pakistan 2022-23 and is being printed as Pilot Edition for a period of one academic year'")}, "KPTBB": { "History|8": ("SNC / NCP 2022", "2022-23", "DCTE KP letter 3427-30/SNC-2022History(U)/VIII dated 16-07-2022; (c) National Book Foundation"), "Geography|8": ("SNC / NCP 2022", "2022-23", "DCTE KP approval dated 16-07-2022; (c) National Book Foundation"), "Urdu|8": ("SNC / NCP 2022", "2022-23", "DCTE KP letter 2413-17/SNC/2022/Urdu dated 12-05-2022"), "Islamiyat|8": ("SNC / NCP 2022", "2022-23", "NOC 2433-37/SNC-2022/Islamiat dated 12-05-2022"), "Pakistan Studies|9": ("National Curriculum 2006", "printed 2024-25", "Federal Curriculum Wing letter F.6-8/2009-SSG dated 28-05-2010 — print run, not a new edition"), "Pakistan Studies|10": ("National Curriculum 2006", "2022-23", "NOC F.8-5/2011 SSG dated 4 April 2011"), "Biology|9": ("National Curriculum 2006", "printed 2025-26", "DCTE NOC 1304-06/F.03/Vol-I dated 23-02-2018 — print run, not a new edition"), "Chemistry|9": ("National Curriculum 2006", "2019-20", "DCTE NOC 1180-82/F.03/Vol-I dated 15-02-2018"), "History|6": ("National Curriculum 2006", "2021-22", "Letter F.3-1/2009-History dated 7 December 2010"), "Geography|7": ("National Curriculum 2006", "2020-21", "NOC F.1-18/2011-SSG dated 5 April 2011"), "English|10": ("National Curriculum 2006", "Test Edition, scan 2021", "Cover reads 'Test Edition'; no approval date published by any mirror"), "Urdu|10": ("National Curriculum 2006", "scan 2021", "No approval date published by any mirror"), "Biology|10": ("National Curriculum 2006", "scan 2021", "No approval date published by any mirror"), "Chemistry|10": ("National Curriculum 2006", "'New Edition', scan 2021", "Unqualified 'New Edition' string; no approval date published"), "Physics|10": ("National Curriculum 2006", "scan 2021", "No approval date published by any mirror"), "*": ("National Curriculum 2006", "2006-curriculum reprint", "DCTE Abbottabad, NC-2006 series"), }, } _SHA_CACHE = os.path.join(A, ".sha-cache.json") if 'A' in dir() else None def _sha256(path, _cache={}): """SHA-256 of the analysed file, so a reader can prove they hold the same bytes we did. Cached on (path, size, mtime) — hashing ~7 GB on every build would be absurd.""" if not _cache: try: _cache.update(json.load(open(os.path.join(A, ".sha-cache.json")))) except Exception: pass try: st = os.stat(path) except OSError: return None key = "%s|%d|%d" % (path, st.st_size, int(st.st_mtime)) if key in _cache: return _cache[key] h = hashlib.sha256() with open(path, "rb") as fh: for chunk in iter(lambda: fh.read(1 << 20), b""): h.update(chunk) _cache[key] = h.hexdigest() json.dump(_cache, open(os.path.join(A, ".sha-cache.json"), "w")) return _cache[key] def _acquired(path): try: return datetime.date.fromtimestamp(os.path.getmtime(path)).isoformat() except OSError: return None # Three kinds of evidence that must never render alike. The editorial guideline insists a # negative result is a result, and an absence claim is the weakest thing this project asserts # — so it has to announce itself rather than sit in the same typography as a quotation. # quotation — verbatim wording. English-medium only, ever. # observation — structural or thematic. The ONLY permitted kind for Urdu-tier material. # absence — a negative result: a probe was run and returned nothing. _ABSENCE = re.compile( r"\b(zero|no recoverable|not established|returns? 0\b|0 (?:hits|occurrences)|" r"none found|no named|not one|no instance|absent|no occurrences|nothing but)\b", re.I) _QUOTE_CHARS = '\u201c\u201d\u2018\u2019"\'' _QUOTED = re.compile('[%s][^%s]{25,}[%s]' % (_QUOTE_CHARS, _QUOTE_CHARS, _QUOTE_CHARS)) _PAGE = re.compile(r"\bp{1,2}\.?\s?(\d{1,3})(?:\s*[-\u2013]\s*\d{1,3})?\b|PAGE\s+(\d{1,3})", re.I) def _evidence_kind(text, medium): """Urdu/Sindhi can never yield a quotation under this project's evidence rules, so the medium check comes first and overrides everything else.""" if _ABSENCE.search(text): return "absence" if medium != "English": return "observation" return "quotation" if _QUOTED.search(text) else "observation" def _evidence_page(text): m = _PAGE.search(text) return int(m.group(1) or m.group(2)) if m else None def vintage_for(board, subject, klass): """Most specific rule wins: subject+class, then a class-wide rule, then the board default. The class-wide form exists because Balochistan runs class 10 on a different curriculum generation from classes 6-9, across every subject at that class.""" tbl = VINTAGE[board] return (tbl.get(f"{subject}|{klass}") or tbl.get(f"|{klass}") or tbl["*"]) # ---------- 1. spine: the manifest ---------- TIER = _density_tiers(os.path.join(A, "manifest.csv")) books = {} with open(os.path.join(A, "manifest.csv")) as fh: for row in csv.DictReader(fh): # other/ holds the scholarly library and stray unattributed files, not board textbooks. # It is catalogued separately as a bibliography, never as a corpus book. if row["board"] not in BOARD_DIR: continue board = BOARD_DIR[row["board"]] klass = row["class"].replace("class-", "") stem = row["file"][:-4] year = None m = re.match(r"^(.*?)-(\d+(?:-\d+)?)-((?:19|20)\d{2})-(en|ur|si|un)$", stem) if m: subj_key, klabel, year, med = m.groups() else: m = re.match(r"^(.*?)-(\d+(?:-\d+)?)-(en|ur|si|un)$", stem) if not m: print(" ! unparsed filename:", stem); continue subj_key, klabel, med = m.groups() subject = SUBJ.get(subj_key, subj_key.replace("-", " ").title()) medium = {"en": "English", "ur": "Urdu", "si": "Sindhi", "un": "Unknown"}[med] pages = row["pages"] key = f"{board}|{klabel}|{subject}|{medium}" books[key] = { "id": f"{board.lower()}-{stem}", "board": board, "boardName": BOARD_NAME[board], "boardFull": BOARD_FULL[board], "class": klabel, "classSort": int(klabel.split("-")[0]), "subject": subject, "medium": medium, "pages": int(pages) if pages.isdigit() else None, "sizeMB": round(int(row["size_bytes"]) / 1048576, 1), "textTier": TIER.get(f"{row['board']}/{row['class']}/{row['file']}", {"yes": "native", "partial": "partial", "no": "scanned"}[row["text_extractable"]]), "producer": (row["producer"] or "").strip('"') or None, "file": f"{row['board']}/{row['class']}/{row['file']}", "year": year, # Date this copy was fetched to disk. Not the same as a formal archival capture — # it is the local acquisition date, and it is labelled as such on the site. Recorded # because a bare URL with no date is unusable as provenance once the URL rots. "acquired": _acquired(os.path.join(ROOT, row["board"], row["class"], row["file"])), "sha256": _sha256(os.path.join(ROOT, row["board"], row["class"], row["file"])), "title": None, "sourceUrl": None, "scores": {}, "evidence": [], } # ---------- 2. join sources ---------- # Sources files were written by different passes and do not agree on board codes: the historical # set says "PTB-historic" where the catalogue says "PTB-H". That mismatch silently cost 27 books # their title and their source URL — the join just never fired. Normalise before matching. SRC_BOARD_ALIAS = { "PTB-HISTORIC": "PTB-H", "PTB HISTORIC": "PTB-H", "PTBH": "PTB-H", "WEST-PAKISTAN-ERA (BOARD UNCONFIRMED)": "PTB-H", "PCTB-PUNJAB": "PCTB", "STBB-SINDH": "STBB", "KPTBB-KP": "KPTBB", "BTBB-BALOCHISTAN": "BTBB", "FBISE-FEDERAL": "FBISE", "FEDERAL": "FBISE", } def _src_board(raw): code = (raw or "").strip() return SRC_BOARD_ALIAS.get(code.upper(), code) def _subj_key(x): """Subjects for historical books are derived from the filename, so they read 'History Tarikh E Islam' where the sources file writes 'History (Tarikh-e-Islam)'. Compare on letters only.""" return re.sub(r"[^a-z0-9]", "", (x or "").lower()) matched = 0 for f in glob.glob(os.path.join(A, "sources-*.json")): for r in json.load(open(f)): r = dict(r, board=_src_board(r.get("board"))) k = f"{r['board']}|{r['class']}|{r['subject']}|{r['medium']}" if k in books: books[k]["title"] = r.get("title") books[k]["sourceUrl"] = r.get("url") matched += 1 else: # medium may have been re-detected after download cands = [kk for kk in books if kk.startswith(f"{r['board']}|{r['class']}|{r['subject']}|")] if not cands: # fall back to a punctuation-insensitive subject comparison want = (r["board"], str(r["class"]), _subj_key(r["subject"])) cands = [kk for kk, bb in books.items() if (bb["board"], str(bb["class"]), _subj_key(bb["subject"])) == want] if len(cands) == 1: books[cands[0]]["title"] = r.get("title") books[cands[0]]["sourceUrl"] = r.get("url") matched += 1 # ---------- 3. vintage ---------- for b in books.values(): curric, ed, note = vintage_for(b["board"], b["subject"], b["class"]) b["curriculum"] = curric b["edition"] = ed b["editionNote"] = note # ---------- 4. lens scores ---------- AXIS = {"religion": "A1", "civics": "A2", "history": ("A3", "A4"), "patriotism": "A5", "othering": "A6", "gender": "A7"} BOARD_PAT = re.compile(r"(PCTB|STBB|KPTBB)|Punjab|Sindh|Khyber|\bKP\b") def board_from(text): t = text.upper() if "PCTB" in t or "PUNJAB" in t: return "PCTB" if "STBB" in t or "SINDH" in t: return "STBB" if "KPTBB" in t or "KHYBER" in t or re.search(r"\bKP\b", t): return "KPTBB" if "BTBB" in t or "BALOCHISTAN" in t: return "BTBB" if "FBISE" in t or "FEDERAL" in t or "NBF" in t: return "FBISE" return None def cells(line): return [c.strip() for c in line.strip().strip("|").split("|")] def num(s): m = re.search(r"(\d)", re.sub(r"[^\d]", " ", s.replace("**", ""))) return int(m.group(1)) if m and 1 <= int(m.group(1)) <= 5 else None score_hits = collections.Counter() for lens, axis in AXIS.items(): path = os.path.join(A, f"lens-{lens}.md") if not os.path.exists(path): continue cur_board = None hdr = None for line in open(path, encoding="utf-8"): s = line.strip() if s.startswith("#"): b = board_from(s) if b: cur_board = b hdr = None continue if not s.startswith("|"): continue c = cells(s) low = [x.lower() for x in c] if "class" in low and ("subject" in low or "book" in low): hdr = low; continue if set(c) <= {"---", ":---", "---:", ""}: continue if not hdr: continue rec = dict(zip(hdr, c)) b = board_from(rec.get("board", "")) or cur_board if not b: continue klass = re.sub(r"[^\d\-]", "", rec.get("class", "")) subj = re.sub(r"[‡*]", "", rec.get("subject", rec.get("book", ""))).strip() subj = re.sub(r"\s*\(.*?\)", "", subj).strip() # "Book" columns embed the class, e.g. "Pakistan Studies 9", "History 6" -> strip it subj = re.sub(r"\s+\d+(?:\s*[-–]\s*\d+)?$", "", subj).strip() if not klass or not subj: continue # a lens row may cite "9" or "10" for a book the board publishes as a combined # "9-10" volume (Islamiyat, Urdu, Pakistan Studies) — accept those too def _klass_ok(bookklass): if bookklass == klass: return True parts = bookklass.split("-") return len(parts) > 1 and klass in parts med = (rec.get("medium") or rec.get("med") or "").strip().lower() def _pick(exact): out = [] for k in books: bb, bk, bs, _ = k.split("|") if bb != b or not _klass_ok(bk): continue if med and not books[k].get("medium", "").lower().startswith(med[:2]): continue if (bs.lower() == subj.lower()) if exact else (subj.lower() in bs.lower()): out.append(k) return out cands = _pick(True) or _pick(False) if len(cands) != 1: continue tgt = books[cands[0]] axes = axis if isinstance(axis, tuple) else (axis,) for ax in axes: for col in (ax.lower(), f"{ax.lower()} "): if col in rec: v = num(rec[col]) if v: tgt["scores"][ax] = v score_hits[ax] += 1 break if "a1b" in rec: v = num(rec["a1b"]) if v: tgt["scores"]["A1b"] = v score_hits["A1b"] += 1 # some religion-lens rows put "A1b 3" inside the A1 column for Islamiyat books a1cell = rec.get("a1", "") if "a1b" in a1cell.lower(): # "A1b 3" — strip the axis label first, or num() reads the 1 out of "A1b" # and every Islamiyat book in the corpus is scored 1 instead of its real value. v = num(re.sub(r"a1b", " ", a1cell, flags=re.I)) if v: tgt["scores"].pop("A1", None) tgt["scores"]["A1b"] = v score_hits["A1b"] += 1 ev = rec.get("evidence", "") or rec.get("key evidence", "") if ev and len(ev) > 40: scored_val = None for ax in (axis if isinstance(axis, tuple) else (axis,)): if ax in tgt["scores"]: scored_val = tgt["scores"][ax] # strip markdown emphasis/backticks — the site renders these as plain prose clean = re.sub(r"\*\*(.+?)\*\*", r"\1", ev.strip()) clean = re.sub(r"(?= 2} # the Social Studies structural split is a comparison even though the subject names differ for cls in ("6", "7", "8"): key = f"social-studies-split-{cls}" ids = [b["id"] for b in books.values() if b["board"] != "PTB-H" and b["class"] == cls and b["subject"] in ("Social Studies", "History", "Geography")] if ids: cmp_index[key] = {"subject": "Social Studies (split comparison)", "subjectSlug": "social-studies", "class": cls, "classSort": int(cls), "books": ids, "boards": sorted({books_by_id[i]["board"] for i in ids}), "commentary": cells.get(key)} json.dump(cmp_index, open(os.path.join(A, "comparisons.json"), "w"), indent=1, ensure_ascii=False) json.dump(subs, open(os.path.join(A, "summaries.json"), "w"), indent=1, ensure_ascii=False) print("comparison cells:", len(cmp_index), "| with commentary:", sum(1 for v in cmp_index.values() if v["commentary"])) # A cell without commentary used to render as "commentary pending" on a public page, which # reads as an unfinished site rather than as the missing paragraph it actually is. The page no # longer says anything in that case, so the omission has to be caught here instead. _nocomm = sorted(k for k, v in cmp_index.items() if not v["commentary"]) if _nocomm: print(f" WARNING: {len(_nocomm)} comparison cell(s) have no commentary and will render " f"with none: {', '.join(_nocomm)}") print("subject summaries:", len(subs.get("subjects", {})), "| overall:", bool(subs.get("overall"))) out = sorted(books.values(), key=lambda b: (b["board"], b["classSort"], b["subject"], b["medium"])) json.dump(out, open(os.path.join(A, "books.json"), "w"), indent=1, ensure_ascii=False) # ---------- report ---------- print(f"books: {len(out)} sources matched: {matched}") print("score coverage:", dict(score_hits)) missing_title = [b["id"] for b in out if not b["title"]] print(f"without title/url: {len(missing_title)}") by_axis = collections.Counter() for b in out: for ax in b["scores"]: by_axis[ax] += 1 print("books carrying each axis:", dict(sorted(by_axis.items()))) print("books with >=1 score:", sum(1 for b in out if b["scores"])) _GLUE = re.compile(r"\b[a-z]{2,}[A-Z][a-z]{2,}\b|\b\d{4}[a-z]{3,}\b") _glued = [] for _b in out: for _fld in ("note", "title"): _v = _b.get(_fld) or "" for _m in _GLUE.finditer(_v): _glued.append((_b["id"], _fld, _m.group(0))) for _e in _b.get("evidence", []): for _m in _GLUE.finditer(_e.get("text", "")): _glued.append((_b["id"], "evidence", _m.group(0))) if _glued: print("! WORDS RUN TOGETHER in prose (missing space?):") for _i, _f, _w in _glued[:20]: print(f" {_i} [{_f}] {_w}") print("books with evidence snippets:", sum(1 for b in out if b["evidence"])) print("books with analytical note:", sum(1 for b in out if b.get("note")))