"""
sources.py: sources that aren't ordinary web articles, for fetch_article.py.

Each adapter turns its source into plain article HTML (<article id="glance-body">
with h2/p/ul/blockquote/figure) plus metadata, so the rest of fetch_article.py
(images, maths, blocks) works unchanged.

  X / Twitter   long-form X Articles (and single posts) via the public FxTwitter API
  Bluesky       a post and the author's own replies continuing it as a thread, with
                pictures (alt text kept) and quoted posts, via the public AppView API
  PDF           text PDFs via PyMuPDF: paragraphs from layout, headings from font
                size, weight or centred numbering, running headers/footers and
                footnotes dropped, words hyphenated across lines rejoined,
                displayed formulas cropped from the page as images, and
                embedded figures placed in the text with their captions
  arXiv         the paper's HTML rendering (LaTeXML, with MathML) for the body and
                the abstract page for title/authors/date

Imported, not run directly; fetch_article.py's uv header supplies dependencies.
"""

import collections
import html
import json
import re
import unicodedata
import urllib.parse

X_STATUS = re.compile(r"https?://(?:www\.|mobile\.)?(?:x|twitter)\.com/([^/]+)/status/(\d+)")
BSKY_POST = re.compile(r"https?://(?:www\.)?bsky\.app/profile/([^/]+)/post/([\w]+)")
ARXIV = re.compile(r"https?://(?:www\.)?arxiv\.org/(?:abs|html|pdf)/(\d{4}\.\d{4,5})(v\d+)?")


def esc(text):
  return html.escape(text or "", quote=True)


def wrap(parts):
  return '<html><body><article id="glance-body">' + "".join(parts) + "</article></body></html>"


# --- X / Twitter -----------------------------------------------------------

def x_media_url(info):
  for key in ("original_img_url", "url", "media_url_https", "media_url"):
    if info.get(key):
      return info[key]
  return None


def x_source(url, http_get):
  """An X Article (Draft.js blocks) or, failing that, the post itself."""
  match = X_STATUS.match(url)
  user, status = match[1], match[2]
  data = json.loads(http_get(f"https://api.fxtwitter.com/{user}/status/{status}")[0])
  tweet = data.get("tweet") or {}
  author = tweet.get("author") or {}
  created = (tweet.get("created_at") or "")
  meta = {"author": author.get("name"), "site": "X", "link": tweet.get("url") or url, "subtitle": None}

  article = tweet.get("article")
  if not article or not (article.get("content") or {}).get("blocks"):
    # A plain post: its text, and any photos.
    parts = [f"<p>{esc(line)}</p>" for line in (tweet.get("text") or "").split("\n") if line.strip()]
    for photo in ((tweet.get("media") or {}).get("photos") or []):
      parts.append(f'<figure><img src="{esc(photo.get("url"))}" alt="{esc(photo.get("altText"))}"></figure>')
    meta["title"] = f"Post by {author.get('name') or user}"
    return wrap(parts), meta

  content = article["content"]
  entities = {str(e["key"]): e["value"] for e in content.get("entityMap", [])}
  media = {m.get("media_id"): m.get("media_info") or {} for m in article.get("media_entities") or []}
  parts, open_list = [], None
  for block in content["blocks"]:
    kind, text = block.get("type"), block.get("text") or ""
    list_tag = {"ordered-list-item": "ol", "unordered-list-item": "ul"}.get(kind)
    if open_list and list_tag != open_list:
      parts.append(f"</{open_list}>")
      open_list = None
    if list_tag:
      if not open_list:
        parts.append(f"<{list_tag}>")
        open_list = list_tag
      parts.append(f"<li>{esc(text)}</li>")
    elif kind in ("header-one", "header-two", "header-three"):
      parts.append(f"<h2>{esc(text)}</h2>")
    elif kind == "blockquote":
      parts.append(f"<blockquote>{esc(text)}</blockquote>")
    elif kind == "atomic":
      for rng in block.get("entityRanges", []):
        entity = entities.get(str(rng.get("key"))) or {}
        for item in (entity.get("data") or {}).get("mediaItems", []):
          info = media.get(item.get("mediaId")) or {}
          src = x_media_url(info)
          if src:
            parts.append(f'<figure><img src="{esc(src)}" alt="{esc(info.get("alt_text"))}"></figure>')
    elif text.strip():
      parts.append(f"<p>{esc(text)}</p>")
  if open_list:
    parts.append(f"</{open_list}>")

  cover = x_media_url((article.get("cover_media") or {}).get("media_info") or {})
  meta.update({"title": article.get("title") or "X Article", "date": (article.get("created_at") or created)[:10] or None,
               "cover_url": cover})
  return wrap(parts), meta


# --- Bluesky ---------------------------------------------------------------

def bsky_text(record):
  """A post's text as HTML paragraphs, with shortened links restored from its facets."""
  text = (record.get("text") or "").encode("utf-8")
  links = []
  for facet in record.get("facets") or []:
    for feature in facet.get("features") or []:
      if feature.get("$type", "").endswith("#link"):
        links.append((facet["index"]["byteStart"], facet["index"]["byteEnd"], feature.get("uri")))
  out, pos = "", 0
  for start, end, uri in sorted(links):
    if start < pos:
      continue
    out += esc(text[pos:start].decode("utf-8", "replace"))
    out += f'<a href="{esc(uri)}">{esc(text[start:end].decode("utf-8", "replace"))}</a>'
    pos = end
  out += esc(text[pos:].decode("utf-8", "replace"))
  return [f"<p>{line}</p>" for line in out.split("\n") if line.strip()]


def bsky_embed(embed):
  """Pictures (with their alt text) and quoted posts from a post's embed view."""
  parts, kind = [], (embed or {}).get("$type", "")
  if kind.startswith("app.bsky.embed.recordWithMedia"):
    return bsky_embed(embed.get("media")) + bsky_embed(embed.get("record"))
  if kind.startswith("app.bsky.embed.images"):
    for img in embed.get("images") or []:
      parts.append(f'<figure><img src="{esc(img.get("fullsize"))}" alt="{esc((img.get("alt") or "").strip())}"></figure>')
  elif kind.startswith("app.bsky.embed.video") and embed.get("thumbnail"):
    parts.append(f'<figure><img src="{esc(embed["thumbnail"])}" alt="{esc((embed.get("alt") or "").strip())}"></figure>')
  elif kind.startswith("app.bsky.embed.record"):
    quoted = embed.get("record") or {}
    value = quoted.get("value") or {}
    if value.get("text"):
      who = (quoted.get("author") or {}).get("displayName") or (quoted.get("author") or {}).get("handle")
      lines = bsky_text(value)
      parts.append(f"<blockquote><p>Quoting {esc(who)}:</p>{''.join(lines)}</blockquote>")
      for inner in quoted.get("embeds") or []:
        parts += bsky_embed(inner)
  return parts


def bsky_source(url, http_get):
  """A Bluesky post and the author's own replies that continue it as a thread."""
  match = BSKY_POST.match(url)

  def thread(uri):
    query = urllib.parse.urlencode({"uri": uri, "depth": 1000, "parentHeight": 0})
    data = json.loads(http_get(f"https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread?{query}")[0])
    return data.get("thread") or {}

  node = thread(f"at://{match[1]}/app.bsky.feed.post/{match[2]}")
  if "post" not in node:
    raise ValueError("Bluesky didn't return that post (deleted, or the account needs a sign-in to view)")
  first, author = node["post"], node["post"]["author"]
  parts, posts = [], 0
  while node:
    post = node["post"]
    if post.get("replyCount") and not node.get("replies"):
      node = thread(post["uri"])  # the API stops about ten replies deep; carry on from here
    parts += bsky_text(post.get("record") or {}) + bsky_embed(post.get("embed"))
    posts += 1
    # The thread goes on in the author's earliest reply to themselves.
    own = [r for r in node.get("replies") or [] if (r.get("post") or {}).get("author", {}).get("did") == author["did"]]
    node = min(own, key=lambda r: r["post"]["record"].get("createdAt", ""), default=None)
  name = author.get("displayName") or author.get("handle")
  meta = {"title": f"{'Thread' if posts > 1 else 'Post'} by {name}", "author": name, "site": "Bluesky",
          "subtitle": None, "link": url, "date": (first["record"].get("createdAt") or "")[:10] or None,
          "cover_url": None}
  return wrap(parts), meta


# --- PDF -------------------------------------------------------------------

def fix_accents(text):
  """TeX PDFs often store an accent as a separate character before its letter
  ("G ̈odel", "Poincar ́e"); put it back on the letter: Gödel, Poincaré."""
  text = re.sub(r"\s?([\u0300-\u036f\u00a8\u00b4\u02c6\u02dc\u00b8])(\w)",
                lambda m: m[2] + {"\u00a8": "\u0308", "\u00b4": "\u0301", "\u02c6": "\u0302",
                                  "\u02dc": "\u0303", "\u00b8": "\u0327"}.get(m[1], m[1]), text)
  return unicodedata.normalize("NFC", text)


SMALL_WORDS = {"a", "an", "and", "as", "at", "but", "by", "for", "in", "nor", "of", "on", "or", "the", "to", "vs", "with"}


def smart_title(text):
  """ON PROOF AND PROGRESS IN MATHEMATICS -> On Proof and Progress in Mathematics"""
  words = text.lower().split()
  return " ".join(w if i and w in SMALL_WORDS else w[:1].upper() + w[1:] for i, w in enumerate(words))


CAPTION = re.compile(r"^(Figure|Fig\.|Table|Exhibit|Chart)\s*\d+", re.I)


def pdf_figures(doc, lines):
  """Embedded images big enough to be figures, each with the "Figure N: …"
  caption nearest below (or above) it on the page. Vector-drawn charts
  aren't found; they have no image to crop."""
  import pymupdf

  figures = []
  for pno, page in enumerate(doc):
    page_lines = [l for l in lines if l["page"] == pno]
    captions = []
    for i, l in enumerate(page_lines):
      if CAPTION.match(l["text"]):
        group = [l]
        for nxt in page_lines[i + 1:]:  # caption text continues on following small lines
          if abs(nxt["size"] - l["size"]) < 0.3 and -l["size"] * 0.5 <= nxt["y0"] - group[-1]["y1"] < l["size"] * 1.6 and abs(nxt["x0"] - l["x0"]) < 60:
            group.append(nxt)
          else:
            break
        captions.append(group)
    seen = set()
    for info in page.get_image_info(xrefs=True):
      rect = pymupdf.Rect(info["bbox"]) & page.rect
      key = tuple(round(v) for v in rect)
      if rect.width < 80 or rect.height < 60 or key in seen:
        continue
      seen.add(key)
      best, distance = None, 1e9
      for group in captions:
        first = group[0]
        overlaps = first["x0"] < rect.x1 and first["x1"] > rect.x0
        gap = first["y0"] - rect.y1 if first["y0"] >= rect.y1 - 4 else rect.y0 - group[-1]["y1"]
        if overlaps and -4 <= gap < 60 and gap < distance:
          best, distance = group, gap
      if best:
        captions.remove(best)
      figures.append({"page": pno, "rect": rect, "x0": rect.x0, "x1": rect.x1, "y0": rect.y0, "y1": rect.y1,
                      "caption": " ".join(l["text"] for l in best) if best else "", "caption_lines": best or []})
  return figures


MATH_FONT = re.compile(r"CMMI|CMSY|CMEX|MSAM|MSBM|Symbol|Math|STIX", re.I)
BOLD_FONT = re.compile(r"Bold|BX|Black|Heavy|Semibold", re.I)


def pdf_source(data, url):
  """Rebuild paragraphs and headings from a text PDF's layout. Displayed
  formulas are cropped from the page as images, never read out."""
  import tempfile
  import pymupdf

  doc = pymupdf.open(stream=data, filetype="pdf")
  lines = []
  for pno, page in enumerate(doc):
    width = page.rect.width
    blocks = page.get_text("dict")["blocks"]
    # Two-column pages have several multi-line text blocks that end before mid-page.
    left_only = sum(1 for b in blocks if len(b.get("lines", [])) > 2 and b["bbox"][2] < width / 2 + 15)
    def reading_order(block):
      """Columns left to right, each top to bottom. Blocks spanning most of the
      page width (titles, banners) sort with the left column by height."""
      x0, y0, x1, _ = block["bbox"]
      column = 0 if left_only < 2 or (x1 - x0) > width * 0.6 or x0 < width / 2 - 20 else 1
      return (column, round(y0))
    for block in (sorted(blocks, key=reading_order) if left_only >= 2 else blocks):
      for line in block.get("lines", []):
        spans = [s for s in line["spans"] if s["text"].strip()]
        if not spans:
          continue
        top = max(s["size"] for s in spans)
        # Footnote markers are small raised digits glued to a word ("system.6"): leave them out.
        kept = [s for s in line["spans"] if not (s["text"].strip().isdigit() and s["size"] < top * 0.8)]
        text = fix_accents(unicodedata.normalize("NFKC", "".join(s["text"] for s in kept)).strip())
        if not text:
          continue
        chars = sum(len(s["text"]) for s in spans)
        x0, y0, x1, y1 = line["bbox"]
        lines.append({
          "page": pno, "text": text, "size": round(max(s["size"] for s in spans), 1),
          "x0": x0, "x1": x1, "y0": y0, "y1": y1, "width": page.rect.width, "height": page.rect.height,
          "bold": all(BOLD_FONT.search(s["font"]) or s["flags"] & 16 for s in spans),
          "maths": sum(len(s["text"]) for s in spans if MATH_FONT.search(s["font"])) / max(chars, 1),
        })
  if not lines:
    raise ValueError("this PDF has no text layer (a scan?); OCR isn't supported yet")

  body_size = collections.Counter(l["size"] for l in lines).most_common(1)[0][0]
  figures = pdf_figures(doc, lines)
  caption_lines = {id(l) for f in figures for l in f["caption_lines"]}
  lines = [l for l in lines if id(l) not in caption_lines]
  margin = collections.Counter(round(l["x0"]) for l in lines if l["size"] == body_size).most_common(1)[0][0]
  centred = lambda l: abs((l["x0"] + l["x1"]) / 2 - l["width"] / 2) < 25 and l["x0"] > margin + 8

  # Running heads/feet: lines that repeat (digits aside) near the page edges, and page numbers.
  edge = lambda l: l["y0"] < l["height"] * 0.1 or l["y1"] > l["height"] * 0.92
  repeats = collections.Counter(re.sub(r"\d+", "#", l["text"]) for l in lines if edge(l))
  def furniture(l):
    return (re.fullmatch(r"[\d\s]+", l["text"]) is not None
            or (edge(l) and repeats[re.sub(r"\d+", "#", l["text"])] >= 3)
            or (edge(l) and len(l["text"]) < 80 and re.match(r"^\d+\s+[A-Z .]+$|^[A-Z .]+\s+\d+$", l["text"]))
            or l["size"] < body_size * 0.85)  # footnotes, venue lines, small print
  first_page = [l for l in lines if l["page"] == 0]
  title_lines = [l for l in first_page if l["bold"] and l["size"] >= body_size and l["y0"] < l["height"] * 0.5][:3]
  if not title_lines:
    biggest = max(l["size"] for l in first_page)
    title_lines = [l for l in first_page if l["size"] == biggest and biggest > body_size * 1.15]
  title = " ".join(l["text"] for l in title_lines).strip() or None
  author = None
  if title_lines:
    after = [l for l in first_page if l["y0"] > title_lines[-1]["y1"] and l["y0"] < title_lines[-1]["y1"] + 60]
    names = [l for l in after if centred(l) and l["text"].isupper() and len(l["text"].split()) <= 6]
    if names:
      author = names[0]["text"].title()
      lines.remove(names[0])  # don't also read it as text
  lines = [l for l in lines if not furniture(l) and l not in title_lines]

  # Displayed formulas: lines indented well past a paragraph indent and set
  # partly in maths fonts; then the fragments around them (fraction parts,
  # delimiters) that sit within a line's height. Two passes, until stable.
  def display(l):
    return len(l["text"]) < 70 and l["maths"] >= 0.45 and l["x0"] > margin + 8
  for l in lines:
    l["display"] = display(l)
  changed = True
  while changed:
    changed = False
    marked = [l for l in lines if l["display"]]
    for l in lines:
      if not l["display"] and len(l["text"]) < 50 and (l["maths"] >= 0.1 or len(l["text"]) < 12) and l["x0"] > margin + 20 and any(
          m["page"] == l["page"] and min(abs(l["y0"] - m["y1"]), abs(m["y0"] - l["y1"]), 0 if l["y0"] < m["y1"] and m["y0"] < l["y1"] else 99) < body_size * 1.2
          for m in marked):
        l["display"] = changed = True

  shots = tempfile.mkdtemp(prefix="glance-pdf-")
  parts, para, prev, eq, last_heading = [], [], None, [], None

  def flush():
    if para:
      text = ""
      for piece in para:
        if text.endswith("-") and piece[:1].islower():
          first = piece.split()[0]
          if first in ("or", "and", "to") and not text[-2:-1].isalpha():
            text = text + " " + piece  # "15- or 20-page"
          elif text[-2:-1].isdigit() or "-" in first:
            text = text + piece        # "3-manifolds", "hard-to-learn": a real hyphen
          else:
            text = text[:-1] + piece   # a word hyphenated across lines
        else:
          text = (text + " " + piece).strip()
      parts.append(f"<p>{esc(text)}</p>")
      para.clear()

  def flush_equation():
    if eq:
      page = doc[eq[0]["page"]]
      rect = pymupdf.Rect(min(l["x0"] for l in eq) - 6, min(l["y0"] for l in eq) - 1,
                          max(l["x1"] for l in eq) + 6, max(l["y1"] for l in eq) + 2)
      path = f"{shots}/eq{len(parts):04}.png"
      page.get_pixmap(clip=rect, dpi=220).save(path)
      parts.append(f'<figure class="pdf-equation"><img src="file://{path}" alt=""></figure>')
      eq.clear()

  placed = {}
  for f in figures:
    after = next((i for i, l in enumerate(lines) if l["page"] == f["page"] and l["y0"] >= f["y1"] - 2
                  and l["x0"] < f["x1"] and l["x1"] > f["x0"]), None)
    if after is None:
      after = next((i + 1 for i in range(len(lines) - 1, -1, -1) if lines[i]["page"] <= f["page"]), 0)
    placed.setdefault(after, []).append(f)

  def emit_figures(i):
    for f in placed.pop(i, []):
      flush()
      path = f"{shots}/fig{len(parts):04}.png"
      doc[f["page"]].get_pixmap(clip=f["rect"], dpi=150).save(path)
      caption = f"<figcaption>{esc(f['caption'])}</figcaption>" if f["caption"] else ""
      parts.append(f'<figure><img src="file://{path}" alt="">{caption}</figure>')

  for index, l in enumerate(lines):
    emit_figures(index)
    words = len(l["text"].split())
    if l["display"]:
      if not eq:
        flush()
      if eq and (l["page"] != eq[-1]["page"] or l["y0"] - eq[-1]["y1"] > body_size * 1.5):
        flush_equation()
      eq.append(l)
      prev = None
      continue
    flush_equation()
    numbered = bool(re.fullmatch(r"\d+(\.\d+)*\.?\s+[A-Z].{0,90}", l["text"]))
    heading = words <= 14 and not l["text"].endswith((",", ";")) and (
      (numbered and (centred(l) or l["x0"] <= margin + 2))
      or (l["size"] > body_size * 1.12 and not l["text"].endswith(".")))  # bold alone is emphasis, not a heading
    if heading and l["size"] >= body_size:
      flush()
      if (last_heading and parts and parts[-1].startswith("<h2>") and last_heading["page"] == l["page"]
          and abs(last_heading["size"] - l["size"]) < 0.5 and -l["size"] * 0.5 <= l["y0"] - last_heading["y1"] < l["size"] * 0.8):
        # (Line boxes often overlap a little, so the gap can be slightly negative.)
        # A heading set over several lines: one heading, not several.
        joined = re.sub(r"</?h2>", "", parts.pop()) + " " + esc(l["text"])
        parts.append(f"<h2>{joined}</h2>")
      else:
        parts.append(f"<h2>{esc(l['text'])}</h2>")
      last_heading, prev = l, None
      continue
    last_heading = None
    new_para = (prev is None
                or (l["x0"] > margin + 4 and prev["x0"] <= margin + 4)                       # first-line indent
                or (l["page"] == prev["page"] and l["y0"] - prev["y1"] > prev["size"] * 0.9))  # a gap
    if prev is not None and l["page"] != prev["page"] and l["x0"] <= margin + 4:
      new_para = False  # a paragraph carried over a page break
    if new_para:
      flush()
    para.append(l["text"])
    prev = l
  flush_equation()
  flush()
  for i in sorted(placed):
    emit_figures(i)

  info = doc.metadata or {}
  meta = {"title": (smart_title(title) if title and title.isupper() else title) or info.get("title")
                   or urllib.parse.unquote(url.rstrip("/").split("/")[-1]),
          "author": author or info.get("author") or None, "subtitle": None, "site": None, "link": url,
          "cover_url": None, "date": None}
  return wrap(parts), meta


# --- arXiv -----------------------------------------------------------------

def arxiv_source(url, http_get, appendices=False):
  """The LaTeXML HTML rendering for the body, the abstract page for metadata."""
  from bs4 import BeautifulSoup

  match = ARXIV.match(url)
  paper = match[1] + (match[2] or "")
  abs_page = BeautifulSoup(http_get(f"https://arxiv.org/abs/{paper}")[0], "lxml")
  get = lambda name: [m["content"] for m in abs_page.find_all("meta", attrs={"name": name}) if m.get("content")]
  authors = [" ".join(reversed(a.split(", "))) if ", " in a else a for a in get("citation_author")]
  try:
    page, final = http_get(f"https://arxiv.org/html/{paper}")
  except Exception as err:
    raise ValueError(f"arXiv has no HTML version of this paper ({err}); pass its PDF link instead") from err
  doc = BeautifulSoup(page, "lxml")
  body = doc.select_one("article.ltx_document")
  if not body:
    raise ValueError("arXiv's HTML for this paper has no article body; pass its PDF link instead")
  for junk in body.select("h1.ltx_title_document, .ltx_authors, .ltx_dates, section.ltx_bibliography, "
                          ".ltx_note, cite.ltx_citemacro_citep, .ltx_page_footer, nav"):
    junk.decompose()
  if not appendices:
    for section in body.select("section.ltx_appendix"):
      section.decompose()
  # Numeric [12]-style citations read as noise; author-year \citet citations are prose, so they stay.
  for cite in body.select("cite.ltx_cite"):
    if re.fullmatch(r"[\[\(]?[\d,\s–-]+[\]\)]?", cite.get_text("", strip=True)):
      cite.decompose()
  for heading in body.select(".ltx_title"):
    tag = heading.select_one(".ltx_tag")
    if tag and not tag.get_text().endswith(" "):
      tag.append(" ")  # "1Introduction" -> "1 Introduction"
  date = (get("citation_date") or get("citation_online_date") or [""])[0].replace("/", "-")
  meta = {"title": (get("citation_title") or ["Untitled"])[0], "subtitle": None,
          "author": authors[0] if len(authors) == 1 else (", ".join(authors[:-1]) + " and " + authors[-1] if authors else None),
          "site": "arXiv", "date": date or None, "link": f"https://arxiv.org/abs/{paper}", "cover_url": None}
  # Relative figure paths ("2608.16578v2/main/…") resolve against /html/.
  return wrap([str(body)]), meta, final
