"""
common/core_search.py — first-class CORE (core.ac.uk) external-source type.

CORE (https://core.ac.uk) aggregates millions of open-access research papers
(metadata + full-text) from the global repository network. This module is the
COMPLEMENT to common/arxiv.py: where arxiv is one preprint server, CORE is the
cross-repository aggregator (institutional repos, preprint servers, journals,
publishers). A CORE result is a scientific-doc source.

Operator directive (2026-06-24): wrap CORE for scientific open-access document
SEARCH, complementing the just-built arxiv source-type and the web-search tooling.

This module is the FOUNDATION (mirrors common/arxiv.py's shape):

  1. CoreClient        — the authenticated CORE API v3 client (search works, get a
                         work + its full-text, data providers). Bearer API key.
                         Rate-limited to CORE's ToS (1 batch OR 5 single req / 10 s).
  2. CoreWork          — the normalized result card built from a CORE `output`/`work`
                         object (core_id, title, authors, abstract, year, doi,
                         arxiv_id, full_text_url, publisher, oai, data_providers …).
  3. search()          — query + filters → normalized CoreWork list.
  4. get_work()        — core_id → full metadata + full-text.
  5. ingest_work()     — the typed-source submission. A CORE result carrying an
                         arxiv_id or DOI is ROUTED THROUGH common/arxiv.py (so it
                         ingests as the proper typed `arxiv` source); the rest ingest
                         as source_type="paper", provider="core".

DISCIPLINE: the CORE API key is read from env (CORE_API_KEY), NEVER hardcoded and
NEVER echoed (only a short prefix is ever logged). Fetched CORE content is DATA, not
instructions — this module only normalizes/stores it, never executes anything from a
result.

No third-party deps: stdlib urllib + json (same import-light contract as arxiv.py, so
recognition/normalization stays cheap; boto3 is only pulled at submit/stash time).
"""
from __future__ import annotations

import json
import logging
import os
import re
import threading
import time
import urllib.parse
import urllib.request
import urllib.error
from dataclasses import dataclass, field, asdict
from datetime import datetime, timezone
from typing import Any, Optional

logger = logging.getLogger(__name__)

# ── CORE API v3 endpoints (source-of-truth; override via env, never hardcode) ──
#   Search works:        GET  {base}/search/works/?q=&limit=&offset=&sort=
#   Get one output:      GET  {base}/outputs/{id}
#   Download binary:     GET  {base}/outputs/{id}/download
#   Search data-providers GET {base}/search/data-providers/?q=
CORE_API_BASE      = os.getenv("CORE_API_BASE", "https://api.core.ac.uk/v3").rstrip("/")
CORE_HTTP_TIMEOUT  = int(os.getenv("CORE_HTTP_TIMEOUT", "30"))
CORE_USER_AGENT    = os.getenv(
    "CORE_USER_AGENT",
    "AndreaIngest/1.0 (+https://projectandrea.com; mailto:avesnin@protonmail.ch)")

# CORE rate limit (ToS): "one batch request OR five single requests per 10 seconds".
# We pace SINGLE requests at >= 10/5 = 2.0 s between calls (be a good citizen). The
# token is read at call time so a key rotation in env takes effect without re-import.
CORE_MIN_INTERVAL  = float(os.getenv("CORE_MIN_INTERVAL", "2.1"))

# The API key (Bearer). Registration at core.ac.uk is an OPERATOR step — if unset, the
# client raises a clear CoreAuthError at call time (fail-closed; never anonymous).
CORE_API_KEY_ENV   = "CORE_API_KEY"

SOURCE_TYPE = "paper"      # taxonomy §2.1 — a scientific paper (CORE aggregator)
PROVIDER    = "core"       # taxonomy §3a — origin = the CORE aggregator


# ── DOI recogniser (for the arxiv/DOI routing in §5) ──────────────────────────
# A DOI is "10." + a registrant + "/" + a suffix. We only need to extract one from a
# free-text/url field to route through the arxiv DOI path or tag the card.
_RE_DOI = re.compile(r"\b(10\.\d{4,9}/[-._;()/:A-Za-z0-9]+)", re.IGNORECASE)


def extract_doi(text: Optional[str]) -> Optional[str]:
    """Return the first DOI found in `text` (a bare doi, a doi.org URL, or a field),
    lower-cased and trailing-punctuation-trimmed, or None."""
    if not text:
        return None
    m = _RE_DOI.search(text)
    if not m:
        return None
    doi = m.group(1).rstrip(".,;)]>")
    return doi.lower()


# ════════════════════════════════════════════════════════════════════════════════
#  ERRORS
# ════════════════════════════════════════════════════════════════════════════════

class CoreError(Exception):
    """Base for CORE wrapper errors."""


class CoreAuthError(CoreError):
    """No CORE_API_KEY configured, or CORE rejected the key (401/403). Fail-closed:
    we NEVER fall back to an anonymous call."""


class CoreRateLimited(CoreError):
    """CORE returned 429 — caller should back off (Retry-After honoured by the client
    when present)."""


# ════════════════════════════════════════════════════════════════════════════════
#  2. THE NORMALIZED WORK CARD
# ════════════════════════════════════════════════════════════════════════════════

@dataclass
class CoreAuthor:
    name: str


@dataclass
class CoreWork:
    """The normalized card for one CORE work/output. Built from a CORE v3 `output`
    object; field names below are OUR canonical schema (a stable contract for
    downstream consumers), populated from CORE's keys in `from_api`.

    `core_id` is the natural key; `provider` is always "core". When `arxiv_id` or
    `doi` is present the ingestion router (§5) sends it through common/arxiv.py so it
    lands as the proper typed `arxiv` source instead of a generic paper."""
    core_id: str                                   # CORE output id (string)
    title: str = ""
    authors: list[CoreAuthor] = field(default_factory=list)
    abstract: str = ""
    year: Optional[int] = None                     # yearPublished
    published_date: Optional[str] = None           # publishedDate (ISO) when present
    doi: Optional[str] = None                       # normalized DOI
    arxiv_id: Optional[str] = None                  # CORE's arxivId, normalized via arxiv.py
    full_text_url: Optional[str] = None            # best downloadable full-text URL
    download_url: Optional[str] = None             # CORE downloadUrl (raw)
    full_text: Optional[str] = None                # inline full text (only on get_work)
    publisher: Optional[str] = None
    language: Optional[str] = None
    document_type: Optional[str] = None            # CORE documentType (research/thesis/…)
    oai: Optional[str] = None                       # OAI identifier
    data_providers: list[str] = field(default_factory=list)   # source repository names
    links: list[dict] = field(default_factory=list)            # raw CORE links[]
    core_url: str = ""                              # canonical core.ac.uk landing URL
    provider: str = PROVIDER
    fetched_at: str = ""

    # -- construction from a raw CORE v3 output object --
    @classmethod
    def from_api(cls, obj: dict) -> "CoreWork":
        """Normalize one CORE v3 `output`/`work` dict into a CoreWork. Tolerant of
        missing/renamed fields across CORE response shapes (search vs single-get)."""
        core_id = str(obj.get("id") or obj.get("coreId") or "").strip()

        authors = []
        for a in obj.get("authors") or []:
            if isinstance(a, dict):
                name = (a.get("name") or "").strip()
            else:
                name = str(a).strip()
            if name:
                authors.append(CoreAuthor(name=name))

        # data providers: list of {id, name?} OR list of ids — keep human-readable name.
        dps: list[str] = []
        for dp in obj.get("dataProviders") or []:
            if isinstance(dp, dict):
                nm = dp.get("name") or dp.get("id")
                if nm is not None:
                    dps.append(str(nm))
            elif dp is not None:
                dps.append(str(dp))

        # publisher may be a string OR nested under journals[].title — prefer publisher.
        publisher = obj.get("publisher")
        if not publisher:
            for j in obj.get("journals") or []:
                if isinstance(j, dict) and j.get("title"):
                    publisher = j["title"]
                    break

        # best full-text URL: explicit downloadUrl, else first sourceFulltextUrls.
        download_url = obj.get("downloadUrl")
        full_text_url = download_url
        if not full_text_url:
            sfu = obj.get("sourceFulltextUrls") or []
            if sfu:
                full_text_url = sfu[0]

        # DOI: prefer the explicit field; normalize. arxivId: normalize via arxiv.py.
        doi = extract_doi(obj.get("doi")) or extract_doi(obj.get("identifiers", {}).get("doi")
                                                         if isinstance(obj.get("identifiers"), dict) else None)
        arxiv_id = _normalize_arxiv_id(obj.get("arxivId"))

        year = obj.get("yearPublished")
        try:
            year = int(year) if year is not None else None
        except (TypeError, ValueError):
            year = None

        return cls(
            core_id=core_id,
            title=re.sub(r"\s+", " ", str(obj.get("title") or "")).strip(),
            authors=authors,
            abstract=re.sub(r"\s+", " ", str(obj.get("abstract") or "")).strip(),
            year=year,
            published_date=obj.get("publishedDate") or obj.get("depositedDate"),
            doi=doi,
            arxiv_id=arxiv_id,
            full_text_url=full_text_url,
            download_url=download_url,
            full_text=(obj.get("fullText") or None),
            publisher=publisher or None,
            language=_language_of(obj.get("language")),
            document_type=obj.get("documentType") or None,
            oai=obj.get("oai") or None,
            data_providers=dps,
            links=[l for l in (obj.get("links") or []) if isinstance(l, dict)],
            core_url=f"https://core.ac.uk/works/{core_id}" if core_id else "",
            provider=PROVIDER,
            fetched_at=datetime.now(timezone.utc).isoformat(),
        )

    def to_metadata(self, title_override: Optional[str] = None) -> dict[str, Any]:
        """Render the pipeline `metadata` dict. The pipeline keys (`title`,
        `source_type`, `provider`) sit at top level so storage/worker.py lands them on
        the source_documents row; the full card nests under `core`."""
        return {
            "title":       title_override or self.title,
            "source_type": SOURCE_TYPE,
            "provider":    PROVIDER,
            "core":        asdict(self),
        }


def _language_of(lang) -> Optional[str]:
    """CORE `language` is sometimes a string, sometimes {code, name}. Return a code."""
    if not lang:
        return None
    if isinstance(lang, dict):
        return lang.get("code") or lang.get("name")
    return str(lang)


def _normalize_arxiv_id(raw) -> Optional[str]:
    """Normalize a CORE arxivId via common/arxiv.py's recogniser so it matches our
    canonical id form (and so a value like a full arxiv URL is handled). Import is
    local to keep this module import-light when arxiv routing isn't needed."""
    if not raw:
        return None
    try:
        from common import arxiv as _arxiv
    except Exception:                                  # noqa: BLE001
        try:
            import arxiv as _arxiv                       # flat sys.path (tests/containers)
        except Exception:
            return str(raw).strip() or None
    rid = _arxiv.recognize(str(raw))
    return rid or (str(raw).strip() or None)


# ════════════════════════════════════════════════════════════════════════════════
#  1. THE AUTHENTICATED CLIENT (rate-limited, ToS-respecting, fail-closed)
# ════════════════════════════════════════════════════════════════════════════════

class CoreClient:
    """Authenticated CORE API v3 client. The API key is read from env (CORE_API_KEY)
    by default — NEVER hardcoded, NEVER echoed. A request paces itself to CORE's ToS
    (CORE_MIN_INTERVAL between single calls) and honours Retry-After on 429.

    Fail-closed: with no key configured, every network method raises CoreAuthError —
    we do NOT issue an anonymous call."""

    def __init__(self, api_key: Optional[str] = None, *, base: Optional[str] = None,
                 timeout: Optional[int] = None, min_interval: Optional[float] = None):
        self._api_key = api_key if api_key is not None else os.getenv(CORE_API_KEY_ENV, "")
        self.base = (base or CORE_API_BASE).rstrip("/")
        self.timeout = timeout or CORE_HTTP_TIMEOUT
        self.min_interval = min_interval if min_interval is not None else CORE_MIN_INTERVAL
        self._lock = threading.Lock()
        self._last_ts = 0.0

    # -- key handling: never echo the secret (prefix only for diagnostics) --
    @property
    def has_key(self) -> bool:
        return bool(self._api_key)

    def key_prefix(self) -> str:
        """A short, non-secret prefix for logs/health (NEVER the full key)."""
        return (self._api_key[:4] + "…") if self._api_key else "<unset>"

    def _require_key(self) -> str:
        if not self._api_key:
            raise CoreAuthError(
                f"{CORE_API_KEY_ENV} is not set — register at core.ac.uk to obtain an "
                "API key (operator step); the wrapper is fail-closed and will not call "
                "CORE anonymously.")
        return self._api_key

    def _throttle(self) -> None:
        with self._lock:
            wait = self.min_interval - (time.monotonic() - self._last_ts)
            if wait > 0:
                time.sleep(wait)
            self._last_ts = time.monotonic()

    def _get(self, path: str, params: Optional[dict] = None) -> dict:
        """Authenticated GET → parsed JSON. Raises CoreAuthError (401/403),
        CoreRateLimited (429), or CoreError for other HTTP failures."""
        key = self._require_key()
        url = f"{self.base}/{path.lstrip('/')}"
        if params:
            url += "?" + urllib.parse.urlencode(
                {k: v for k, v in params.items() if v is not None})
        req = urllib.request.Request(
            url,
            headers={
                "Authorization": f"Bearer {key}",
                "Accept": "application/json",
                "User-Agent": CORE_USER_AGENT,
            },
        )
        self._throttle()
        try:
            with urllib.request.urlopen(req, timeout=self.timeout) as resp:
                return json.loads(resp.read().decode("utf-8"))
        except urllib.error.HTTPError as e:
            if e.code in (401, 403):
                raise CoreAuthError(f"CORE rejected the API key (HTTP {e.code})") from e
            if e.code == 429:
                retry_after = e.headers.get("Retry-After") if e.headers else None
                raise CoreRateLimited(
                    f"CORE rate limit hit (HTTP 429"
                    + (f", Retry-After={retry_after}s" if retry_after else "") + ")") from e
            body = ""
            try:
                body = e.read().decode("utf-8", "replace")[:300]
            except Exception:                          # noqa: BLE001
                pass
            raise CoreError(f"CORE API error HTTP {e.code}: {body}") from e
        except urllib.error.URLError as e:
            raise CoreError(f"CORE API unreachable: {e}") from e

    # -- search works ----------------------------------------------------------
    def search_works(self, query: str, *, limit: int = 10, offset: int = 0,
                     sort: Optional[str] = None,
                     filters: Optional[dict] = None) -> tuple[list[CoreWork], int]:
        """Search CORE works. Returns (works, total_hits). `filters` are appended to
        the query in CORE's query-string syntax (e.g. {"yearPublished>=": 2020} →
        `yearPublished>=2020`, {"language.code": "en"} → `language.code:en`)."""
        q = query.strip()
        for f_key, f_val in (filters or {}).items():
            # CORE query syntax: `field:value` for equality, operators inline in the key.
            if any(op in f_key for op in (">=", "<=", ">", "<", "=")):
                q += f" AND {f_key}{f_val}"
            else:
                q += f' AND {f_key}:"{f_val}"' if isinstance(f_val, str) else f" AND {f_key}:{f_val}"
        data = self._get("search/works", {
            "q": q, "limit": max(1, min(int(limit), 100)),
            "offset": max(0, int(offset)), "sort": sort,
        })
        results = data.get("results") or []
        works = [CoreWork.from_api(r) for r in results if isinstance(r, dict)]
        total = int(data.get("totalHits") or data.get("total") or len(works))
        return works, total

    # -- get one work ----------------------------------------------------------
    def get_output(self, core_id: str) -> CoreWork:
        """Fetch a single CORE output by id → full metadata (+ inline fullText when
        CORE provides it). Raises LookupError if the id is unknown (404)."""
        cid = str(core_id).strip()
        if not re.fullmatch(r"[0-9A-Za-z._:-]+", cid):
            raise ValueError(f"invalid CORE id: {core_id!r}")
        try:
            obj = self._get(f"outputs/{urllib.parse.quote(cid, safe='')}")
        except CoreError as e:
            if isinstance(e, (CoreAuthError, CoreRateLimited)):
                raise
            # the _get HTTPError path raised CoreError on 404 too — surface as LookupError
            if "HTTP 404" in str(e):
                raise LookupError(f"CORE output not found: {core_id!r}") from e
            raise
        return CoreWork.from_api(obj)

    # -- data providers (discovery) -------------------------------------------
    def search_data_providers(self, query: str, *, limit: int = 10,
                              offset: int = 0) -> tuple[list[dict], int]:
        """Search CORE data providers (source repositories). Returns the raw provider
        dicts (name, type, OAI endpoint, location…) + total. Discovery surface — not
        on the ingest hot path."""
        data = self._get("search/data-providers", {
            "q": query.strip(), "limit": max(1, min(int(limit), 100)),
            "offset": max(0, int(offset))})
        results = [r for r in (data.get("results") or []) if isinstance(r, dict)]
        total = int(data.get("totalHits") or data.get("total") or len(results))
        return results, total


# ════════════════════════════════════════════════════════════════════════════════
#  3 & 4. MODULE-LEVEL CONVENIENCE — search() / get_work()
# ════════════════════════════════════════════════════════════════════════════════

_default_client: Optional[CoreClient] = None


def _client() -> CoreClient:
    """Lazily build (and reuse) a module-level client from env. The throttle state is
    shared, so all module-level calls respect ONE rate budget."""
    global _default_client
    if _default_client is None:
        _default_client = CoreClient()
    return _default_client


def search(query: str, *, limit: int = 10, offset: int = 0,
           sort: Optional[str] = None,
           filters: Optional[dict] = None) -> list[CoreWork]:
    """CORE search → normalized CoreWork list (title, authors, abstract, year, doi,
    arxiv_id if present, full_text_url, core_id, provider="core").

    `filters` example: {"yearPublished>=": 2020, "language.code": "en"}."""
    works, _total = _client().search_works(
        query, limit=limit, offset=offset, sort=sort, filters=filters)
    return works


def get_work(core_id: str) -> CoreWork:
    """core_id → full metadata + full-text (CoreWork with `full_text` populated when
    CORE provides it inline)."""
    return _client().get_output(core_id)


# ════════════════════════════════════════════════════════════════════════════════
#  5. TYPED-SOURCE SUBMISSION — provider="core", with arxiv/DOI routing
# ════════════════════════════════════════════════════════════════════════════════

def best_body(work: CoreWork) -> str:
    """Foundation body for ingestion when CORE has no inline full text: title +
    abstract (+ a provenance line). If `work.full_text` is present, that is used
    instead by ingest_work()."""
    parts = [work.title.strip(), "", work.abstract.strip()]
    prov = []
    if work.publisher:
        prov.append(f"Publisher: {work.publisher}")
    if work.data_providers:
        prov.append("Source repositories: " + ", ".join(work.data_providers[:5]))
    if work.doi:
        prov.append(f"DOI: {work.doi}")
    if prov:
        parts += [""] + prov
    return "\n".join(p for p in parts if p is not None).strip()


def ingest_work(
    work_or_id,
    context_uuid: str = "00000000-0000-0000-0000-000000000000",
    job_id: Optional[str] = None,
    prefer_arxiv: bool = True,
) -> dict[str, Any]:
    """Ingest one CORE work as a typed scientific-doc source.

    ROUTING (operator directive): a CORE result that carries an arxiv_id (or an
    arxiv DOI) is ROUTED THROUGH common/arxiv.py — it ingests as the proper typed
    `arxiv` source (full card, e-print/PDF stash, the arxiv extraction seam), NOT a
    generic paper. Everything else ingests as source_type="paper", provider="core".

    `work_or_id` may be a CoreWork (already fetched) or a core_id string (fetched
    here, pulling the full record incl. inline full-text when available)."""
    work = work_or_id if isinstance(work_or_id, CoreWork) else get_work(str(work_or_id))

    # -- arxiv routing: prefer the typed arxiv source when we can resolve an id --
    if prefer_arxiv:
        arxiv_id = work.arxiv_id
        if not arxiv_id and work.doi and work.doi.startswith("10.48550/arxiv."):
            arxiv_id = work.doi.split("10.48550/arxiv.", 1)[1]
        if arxiv_id:
            try:
                from common import arxiv as _arxiv
            except Exception:                          # noqa: BLE001
                import arxiv as _arxiv                   # flat sys.path
            logger.info("core→arxiv route: core_id=%s arxiv_id=%s", work.core_id, arxiv_id)
            res = _arxiv.ingest_arxiv(
                arxiv_id, context_uuid=context_uuid, job_id=job_id)
            res["routed_from"] = "core"
            res["core_id"] = work.core_id
            return res

    # -- generic paper path: source_type="paper", provider="core" --------------
    meta = work.to_metadata()
    text = work.full_text if work.full_text else best_body(work)

    try:
        from common.submit import submit as _pipeline_submit
    except Exception:                                  # noqa: BLE001 — flat sys.path
        from submit import submit as _pipeline_submit
    result = _pipeline_submit(
        content=text,
        title=work.title,
        source_type=SOURCE_TYPE,
        language=work.language or "en",
        context_uuid=context_uuid,
        metadata=meta,
        provider=PROVIDER,
        job_id=job_id,
    )
    result["core_id"] = work.core_id
    result["provider"] = PROVIDER
    return result


def route_core_id(
    text: str,
    context_uuid: str = "00000000-0000-0000-0000-000000000000",
    job_id: Optional[str] = None,
) -> Optional[dict[str, Any]]:
    """Recognition entry for a core.ac.uk URL / `core:<id>` in a fetch path. If `text`
    references a CORE work id, fetch + ingest it (with arxiv/DOI routing) and return
    the submit result; else None so the caller falls through to generic handling.

    Wire ABOVE the generic webpage handler (and BELOW arxiv.route_url, since an arxiv
    link should still take the arxiv path directly):

        res = arxiv.route_url(url, ctx) or core_search.route_core_id(url, ctx)
        if res is None:
            res = generic_webpage_ingest(url, ctx)
    """
    cid = recognize_core_id(text)
    if cid is None:
        return None
    logger.info("core router: %r → core_id=%s", (text or "")[:80], cid)
    return ingest_work(cid, context_uuid=context_uuid, job_id=job_id)


_RE_CORE_URL = re.compile(
    r"core\.ac\.uk/(?:works|download|outputs|reader|display)/(\d+)", re.IGNORECASE)
_RE_CORE_PREFIX = re.compile(r"\bcore:(\d+)\b", re.IGNORECASE)


def recognize_core_id(text: str) -> Optional[str]:
    """Return the CORE work id referenced in `text` (a core.ac.uk URL or `core:<id>`),
    or None. Bare integers are NOT matched (too ambiguous) — only the explicit shapes."""
    if not text:
        return None
    m = _RE_CORE_URL.search(text)
    if m:
        return m.group(1)
    m = _RE_CORE_PREFIX.search(text)
    if m:
        return m.group(1)
    return None
