"""
submit.py — v2 pipeline submission entry point.

Called by memory_server when POST /jobs/dissect arrives, and by
scripts/feed_docstore_to_pipeline.py for the bulk docstore→pipeline feed.

Submits via HTTP POST to the pipeline dispatcher (Django+DRF+Celery,
session-46; replaces the broken n8n sub-workflow path). Contract is
unchanged: {doc_id, storage_key, context_uuid, metadata}.

Changes vs original:
  - doc_id comes from uuid_service (not uuid.uuid4().hex)
  - content_hash (SHA256) computed and stored for deduplication
  - Dedup check: if document with same hash exists in context → return existing record
  - Submits to pipeline-dispatcher webhook (not n8n, not Redis stream)

Dedup — submit-time atomic claim (infra#174, 2026-06-29):
  The original dedup checked ONLY ArangoDB `source_documents`, which the storage
  worker writes at STORE (stage 6/7 — the END of the pipeline). During a bulk feed
  (feed_docstore_to_pipeline loops submit() over a stdin batch) ALL identical docs
  are submitted within seconds, BEFORE the first traverses tokenize→…→store and
  writes its row → the dedup window is empty for the whole batch → every copy is
  treated as "new" → N doc_ids + N forward_pass tasks. Observed: openssl-doc shipped
  432 function-family man-pages with ONE identical body; 989 of 1481 FP-queue docs
  were exact content duplicates.
  Fix: an ATOMIC Redis `SET NX` claim keyed by (context_uuid, content_hash), written
  HERE at submit, catches within-batch + recent duplicates immediately — before the
  doc enters the pipeline. The ArangoDB `source_documents` check stays as the durable
  backstop (cross-restart / claim-evicted, doc already stored). Degrades gracefully:
  if Redis is unreachable the path falls back to Arango-only (== original behaviour).
"""
from __future__ import annotations

import hashlib
import json
import logging
import os
import sys
import urllib.request
import urllib.error
from datetime import datetime, timezone

sys.path.insert(0, os.path.dirname(__file__))
from common.pipeline import write_stage, write_meta, get_redis, ensure_streams

logger = logging.getLogger(__name__)

UUID_SERVICE_URL   = os.getenv("UUID_SERVICE_URL",   "http://10.121.1.50:8769")
UUID_SERVICE_TO    = int(os.getenv("UUID_SERVICE_TIMEOUT", "5"))
PIPELINE_URL       = os.getenv("PIPELINE_DISPATCHER_URL",
                                "http://10.121.1.1:8782/webhook/andrea-ingest")
ARANGO_URL         = os.getenv("ARANGO_URL",         "http://10.121.1.1:8529")
ARANGO_DB          = os.getenv("ARANGO_DB",          "memory_server")
ARANGO_USERNAME    = os.getenv("ARANGO_USERNAME",    "root")
ARANGO_PASSWORD    = os.getenv("ARANGO_PASSWORD",    "")

# Submit-time dedup claim (infra#174). The claim bridges submit→store; once a doc is
# stored the ArangoDB source_documents backstop is authoritative, so the claim only
# needs to outlive the worst-case submit→store latency. Default 30d matches the task
# TTL (spooler ATOM_TTL / dispatcher TASK_TTL_SECONDS). Live-tunable via env.
DEDUP_KEY_PREFIX   = os.getenv("PIPELINE_DEDUP_KEY_PREFIX", "pipeline:dedup")
DEDUP_TTL_SECONDS  = int(os.getenv("PIPELINE_DEDUP_TTL_SECONDS", "2592000"))   # 30d
_CLAIM_PENDING     = "PENDING"   # placeholder value held between claim and real doc_id


# ── UUID service ──────────────────────────────────────────────────────────────

def _new_uuid() -> str:
    """Get a collision-proof UUID4 from the uuid_service."""
    req = urllib.request.Request(f"{UUID_SERVICE_URL}/uuid")
    try:
        with urllib.request.urlopen(req, timeout=UUID_SERVICE_TO) as resp:
            return json.loads(resp.read())["uuid"]
    except Exception as exc:
        raise RuntimeError(f"uuid_service unreachable: {exc}") from exc


# ── Content hash ──────────────────────────────────────────────────────────────

def _content_hash(content: str) -> str:
    return hashlib.sha256(content.encode("utf-8")).hexdigest()


# ── Deduplication: durable backstop (ArangoDB source_documents) ───────────────

def _find_existing(content_hash: str, context_uuid: str) -> dict | None:
    """
    Check ArangoDB for an existing source_document with this hash.
    Checks the given context_uuid first, then the global context.
    Returns the existing doc dict, or None.

    NB: source_documents is written by the STORAGE worker at stage 6/7 — so this
    catches an already-STORED duplicate (a re-feed of a fully-processed doc), but
    NOT a within-batch duplicate still mid-pipeline. The Redis claim below closes
    that window; this remains the durable cross-restart backstop.
    """
    import base64
    creds = base64.b64encode(
        f"{ARANGO_USERNAME}:{ARANGO_PASSWORD}".encode()).decode()

    query = (
        "FOR d IN source_documents "
        "FILTER d.content_hash == @hash "
        "FILTER d.context_uuid IN [@ctx, '00000000-0000-0000-0000-000000000000'] "
        "LIMIT 1 RETURN d"
    )
    payload = json.dumps({
        "query": query,
        "bindVars": {"hash": content_hash, "ctx": context_uuid},
    }).encode()

    req = urllib.request.Request(
        f"{ARANGO_URL}/_db/{ARANGO_DB}/_api/cursor",
        data=payload,
        headers={
            "Content-Type": "application/json",
            "Authorization": f"Basic {creds}",
        },
        method="POST",
    )
    try:
        with urllib.request.urlopen(req, timeout=10) as resp:
            result = json.loads(resp.read()).get("result", [])
            return result[0] if result else None
    except Exception as exc:
        logger.warning("Dedup check failed: %s — proceeding without dedup", exc)
        return None


# ── Deduplication: submit-time atomic claim (Redis SET NX) ────────────────────

def _claim_key(context_uuid: str, content_hash: str) -> str:
    return f"{DEDUP_KEY_PREFIX}:{context_uuid}:{content_hash}"


def _try_claim(context_uuid: str, content_hash: str) -> tuple[str, str | None]:
    """Atomically claim this (context, content_hash) at submit time.

    Returns one of:
      ("hit", canonical_doc_id_or_empty) — a prior submit already claimed this hash;
                                           this content is a DUPLICATE. canonical is the
                                           winning doc_id ("" if it is still PENDING).
      ("held", None)                     — WE won the claim; proceed to ingest.
      ("nored", None)                    — Redis unreachable; fall back to Arango-only
                                           (original behaviour — no within-batch dedup).
    """
    try:
        r = get_redis()
    except Exception as exc:
        logger.warning("dedup: redis unavailable (%s) — claim skipped, Arango-only", exc)
        return ("nored", None)
    key = _claim_key(context_uuid, content_hash)
    try:
        if r.set(key, _CLAIM_PENDING, nx=True, ex=DEDUP_TTL_SECONDS):
            return ("held", None)
        canonical = r.get(key) or ""
        canonical = "" if canonical == _CLAIM_PENDING else canonical
        logger.info("dedup CLAIM hit: hash=%s canonical=%s",
                    content_hash[:16], (canonical or "pending")[:12])
        return ("hit", canonical)
    except Exception as exc:
        logger.warning("dedup: claim op failed (%s) — Arango-only", exc)
        return ("nored", None)


def _set_claim(context_uuid: str, content_hash: str, doc_id: str) -> None:
    """Stamp the claim with the canonical doc_id (upgrades PENDING → real id) so
    within-batch siblings dedup straight to the surviving document. Best-effort."""
    try:
        get_redis().set(_claim_key(context_uuid, content_hash), doc_id or _CLAIM_PENDING,
                        ex=DEDUP_TTL_SECONDS)
    except Exception as exc:
        logger.warning("dedup: set_claim failed (%s) — claim left PENDING", exc)


def _release_claim(context_uuid: str, content_hash: str) -> None:
    """Release a held claim so a retry can re-ingest — used when ingest fails AFTER
    we won the claim (else the content would be wedged as a phantom duplicate). Best-effort."""
    try:
        get_redis().delete(_claim_key(context_uuid, content_hash))
    except Exception as exc:
        logger.warning("dedup: release_claim failed (%s)", exc)


def _duplicate_result(content_hash: str, canonical_doc_id: str | None,
                      job_id: str | None, source_doc_id: str | None = None) -> dict:
    res = {
        "doc_id":       canonical_doc_id or "",
        "job_id":       job_id,
        "status":       "duplicate",
        "pipeline":     "v2",
        "content_hash": content_hash,
        "message":      "Document already ingested — returning existing record",
    }
    if source_doc_id:
        res["source_doc_id"] = source_doc_id
    return res


# ── Pipeline submission ──────────────────────────────────────────────────────

def _submit_to_pipeline(doc_id: str, storage_key: str,
                        context_uuid: str, metadata: dict) -> None:
    payload = json.dumps({
        "doc_id":       doc_id,
        "storage_key":  storage_key,
        "context_uuid": context_uuid,
        "metadata":     metadata,
    }).encode()
    req = urllib.request.Request(
        PIPELINE_URL,
        data=payload,
        headers={"Content-Type": "application/json"},
        method="POST",
    )
    try:
        with urllib.request.urlopen(req, timeout=10) as resp:
            logger.info("dispatcher accepted doc=%s status=%d", doc_id[:8], resp.status)
    except urllib.error.HTTPError as exc:
        raise RuntimeError(
            f"dispatcher webhook error {exc.code}: {exc.read().decode()}") from exc
    except Exception as exc:
        raise RuntimeError(f"dispatcher webhook unreachable: {exc}") from exc


# ── Public API ────────────────────────────────────────────────────────────────

def submit(
    content: str,
    title: str = None,
    source_type: str = "document",
    language: str = "en",
    context_uuid: str = "00000000-0000-0000-0000-000000000000",
    metadata: dict = None,
    job_id: str = None,
    provider: str = None,
) -> dict:
    """
    Submit a document to the async v2 pipeline.

    Returns immediately with doc_id and dedup status.
    Actual processing happens asynchronously via the stage workers.

    `provider` is the typed external-source origin tag (taxonomy §3a) — e.g.
    "arxiv" for an arxiv paper. It rides the stage.raw payload AND metadata so the
    storage worker (sole writer of the source_documents row) lands it on the row.
    None (default) = untyped/legacy source, no provider written.
    """
    meta         = metadata or {}
    # provider may also arrive nested in metadata (e.g. ArxivCard.to_metadata());
    # the explicit arg wins, else fall back to the metadata key, else None.
    provider     = provider or meta.get("provider")
    if provider:
        meta.setdefault("provider", provider)
    content_hash = _content_hash(content)
    # BUG D fix (session-54): thread content_hash through `metadata` so it
    # survives to the storage worker. Root cause of empty
    # source_documents.content_hash: the dispatcher submit payload (see
    # _submit_to_pipeline) carries only {doc_id, storage_key, context_uuid,
    # metadata}; content_hash was written to stage.raw + meta.json but never
    # reached storage/worker.py, which defaulted it to "" — silently breaking
    # BOTH dedup paths (this front door + the n8n watcher). metadata DOES flow
    # intact through every stage (that's how `title` reaches storage), so this
    # is the minimal correct carrier.
    meta["content_hash"] = content_hash

    # ── Deduplication (infra#174) ─────────────────────────────────────────────
    # 1) Submit-time atomic claim — closes the bulk-feed within-batch race that the
    #    end-of-pipeline source_documents check cannot see.
    claim_state, canonical = _try_claim(context_uuid, content_hash)
    if claim_state == "hit":
        return _duplicate_result(content_hash, canonical, job_id)

    # 2) Durable backstop — an already-STORED doc with this hash (re-feed across a
    #    Redis restart / claim expiry). When we hold the claim, stamp it with the
    #    canonical so within-batch siblings dedup to the stored doc.
    existing = _find_existing(content_hash, context_uuid)
    if existing:
        canonical = existing.get("pipeline_doc_id", existing.get("_key"))
        logger.info("Dedup hit (stored): hash=%s existing_doc=%s",
                    content_hash[:16], existing.get("_key", "?"))
        if claim_state == "held":
            _set_claim(context_uuid, content_hash, canonical or "STORED")
        return _duplicate_result(content_hash, canonical, job_id,
                                 source_doc_id=existing.get("_key"))

    # ── New document — get UUID from uuid_service ─────────────────────────────
    doc_id = _new_uuid().replace("-", "")   # hex form for Garage path cleanliness
    if title:
        meta.setdefault("title", title)
    # Upgrade the claim PENDING → real doc_id so within-batch siblings dedup to it.
    if claim_state == "held":
        _set_claim(context_uuid, content_hash, doc_id)

    now = datetime.now(timezone.utc).isoformat()

    try:
        # ── Write stage.raw payload to Garage ────────────────────────────────
        raw_payload = {
            "content":      content,
            "title":        title,
            "source_type":  source_type,
            "provider":     provider,
            "language":     language,
            "metadata":     meta,
            "content_hash": content_hash,
        }
        raw_key = write_stage(context_uuid, doc_id, "raw", raw_payload)

        # ── Write permanent meta.json ─────────────────────────────────────────
        write_meta(context_uuid, doc_id, {
            "doc_id":           doc_id,
            "job_id":           job_id,
            "title":            title,
            "source_type":      source_type,
            "provider":         provider,
            "language":         language,
            "context_uuid":     context_uuid,
            "char_count":       len(content),
            "content_hash":     content_hash,
            "metadata":         meta,
            "submitted_at":     now,
            "pipeline_version": "2.0",
            "status":           "pending",
        })

        # ── Submit to pipeline-dispatcher ─────────────────────────────────────
        _submit_to_pipeline(doc_id, raw_key, context_uuid, meta)
    except Exception:
        # Ingest failed after we won the claim — release it so a retry re-ingests
        # instead of the content being wedged as a phantom duplicate forever.
        if claim_state == "held":
            _release_claim(context_uuid, content_hash)
        raise

    logger.info("Submitted doc_id=%s hash=%s chars=%d",
                doc_id[:8], content_hash[:16], len(content))

    return {
        "doc_id":       doc_id,
        "job_id":       job_id,
        "status":       "pending",
        "pipeline":     "v2",
        "content_hash": content_hash,
        "storage_path": f"pipeline/{context_uuid}/{doc_id}/",
    }
