cache_key = text.strip().lower()
if cache_key in _embedding_cache:
    return _embedding_cache[cache_key]

vec = np.array(_call_ollama_embed(text), dtype=np.float32)
# Normalize to unit length for cosine similarity
norm = np.linalg.norm(vec)
if norm > 0:
    vec = vec / norm
_embedding_cache[cache_key] = vec
return vec

if _index is not None:
    return _index

if os.path.exists(INDEX_PATH) and os.path.exists(ID_MAP_PATH):
    try:
        _index = faiss.read_index(INDEX_PATH)
        with open(ID_MAP_PATH, "r") as f:
            _id_map = json.load(f)
        _rev_id_map = {v: k for k, v in _id_map.items()}
        logger.info(f"Loaded FAISS index with {_index.ntotal} vectors")
        return _index
    except Exception as e:
        logger.warning(f"Failed to load FAISS index, creating new: {e}")

# Create new index — use inner product (cosine with normalized vectors)
dim = _get_embedding_dim()
_index = faiss.IndexFlatIP(dim)
_id_map = {}
_rev_id_map = {}
return _index

conn = get_connection()
rows = conn.execute(
    "SELECT id, subject, description FROM entities"
).fetchall()

dim = _get_embedding_dim()
_index = faiss.IndexFlatIP(dim)
_id_map = {}
_rev_id_map = {}

embeddings = []
ids = []
for row in rows:
    text = f"{row['subject']}: {row['description']}"
    vec = embed_text(text)
    embeddings.append(vec)
    ids.append(row["id"])

if embeddings:
    mat = np.array(embeddings, dtype=np.float32)
    _index.add(mat)
    _id_map = {eid: i for i, eid in enumerate(ids)}
    _rev_id_map = {i: eid for i, eid in enumerate(ids)}

_save_index()
logger.info(f"Rebuilt FAISS index with {len(ids)} vectors")

Validates type, normalizes subject, checks for duplicates via
semantic similarity + same type.
"""
validate_entity_type(type_)
validate_source(source)
validate_confidence(confidence)

subject = subject.strip()
if not subject:
    raise ValidationError("Subject must not be empty")

aliases = aliases or []
tags = tags or []
norm = normalize_subject(subject)

now = _now()
entity_id = _uuid()

with transaction() as conn:
    # Check for existing entity with same normalized subject + type
    existing = conn.execute(
        "SELECT id, subject, description, aliases, tags FROM entities WHERE type = ?",
        (type_,),
    ).fetchall()

    for ex in existing:
        if normalize_subject(ex["subject"]) == norm:
            # Same normalized subject + same type — definite duplicate
            # Use semantic similarity to confirm if descriptions match, 
            # but always merge if the subjects are identical after normalization
            logger.info(
                f"Duplicate detected (same subject + type): "
                f"'{ex['subject']}' == '{subject}', merging aliases/tags"
            )
            # Merge: update aliases, tags, description
            existing_aliases = set(json.loads(ex["aliases"]))
            existing_tags = set(json.loads(ex["tags"]))
            merged_aliases = existing_aliases | {subject} | set(aliases)
            merged_tags = existing_tags | set(tags)

            # Prefer longer description
            merged_desc = ex["description"]
            if len(description) > len(merged_desc):
                merged_desc = description

            conn.execute(
                """UPDATE entities
                   SET aliases = ?, tags = ?, description = ?,
                       updated_at = ?, confidence = MAX(confidence, ?)
                   WHERE id = ?""",
                (
                    json.dumps(sorted(merged_aliases)),
                    json.dumps(sorted(merged_tags)),
                    merged_desc,
                    now,
                    confidence,
                    ex["id"],
                ),
            )
            _update_index(ex["id"], f"{ex['subject']}: {merged_desc}")
            return ex["id"]

    conn.execute(
        """INSERT INTO entities (id, type, subject, description, aliases, tags,
                                 confidence, source, created_at, updated_at)
           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
        (
            entity_id, type_, subject, description,
            json.dumps(aliases), json.dumps(tags),
            confidence, source, now, now,
        ),
    )

# Add to vector index
text = f"{subject}: {description}"
_add_to_index(entity_id, text)

return entity_id

Validates relation type (rejects vague types), checks both entities exist.
"""
validate_relation_type(relation_type)
validate_source(source)
validate_confidence(confidence)

now = _now()
relation_id = _uuid()

with transaction() as conn:
    # Verify both entities exist
    src = conn.execute("SELECT id FROM entities WHERE id = ?", (source_id,)).fetchone()
    if not src:
        raise ValidationError(f"Source entity {source_id} not found")
    tgt = conn.execute("SELECT id FROM entities WHERE id = ?", (target_id,)).fetchone()
    if not tgt:
        raise ValidationError(f"Target entity {target_id} not found")

    # Check for existing relation (unique constraint handles this)
    existing = conn.execute(
        """SELECT id FROM relations
           WHERE source_entity_id = ? AND target_entity_id = ? AND relation_type = ?""",
        (source_id, target_id, relation_type),
    ).fetchone()

    if existing:
        # Update confidence
        conn.execute(
            "UPDATE relations SET confidence = MAX(confidence, ?), updated_at = ? WHERE id = ?",
            (confidence, now, existing["id"]),
        )
        return existing["id"]

    conn.execute(
        """INSERT INTO relations (id, source_entity_id, target_entity_id,
                                  relation_type, confidence, source, created_at, updated_at)
           VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
        (relation_id, source_id, target_id, relation_type,
         confidence, source, now, now),
    )

return relation_id

entity = dict(row)
entity["aliases"] = json.loads(entity["aliases"])
entity["tags"] = json.loads(entity["tags"])

# Get all relations where this entity participates
out_relations = conn.execute(
    """SELECT r.*, e.subject AS target_subject, e.type AS target_type
       FROM relations r JOIN entities e ON r.target_entity_id = e.id
       WHERE r.source_entity_id = ?""",
    (entity_id,),
).fetchall()

in_relations = conn.execute(
    """SELECT r.*, e.subject AS source_subject, e.type AS source_type
       FROM relations r JOIN entities e ON r.source_entity_id = e.id
       WHERE r.target_entity_id = ?""",
    (entity_id,),
).fetchall()

entity["relations_out"] = [dict(r) for r in out_relations]
entity["relations_in"] = [dict(r) for r in in_relations]

return entity

index = _get_faiss_index()
if index.ntotal == 0:
    return []

query_vec = embed_text(query).reshape(1, -1)
k = min(top_k, index.ntotal)
scores, indices = index.search(query_vec, k)

conn = get_connection()
results = []
for score, idx in zip(scores[0], indices[0]):
    eid = _rev_id_map.get(int(idx))
    if not eid:
        continue
    row = conn.execute(
        "SELECT id, type, subject, description, confidence, source FROM entities WHERE id = ?",
        (eid,),
    ).fetchone()
    if row:
        result = dict(row)
        result["similarity"] = float(score)
        results.append(result)

return results

Returns dict with root, nodes (neighbors by ID), and edges.
"""
if depth < 1:
    depth = 1

entity = get_entity(entity_id)
if not entity:
    return None

visited = {entity_id}
frontier = [entity_id]
all_nodes = {entity_id: entity}
all_edges = []

for _ in range(depth):
    next_frontier = []
    for fid in frontier:
        node = get_entity(fid)
        if not node:
            continue
        for rel in node.get("relations_out", []):
            tid = rel["target_entity_id"]
            all_edges.append(rel)
            if tid not in visited:
                visited.add(tid)
                next_frontier.append(tid)
                neighbor = get_entity(tid)
                if neighbor:
                    all_nodes[tid] = neighbor
        for rel in node.get("relations_in", []):
            sid = rel["source_entity_id"]
            all_edges.append(rel)
            if sid not in visited:
                visited.add(sid)
                next_frontier.append(sid)
                neighbor = get_entity(sid)
                if neighbor:
                    all_nodes[sid] = neighbor
    frontier = next_frontier

return {
    "root": entity,
    "nodes": {k: v for k, v in all_nodes.items() if k != entity_id},
    "edges": all_edges,
}

    if not e1:
        raise ValidationError(f"Entity {entity_id_1} not found")
    if not e2:
        raise ValidationError(f"Entity {entity_id_2} not found")

    # Merge metadata
    e1_aliases = set(json.loads(e1["aliases"]))
    e2_aliases = set(json.loads(e2["aliases"]))
    merged_aliases = e1_aliases | e2_aliases | {e1["subject"], e2["subject"]}

    e1_tags = set(json.loads(e1["tags"]))
    e2_tags = set(json.loads(e2["tags"]))
    merged_tags = e1_tags | e2_tags

    # Prefer longer/more recent description
    merged_desc = e1["description"] if len(e1["description"]) >= len(e2["description"]) else e2["description"]
    merged_confidence = max(e1["confidence"], e2["confidence"])
    now = _now()

    conn.execute(
        """UPDATE entities
           SET aliases = ?, tags = ?, description = ?,
               confidence = ?, updated_at = ?
           WHERE id = ?""",
        (
            json.dumps(sorted(merged_aliases)),
            json.dumps(sorted(merged_tags)),
            merged_desc, merged_confidence, now, entity_id_1,
        ),
    )

    # Re-point relations: entity_2 as source → entity_1
    conn.execute(
        """UPDATE OR IGNORE relations
           SET source_entity_id = ?, updated_at = ?
           WHERE source_entity_id = ?""",
        (entity_id_1, now, entity_id_2),
    )

    # Re-point relations: entity_2 as target → entity_1
    conn.execute(
        """UPDATE OR IGNORE relations
           SET target_entity_id = ?, updated_at = ?
           WHERE target_entity_id = ?""",
        (entity_id_1, now, entity_id_2),
    )

    # Delete relations that are now self-loops (entity → itself)
    conn.execute(
        "DELETE FROM relations WHERE source_entity_id = ? AND target_entity_id = ?",
        (entity_id_1, entity_id_1),
    )

    # Delete entity 2
    conn.execute("DELETE FROM entities WHERE id = ?", (entity_id_2,))

# Rebuild index after merge
_rebuild_index()

return {"merged_into": entity_id_1, "deleted": entity_id_2}

with transaction() as conn:
    updated = conn.execute(
        """UPDATE relations
           SET confidence = MAX(0.0, confidence - ?),
               updated_at = ?
           WHERE source IN ('dream', 'extraction')
           AND created_at < ?
           AND confidence > 0""",
        (decay_rate, _now(), cutoff),
    ).rowcount

return {"decayed_count": updated}

candidates = []
for row in rows:
    t1 = set(json.loads(row["tags"]))
    t2 = set(json.loads(row["tags"]))
    common = t1 & t2
    if len(common) >= 2:
        candidates.append({
            "entity_id_1": row["e1_id"],
            "entity_id_2": row["e2_id"],
            "shared_tags": sorted(common),
        })

return {"candidates": candidates}

# Check Ollama connectivity
ollama_ok = False
try:
    import httpx
    resp = httpx.get(f"{OLLAMA_HOST}/api/tags", timeout=5)
    ollama_ok = resp.status_code == 200
except Exception:
    pass

return {
    "status": "ok",
    "database": {
        "path": DB_PATH,
        "entity_count": entity_count,
        "relation_count": relation_count,
    },
    "vector_index": {
        "size": index.ntotal if index else 0,
        "dimension": _get_embedding_dim(),
    },
    "embedding": {
        "model": EMBEDDING_MODEL,
        "host": OLLAMA_HOST,
        "reachable": ollama_ok,
    },
    "controlled_vocabulary": {
        "valid_relation_types": sorted(VALID_RELATION_TYPES),
        "entity_types": sorted(VALID_ENTITY_TYPES),
    },
}