Why AI Hallucinates: The Real Reason

import numpy as np

# Simplified illustration of next-token sampling
# The model outputs a logit score for each token in the vocabulary

vocabulary = ["Lincoln", "Bell", "Edison", "Tesla", "Morse"]
logits = np.array([2.1, 4.8, 1.3, 0.9, 0.7])  # raw scores before softmax

def softmax(logits, temperature=1.0):
    """Convert logits to probabilities with temperature scaling."""
    scaled = logits / temperature
    exp_scores = np.exp(scaled - np.max(scaled))  # numerical stability
    return exp_scores / exp_scores.sum()

# At temperature 1.0 — proportional sampling
probs_t1 = softmax(logits, temperature=1.0)
print("Temp 1.0:", dict(zip(vocabulary, probs_t1.round(3))))
# Bell gets ~83% probability — but other tokens still possible

# At temperature 0.1 — near-deterministic, top token almost always picked
probs_t01 = softmax(logits, temperature=0.1)
print("Temp 0.1:", dict(zip(vocabulary, probs_t01.round(3))))
# Bell gets ~99.9% probability

# At temperature 1.5 — more exploration, lower-probability tokens more likely
probs_t15 = softmax(logits, temperature=1.5)
print("Temp 1.5:", dict(zip(vocabulary, probs_t15.round(3))))
# Distribution flattens — Tesla or Morse more likely to be picked

import anthropic

client = anthropic.Anthropic()

def extract_from_source(source_text: str, question: str) -> str:
    """
    Extract information from provided text only.
    Uses explicit grounding instructions to prevent hallucination.
    """
    response = client.messages.create(
        model="claude-haiku-4-5",
        max_tokens=800,
        temperature=0,  # deterministic — reduces creative "gap-filling"
        system="""You are a precise information extractor.
Rules:
1. Answer ONLY using information explicitly stated in the provided source text.
2. If the answer is not in the source text, say: "This information is not in the provided text."
3. Do not infer, extrapolate, or add context from outside knowledge.
4. Quote directly when exact wording matters.
5. Never fabricate statistics, names, dates, or citations.""",
        messages=[{
            "role": "user",
            "content": f"""Source text:
---
{source_text}
---

Question: {question}"""
        }]
    )
    return response.content[0].text


# Example usage
report_excerpt = """
The Q3 2025 earnings report showed revenue of $4.2M, up 34% year-over-year.
Customer count reached 1,847, with enterprise customers (>$10k ACV) comprising 23% of
the total but 61% of revenue. Net Revenue Retention was 118%.
The company reported a net loss of $890k, improved from a $1.4M loss in Q3 2024.
"""

result = extract_from_source(
    report_excerpt,
    "What was the net revenue retention and how did the loss compare to last year?"
)
print(result)
# Output: NRR was 118%. Net loss was $890k in Q3 2025, compared to $1.4M in Q3 2024 — an improvement of $510k.

result2 = extract_from_source(
    report_excerpt,
    "What was the customer acquisition cost?"
)
print(result2)
# Output: "This information is not in the provided text." — correctly refuses to hallucinate