Semantic Caching

Example

A semantic cache with bag-of-words vectors and cosine similarity (a stand-in for a real embedding model).

import math, re
from collections import Counter
STOP = {"what", "is", "the", "for", "a", "how", "many", "do", "i", "to", "can", "my", "of", "in", "have"}

def embed(text):  # stand-in for an embedding model
    words = [w.rstrip("s") for w in re.findall(r"[a-z]+", text.lower()) if w not in STOP]
    return Counter(words)

def cosine(a, b):
    dot = sum(a[w] * b[w] for w in a)
    return dot / (math.sqrt(sum(v * v for v in a.values())) * math.sqrt(sum(v * v for v in b.values())) or 1)

class SemanticCache:
    def __init__(self, threshold=0.6):
        self.items, self.threshold = [], threshold
    def get(self, q):
        qv = embed(q)
        best = max(self.items, key=lambda it: cosine(qv, it[0]), default=None)
        if best and cosine(qv, best[0]) >= self.threshold:
            return best[1], round(cosine(qv, best[0]), 3)
        return None, round(cosine(qv, best[0]), 3) if best else 0.0
    def put(self, q, answer):
        self.items.append((embed(q), answer))

cache = SemanticCache()
cache.put("What is the return window for shoes?", "Shoes can be returned within 10 days.")
for q in ["return window for shoes", "How many days do I have to return shoes?", "What is the refund time for UPI?"]:
    answer, sim = cache.get(q)
    print(f"{'HIT ' if answer else 'MISS'} sim={sim:.3f} | {q}" + (f" -> {answer}" if answer else " -> call the model"))

Output

HIT  sim=1.000 | return window for shoes -> Shoes can be returned within 10 days.
HIT  sim=0.667 | How many days do I have to return shoes? -> Shoes can be returned within 10 days.
MISS sim=0.000 | What is the refund time for UPI? -> call the model