Exact Match and Answer Metrics

Example

import re
from collections import Counter

def normalise(s):
    s = re.sub(r"[^a-z0-9 ]", " ", s.lower())
    return " ".join(w for w in s.split() if w not in {"a", "an", "the"})

def exact_match(pred, gold):
    return int(normalise(pred) == normalise(gold))

def token_f1(pred, gold):
    p, g = normalise(pred).split(), normalise(gold).split()
    common = sum((Counter(p) & Counter(g)).values())
    if common == 0:
        return 0.0
    precision, recall = common / len(p), common / len(g)
    return round(2 * precision * recall / (precision + recall), 3)

def must_include(pred, facts):
    return all(f.lower() in pred.lower() for f in facts)

gold = "Electronics have a 7 day replacement."
for pred in ["electronics have a 7 day replacement",
             "You get a 7 day replacement for electronics.",
             "Refunds take 5-7 working days."]:
    print(f"EM={exact_match(pred, gold)} F1={token_f1(pred, gold):.3f} "
          f"facts={must_include(pred, ['7', 'replacement'])} | {pred}")

Output

EM=1 F1=1.000 facts=True | electronics have a 7 day replacement
EM=0 F1=0.667 facts=True | You get a 7 day replacement for electronics.
EM=0 F1=0.182 facts=False | Refunds take 5-7 working days.