Validating and Retrying

Example

The scripted stand-in model makes two typical mistakes before getting it right; the loop feeds each error back:

import json, re

SCRIPTED_REPLIES = iter([
    'Here you go: {"product": "Boat Airdopes 141", "price": 1299}',          # extra text + missing field
    '{"product": "Boat Airdopes 141", "price": "1,299", "in_stock": true}',  # price is a string
    '{"product": "Boat Airdopes 141", "price": 1299, "in_stock": true}',     # correct
])
def fake_model(messages):
    """Stand-in for the model: returns the next scripted reply."""
    return next(SCRIPTED_REPLIES)

def check(data):
    errs = [f"missing {k}" for k in ("product", "price", "in_stock") if k not in data]
    if "price" in data and not isinstance(data["price"], (int, float)):
        errs.append("price must be a number, not a string")
    return errs

def extract_json(text, max_attempts=3):
    messages = [{"role": "user", "content": f"Extract product, price (number), in_stock (bool) as JSON:\n{text}"}]
    for attempt in range(1, max_attempts + 1):
        reply = fake_model(messages)
        match = re.search(r"\{.*\}", reply, re.S)          # tolerate text around the JSON
        try:
            data = json.loads(match.group(0)) if match else {}
            errors = check(data)
        except json.JSONDecodeError as e:
            errors = [f"invalid JSON: {e.msg}"]
        if not errors:
            print(f"attempt {attempt}: OK")
            return data
        print(f"attempt {attempt}: {errors}")
        messages += [{"role": "assistant", "content": reply},
                     {"role": "user", "content": f"Fix these problems and reply with JSON only: {errors}"}]
    raise ValueError("could not get valid JSON")

print(extract_json("Boat Airdopes 141 - Rs 1,299 - available"))

Output

attempt 1: ['missing in_stock']
attempt 2: ['price must be a number, not a string']
attempt 3: OK
{'product': 'Boat Airdopes 141', 'price': 1299, 'in_stock': True}