Lab: The inbox package and its rule baseline

import os
import sys

if os.path.basename(os.getcwd()) != "tidewell-inbox":     # not in the project yet
    if "google.colab" in sys.modules:                      # in Colab: the project lives in your Drive
        from google.colab import drive
        drive.mount("/content/drive")
        os.chdir("/content/drive/MyDrive")
    os.makedirs("tidewell-inbox", exist_ok=True)
    os.chdir("tidewell-inbox")
print("working in", os.path.basename(os.getcwd()))

working in tidewell-inbox

%pip install -q ftfy==6.3.1 pytest==9.1.1

from pathlib import Path
from IPython.core.magic import register_cell_magic

WROTE = set()                              # files this session's %%ensure wrote

@register_cell_magic
def ensure(line, cell):
    """%%ensure path: save the cell unless the file exists. %%ensure -a path: add to a file this session wrote."""
    *flags, name = line.split()
    path, text = Path(name), cell.rstrip("\n") + "\n"
    if flags == ["-a"]:
        if name in WROTE:
            with path.open("a", encoding="utf-8") as f:
                f.write("\n\n" + text)
            print("added to", path)
        return
    if path.exists():
        WROTE.discard(name)                # a kept file gets no added parts either
        print("kept", path)
        return
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(text, encoding="utf-8")
    WROTE.add(name)
    print("wrote", path)

%%ensure inbox/__init__.py
"""Tidewell inbox: clean, extract and route the messages in Tidewell Cycles' shared inbox."""

wrote inbox/__init__.py

%%ensure requirements.txt
regex==2026.9.29
ftfy==6.3.1
pytest==9.1.1
nltk==3.10.3
spacy==3.8.16
tokenizers==0.23.3
transformers==5.19.0
huggingface-hub==1.33.0
scikit-learn==1.9.1
gensim==4.4.0
sentence-transformers==6.1.0
rank-bm25==0.2.2
faiss-cpu==1.15.1
datasets==5.1.0
accelerate==1.15.0
torch==2.14.1
pydantic==2.14.1
sacrebleu==2.6.0
rouge-score==0.1.2
seqeval==1.2.2
bert-score==0.3.13
numpy==2.5.4
pandas==3.0.6
scipy==1.18.1
matplotlib==3.11.2
langchain==1.4.5
langchain-core==1.6.10
langgraph
langchain-groq==1.1.3
langchain-google-genai==4.4.2
langchain-ollama==1.1.0
langchain-huggingface==1.2.3
python-dotenv
ipykernel

wrote requirements.txt

%%ensure .gitignore
.env
.venv/
__pycache__/
.pytest_cache/
.ipynb_checkpoints/
models/
cache/

wrote .gitignore

%%ensure inbox/data.py
from datetime import date, timedelta

COURSE_DATE = date(2026, 10, 14)   # facts, versions and model names checked on this day
DATA_AS_OF = date(2026, 8, 31)     # the inbox snapshot: count days to this, never to the clock
WINDOW = (date(2026, 3, 2), date(2026, 8, 30))   # 26 weeks of messages, Monday to Sunday
SEED = 20260831

EVENTS = {   # fixed dates the data and the lessons share
    "spam_wave": (date(2026, 4, 6), date(2026, 4, 19)),
    "battery_notice": date(2026, 6, 9),      # email to Volt 2 owners about pack batch VB-0426
    "eu_shipping": date(2026, 7, 1),          # Germany, Spain and France
    "summer_sale": (date(2026, 7, 15), date(2026, 7, 28)),
    "carrier_change": date(2026, 8, 10),      # ParcelNorth -> SwiftDrop
}
BATCH = "VB-0426"
STAFF = ["Rosa", "Ines", "Dev", "Callum", "Amara"]

PRODUCTS = [   # (sku, name, price in GBP, kind)
    ("TW-VOLT2-BLK", "Volt 2", 2199, "ebike"),
    ("TW-VOLT2-STP", "Volt 2 Step-Through", 2249, "ebike"),
    ("TW-HARB-GRN", "Harbour", 649, "bike"),
    ("TW-HKID-20R", "Harbour Kids 20", 329, "bike"),
    ("TW-CAUS-GRY", "Causeway", 1149, "bike"),
    ("TW-PANN-SET", "Commuter Pannier Set", 89, "accessory"),
    ("TW-BATT-500", "Volt 2 battery pack (500 Wh)", 549, "accessory"),
    ("TW-LOCK-DLK", "LockSure D-lock", 45, "accessory"),
    ("TW-LITE-BCN", "Beacon Light Set", 39, "accessory"),
    ("TW-SERV-BAS", "Basic Service", 65, "service"),
    ("TW-SERV-FUL", "Full Service", 129, "service"),
    ("TW-REPR-PUN", "Puncture Repair", 18, "service"),
    ("TW-DIAG-EBK", "E-bike Diagnostic", 45, "service"),
    ("TW-GIFT-CRD", "Gift Card", 50, "gift"),          # sold from 25 to 200 GBP; 50 is the usual
]
INTENTS = ["order_status", "returns_refund", "repair_booking", "product_question",
           "warranty_fault", "praise", "spam"]
LABELS = {   # Ines's written definitions, used for every label in the course
    "order_status": "Where an order is, delivery problems, missing items, invoices and order documents.",
    "returns_refund": "Sending something back, exchanges without a fault, cancellations, refunds and wrong charges.",
    "repair_booking": "Booking, moving or asking the price of a service or repair for wear and tear.",
    "product_question": "Questions before or after buying: sizes, stock, range, compatibility, delivery areas.",
    "warranty_fault": "A product that is faulty or arrived damaged, including the battery batch VB-0426.",
    "praise": "Thanks, compliments and good reviews, with no request that needs action.",
    "spam": "Unsolicited selling, phishing and scams.",
}

CUSTOMERS = [   # (first, last, gender, language, home town); all fictional
    ("Joan", "Hartley", "f", "en", "Clifton"), ("Oliver", "Bennett", "m", "en", "Bristol"),
    ("Amelia", "Clarke", "f", "en", "Bath"), ("Mohammed", "Khan", "m", "en", "Bristol"),
    ("Aisha", "Begum", "f", "en", "Bedminster"), ("Daniel", "O'Connor", "m", "en", "Bristol"),
    ("Priya", "Patel", "f", "en", "Swindon"), ("Arjun", "Sharma", "m", "en", "Bristol"),
    ("Grace", "Okafor", "f", "en", "Cardiff"), ("Emeka", "Eze", "m", "en", "Bristol"),
    ("Hannah", "Lewis", "f", "en", "Stroud"), ("Tom", "Hughes", "m", "en", "Gloucester"),
    ("Siobhan", "Murphy", "f", "en", "Bristol"), ("Liam", "Walsh", "m", "en", "Bath"),
    ("Mei", "Chen", "f", "en", "Bristol"), ("Wei", "Zhang", "m", "en", "Clifton"),
    ("Fatima", "Hussain", "f", "en", "Bristol"), ("Yusuf", "Ali", "m", "en", "Bedminster"),
    ("Zara", "Ahmed", "f", "en", "Taunton"), ("Jack", "Robinson", "m", "en", "Bristol"),
    ("Ella", "Morgan", "f", "en", "Cardiff"), ("Rhys", "Evans", "m", "en", "Cardiff"),
    ("Megan", "Price", "f", "en", "Weston-super-Mare"), ("Ben", "Carter", "m", "en", "Bristol"),
    ("Lucy", "Ward", "f", "en", "Bath"), ("Kofi", "Asante", "m", "en", "Bristol"),
    ("Ama", "Boateng", "f", "en", "Bristol"), ("Olusegun", "Adeyemi", "m", "en", "Swindon"),
    ("Funmi", "Adebayo", "f", "en", "Bristol"), ("Raj", "Gupta", "m", "en", "Gloucester"),
    ("Neha", "Kapoor", "f", "en", "Bristol"), ("Tomasz", "Nowak", "m", "en", "Bristol"),
    ("Agnieszka", "Kowalska", "f", "en", "Stroud"), ("Stefan", "Ionescu", "m", "en", "Bristol"),
    ("Ioana", "Popescu", "f", "en", "Bath"), ("Hiroshi", "Tanaka", "m", "en", "Bristol"),
    ("Lukas", "Becker", "m", "de", "Berlin"), ("Anna", "Schmidt", "f", "de", "Hamburg"),
    ("Jonas", "Weber", "m", "de", "München"), ("Lea", "Fischer", "f", "de", "Köln"),
    ("Mehmet", "Yılmaz", "m", "de", "Berlin"), ("Elif", "Demir", "f", "de", "Leipzig"),
    ("Felix", "Wagner", "m", "de", "Hamburg"), ("Sophie", "Müller", "f", "de", "München"),
    ("Javier", "García", "m", "es", "Madrid"), ("Lucía", "Fernández", "f", "es", "Sevilla"),
    ("Alejandro", "Martín", "m", "es", "Valencia"), ("Carmen", "Ruiz", "f", "es", "Madrid"),
    ("Mateo", "López", "m", "es", "Barcelona"), ("Valentina", "Torres", "f", "es", "Bilbao"),
    ("Youssef", "El Amrani", "m", "es", "Barcelona"), ("Nuria", "Puig", "f", "es", "Barcelona"),
    ("Hugo", "Martin", "m", "fr", "Paris"), ("Camille", "Dubois", "f", "fr", "Lyon"),
    ("Lucas", "Bernard", "m", "fr", "Nantes"), ("Chloé", "Lefèvre", "f", "fr", "Bordeaux"),
    ("Karim", "Benali", "m", "fr", "Lille"), ("Amina", "Diallo", "f", "fr", "Paris"),
    ("Théo", "Moreau", "m", "fr", "Lyon"), ("Léa", "Laurent", "f", "fr", "Nantes"),
]

print(len(PRODUCTS), "products,", len(CUSTOMERS), "customers,", len(INTENTS), "intents")
print("women:", sum(c[2] == "f" for c in CUSTOMERS), "men:", sum(c[2] == "m" for c in CUSTOMERS))
print({lang: sum(c[3] == lang for c in CUSTOMERS) for lang in ["en", "de", "es", "fr"]})
print((DATA_AS_OF - WINDOW[0]).days, "days from the first message day to DATA_AS_OF")

wrote inbox/data.py

%%ensure -a inbox/data.py
import random, re

# Templates: one dict per wording. Keys en, de, es, fr hold the same message in each language
# (English-only wordings have just "en"). {UPPER} slots become gold entities; {lower} slots do not.
T = {
 "order_status": [
  {"s": "neu", "en": "Hi, I placed an order on {DATE} (order {ORDER_ID}, item: {PRODUCT}) and it still says processing. When will it ship?",
   "de": "Hallo, ich habe am {DATE} bestellt (Bestellung {ORDER_ID}, Artikel: {PRODUCT}) und der Status steht immer noch auf in Bearbeitung. Wann wird sie verschickt?",
   "es": "Hola, hice un pedido el {DATE} (pedido {ORDER_ID}, artículo: {PRODUCT}) y sigue en preparación. ¿Cuándo lo enviaréis?",
   "fr": "Bonjour, j'ai passé commande le {DATE} (commande {ORDER_ID}, article : {PRODUCT}) et elle est toujours en préparation. Quand sera-t-elle expédiée ?"},
  {"s": "neg", "en": "Order {ORDER_ID} was due on {DATE} and nothing has arrived. The {carrier} tracking has not changed for three days. Where is my parcel ({PRODUCT})?",
   "de": "Bestellung {ORDER_ID} sollte am {DATE} ankommen, aber es ist nichts da. Die Sendungsverfolgung von {carrier} hat sich seit drei Tagen nicht geändert. Wo ist mein Paket ({PRODUCT})?",
   "es": "El pedido {ORDER_ID} tenía que llegar el {DATE} y no ha llegado nada. El seguimiento de {carrier} no cambia desde hace tres días. ¿Dónde está mi paquete ({PRODUCT})?",
   "fr": "La commande {ORDER_ID} devait arriver le {DATE} et rien n'est arrivé. Le suivi {carrier} n'a pas bougé depuis trois jours. Où est mon colis ({PRODUCT}) ?"},
  {"s": "neu", "en": "Could you send me the tracking link for order {ORDER_ID}? The confirmation email never came through to {email}.",
   "de": "Könnten Sie mir den Tracking-Link für Bestellung {ORDER_ID} schicken? Die Bestätigung ist nie bei {email} angekommen.",
   "es": "¿Me podéis enviar el enlace de seguimiento del pedido {ORDER_ID}? El correo de confirmación nunca llegó a {email}.",
   "fr": "Pouvez-vous m'envoyer le lien de suivi de la commande {ORDER_ID} ? L'e-mail de confirmation n'est jamais arrivé sur {email}."},
  {"s": "neg", "from": "2026-08-10", "w": 3, "en": "Third time {carrier} has said 'attempted delivery' when I was in all day. Order {ORDER_ID}. This never happened with your old courier."},
  {"s": "neu", "en": "any update on {ORDER_ID}? need the {PRODUCT} before {DATE} if poss"},
 ],
 "returns_refund": [
  {"s": "neu", "en": "I'd like to return the {PRODUCT} from order {ORDER_ID}. It's unused and still in the box. How do I send it back?",
   "de": "Ich möchte {PRODUCT} aus Bestellung {ORDER_ID} zurückgeben. Unbenutzt und noch originalverpackt. Wie schicke ich es zurück?",
   "es": "Quiero devolver {PRODUCT} del pedido {ORDER_ID}. Está sin usar y en su caja. ¿Cómo lo envío?",
   "fr": "Je voudrais retourner {PRODUCT} de la commande {ORDER_ID}. Il n'a pas servi et il est encore dans sa boîte. Comment le renvoyer ?"},
  {"s": "neg", "en": "I sent the return for {ORDER_ID} back on {DATE} and still no refund of {MONEY}. How long does this take?",
   "de": "Ich habe die Rücksendung für {ORDER_ID} am {DATE} abgeschickt und noch keine Erstattung über {MONEY} bekommen. Wie lange dauert das?",
   "es": "Envié la devolución del pedido {ORDER_ID} el {DATE} y todavía no me habéis devuelto {MONEY}. ¿Cuánto tarda?",
   "fr": "J'ai renvoyé la commande {ORDER_ID} le {DATE} et je n'ai toujours pas reçu le remboursement de {MONEY}. Combien de temps faut-il ?"},
  {"s": "neu", "en": "The {KIDBIKE} is the wrong size for my daughter. Can I swap it or get a refund? Order {ORDER_ID}.",
   "de": "{KIDBIKE} ist für meine Tochter die falsche Größe. Kann ich umtauschen oder bekomme ich das Geld zurück? Bestellung {ORDER_ID}.",
   "es": "{KIDBIKE} es de una talla equivocada para mi hija. ¿Puedo cambiarlo o me devolvéis el dinero? Pedido {ORDER_ID}.",
   "fr": "{KIDBIKE} n'est pas à la bonne taille pour ma fille. Puis-je l'échanger ou être remboursé ? Commande {ORDER_ID}."},
  {"s": "neg", "en": "You charged me {MONEY} twice for order {ORDER_ID}. Please refund the duplicate payment."},
  {"s": "neu", "en": "want to cancel order {ORDER_ID} before it ships, changed my mind about the {PRODUCT}. refund to my card pls"},
 ],
 "repair_booking": [
  {"s": "neu", "en": "Can I book a {SERVICE} for {DATE}? It's for my bike ({BIKE}). I can drop it at the workshop in the morning.",
   "de": "Kann ich für den {DATE} einen Termin für {SERVICE} buchen? Es geht um mein Rad ({BIKE}). Ich kann es morgens in die Werkstatt bringen.",
   "es": "¿Puedo reservar {SERVICE} para el {DATE}? Es para mi bici ({BIKE}). Puedo dejarla en el taller por la mañana.",
   "fr": "Puis-je réserver {SERVICE} pour le {DATE} ? C'est pour mon vélo ({BIKE}). Je peux le déposer à l'atelier le matin."},
  {"s": "neu", "en": "My bike ({BIKE}) needs a repair: {wear}. Can I bring it in or send it to you? I live in {LOC}.",
   "de": "Mein Rad ({BIKE}) muss repariert werden: {wear}. Kann ich es vorbeibringen oder einschicken? Ich wohne in {LOC}.",
   "es": "Mi bici ({BIKE}) necesita una reparación: {wear}. ¿Puedo llevarla o enviárosla? Vivo en {LOC}.",
   "fr": "Mon vélo ({BIKE}) a besoin d'une réparation : {wear}. Puis-je l'apporter ou vous l'envoyer ? J'habite à {LOC}."},
  {"s": "neu", "en": "Please can I move my {SERVICE} booking ({ticket}) from {DATE} to the following week? Something came up.",
   "de": "Kann ich meinen Termin für {SERVICE} ({ticket}) vom {DATE} auf die Woche danach verschieben? Mir ist etwas dazwischengekommen.",
   "es": "¿Puedo cambiar mi cita de {SERVICE} ({ticket}) del {DATE} a la semana siguiente? Me ha surgido algo.",
   "fr": "Puis-je déplacer mon rendez-vous {SERVICE} ({ticket}) du {DATE} à la semaine suivante ? J'ai un empêchement."},
  {"s": "neu", "en": "hiya, how much is a {SERVICE} and can u fit me in {DATE}? bike is a {BIKE}, cheers"},
  {"s": "neu", "en": "I'd like to book the {SERVICE} for my {BIKE}. Is {MONEY} still the price? Any day after {DATE} works."},
 ],
 "product_question": [
  {"s": "neu", "en": "Does the {BIKE} come in a smaller frame? I'm 158 cm tall.",
   "de": "Gibt es {BIKE} auch mit einem kleineren Rahmen? Ich bin 1,58 m groß.",
   "es": "¿{BIKE} viene en una talla de cuadro más pequeña? Mido 1,58 m.",
   "fr": "{BIKE} existe-t-il avec un cadre plus petit ? Je mesure 1,58 m."},
  {"s": "neu", "en": "What range does the {EBIKE} get on one charge, and is the {ACCESSORY} compatible with it?",
   "de": "Welche Reichweite hat {EBIKE} mit einer Akkuladung, und passt {ACCESSORY} dazu?",
   "es": "¿Qué autonomía tiene {EBIKE} con una carga, y es compatible con {ACCESSORY}?",
   "fr": "Quelle autonomie a {EBIKE} avec une charge, et {ACCESSORY} est-il compatible ?"},
  {"s": "neu", "en": "Is the {PRODUCT} in stock? The website says {MONEY} but I can't add it to the basket.",
   "de": "Ist {PRODUCT} auf Lager? Auf der Website steht {MONEY}, aber ich kann es nicht in den Warenkorb legen.",
   "es": "¿Tenéis {PRODUCT} en stock? En la web pone {MONEY} pero no puedo añadirlo a la cesta.",
   "fr": "Est-ce que {PRODUCT} est en stock ? Le site indique {MONEY} mais je n'arrive pas à l'ajouter au panier."},
  {"s": "neu", "en": "Do you deliver to {LOC}? And do bikes arrive fully built or do I need to assemble them?"},
  {"s": "neu", "en": "quick q: whats the difference between the {SERVICE} and the {SERVICE2}?"},
 ],
 "warranty_fault": [
  {"s": "neg", "en": "My e-bike ({EBIKE}) is only four months old and {fault}. Is this covered by the warranty?",
   "de": "Mein E-Bike ({EBIKE}) ist erst vier Monate alt und {fault}. Ist das von der Garantie abgedeckt?",
   "es": "Mi bici eléctrica ({EBIKE}) solo tiene cuatro meses y {fault}. ¿Lo cubre la garantía?",
   "fr": "Mon vélo électrique ({EBIKE}) n'a que quatre mois et {fault}. Est-ce couvert par la garantie ?"},
  {"s": "neg", "from": "2026-06-09", "w": 4, "en": "I got your email about battery batch {batch}. My pack has that code and {fault}. What happens now?",
   "de": "Ich habe Ihre E-Mail zur Akku-Charge {batch} bekommen. Mein Akku hat diesen Code und {fault}. Wie geht es jetzt weiter?",
   "es": "Recibí vuestro correo sobre el lote de baterías {batch}. Mi batería tiene ese código y {fault}. ¿Qué hago ahora?",
   "fr": "J'ai reçu votre e-mail sur le lot de batteries {batch}. Ma batterie porte ce code et {fault}. Que se passe-t-il maintenant ?"},
  {"s": "neg", "en": "The {PRODUCT} arrived with {damage}. I've attached photos. Can you send a replacement?",
   "de": "{PRODUCT} kam mit {damage} an. Fotos sind im Anhang. Können Sie Ersatz schicken?",
   "es": "{PRODUCT} llegó con {damage}. Adjunto fotos. ¿Podéis enviarme uno nuevo?",
   "fr": "{PRODUCT} est arrivé avec {damage}. Je joins des photos. Pouvez-vous m'en envoyer un autre ?"},
  {"s": "neg", "en": "Spoke snapped on my {BIKE} after two rides. That's not normal wear, surely? Bought {DATE}, order {ORDER_ID}."},
  {"s": "neg", "en": "{EBIKE} battery dead again. Second pack in a month. I want this sorted properly this time."},
 ],
 "praise": [
  {"s": "pos", "en": "Just wanted to say thank you to {PERSON} in the workshop. My bike ({BIKE}) rides like new after the {SERVICE}.",
   "de": "Ich wollte mich nur bei {PERSON} in der Werkstatt bedanken. Mein Rad ({BIKE}) fährt sich nach {SERVICE} wie neu.",
   "es": "Solo quería dar las gracias a {PERSON} del taller. Mi bici ({BIKE}) va como nueva después de {SERVICE}.",
   "fr": "Je voulais juste remercier {PERSON} de l'atelier. Mon vélo ({BIKE}) roule comme neuf après {SERVICE}."},
  {"s": "pos", "en": "Love my new bike ({BIKE}). Fast delivery to {LOC} and it was easy to set up. Five stars.",
   "de": "Ich liebe mein neues Rad ({BIKE}). Schnelle Lieferung nach {LOC} und leicht aufzubauen. Fünf Sterne.",
   "es": "Me encanta mi nueva bici ({BIKE}). Entrega rápida a {LOC} y fácil de montar. Cinco estrellas.",
   "fr": "J'adore mon nouveau vélo ({BIKE}). Livraison rapide à {LOC} et facile à monter. Cinq étoiles."},
  {"s": "pos", "en": "Great service from {PERSON}, who sorted my problem in one email. Thank you!",
   "de": "Super Service von {PERSON}, das Problem war mit einer E-Mail gelöst. Vielen Dank!",
   "es": "Un servicio genial de {PERSON}, que me lo resolvió en un solo correo. ¡Gracias!",
   "fr": "Super service de la part de {PERSON}, qui a réglé mon problème en un seul e-mail. Merci !"},
  {"s": "pos", "en": "the {ACCESSORY} is brilliant, fits perfectly. cheers"},
  {"s": "pos", "en": "Honestly the best bike shop in {LOC}. Friendly, quick and fair prices."},
 ],
 "spam": [
  {"s": "neu", "en": "Dear partner, we offer guaranteed first-page search ranking for your online store. Reply today for a {MONEY} discount."},
  {"s": "neu", "en": "Exclusive partnership opportunity: invest in our crypto bike-sharing token and earn 300% returns. Contact {email} now."},
  {"s": "neu", "en": "Your mailbox storage is full. Verify your account within 24 hours at http://mail-verify.example/login to avoid suspension."},
  {"s": "neu", "en": "Hello, I am an influencer with 50k followers. Send me a free {BIKE} and I will post about your brand."},
  {"s": "neu", "en": "Congratulations! You have been selected for a {MONEY} gift voucher. Click http://prize-claim.example to claim."},
 ],
}
PHRASES = {   # paired fillers for the {lower} slots: (en, de, es, fr)
 "fault": [("the battery will not charge", "der Akku lädt nicht mehr", "la batería no carga", "la batterie ne se charge plus"),
           ("the motor cuts out on hills", "der Motor setzt am Berg aus", "el motor se para en las cuestas", "le moteur se coupe dans les montées"),
           ("the display shows error E07", "das Display zeigt Fehler E07", "la pantalla muestra el error E07", "l'écran affiche l'erreur E07"),
           ("the battery runs flat very quickly", "der Akku ist sehr schnell leer", "la batería se agota muy rápido", "la batterie se vide très vite")],
 "wear": [("the back tyre has a puncture", "der Hinterreifen hat einen Platten", "la rueda trasera está pinchada", "le pneu arrière est crevé"),
          ("the gears keep slipping", "die Gangschaltung springt ständig", "los cambios saltan todo el rato", "les vitesses sautent sans arrêt"),
          ("the brake pads are worn out", "die Bremsbeläge sind abgenutzt", "las pastillas de freno están gastadas", "les plaquettes de frein sont usées"),
          ("the chain squeaks", "die Kette quietscht", "la cadena chirría", "la chaîne grince")],
 "damage": [("a cracked mudguard", "einem gebrochenen Schutzblech", "un guardabarros roto", "un garde-boue fendu"),
            ("a bent front wheel", "einem verbogenen Vorderrad", "la rueda delantera doblada", "une roue avant voilée"),
            ("a broken clip", "einem kaputten Clip", "un clip roto", "un clip cassé")],
}
SUBJECTS = {"order_status": ["Where is my order?", "Wo bleibt meine Bestellung?", "¿Dónde está mi pedido?", "Où est ma commande ?"],
            "returns_refund": ["Return and refund", "Rücksendung", "Devolución", "Retour"],
            "repair_booking": ["Booking a service", "Werkstatttermin", "Cita en el taller", "Rendez-vous atelier"],
            "product_question": ["Product question", "Frage zum Produkt", "Pregunta sobre un producto", "Question sur un produit"],
            "warranty_fault": ["Fault with my bike", "Defekt", "Avería", "Panne"],
            "praise": ["Thank you!", "Danke!", "¡Gracias!", "Merci !"],
            "spam": ["Partnership opportunity"] * 4}
FORM_TOPIC = {"order_status": "Order enquiry", "returns_refund": "Returns", "repair_booking": "Workshop",
              "product_question": "Product question", "warranty_fault": "Faulty item", "praise": "Feedback", "spam": "Other"}
CHANNELS = {"order_status": (5, 4, 1), "returns_refund": (6, 4, 0), "repair_booking": (4, 6, 0),   # email, web form, review
            "product_question": (4, 6, 0), "warranty_fault": (5, 3, 2), "praise": (3, 1, 6), "spam": (1, 0, 0)}
LANGS = ["en", "de", "es", "fr"]
MONTHS = {"en": "January February March April May June July August September October November December",
          "de": "Januar Februar März April Mai Juni Juli August September Oktober November Dezember",
          "es": "enero febrero marzo abril mayo junio julio agosto septiembre octubre noviembre diciembre",
          "fr": "janvier février mars avril mai juin juillet août septembre octobre novembre décembre"}
TOWNS = {"en": ["Bristol", "Bath", "Clifton", "Cardiff", "Stroud"], "de": ["Berlin", "Hamburg", "München", "Köln"],
         "es": ["Madrid", "Barcelona", "Valencia", "Sevilla"], "fr": ["Paris", "Lyon", "Nantes", "Lille"]}
SPAMMERS = [("Growth Partners", "offers@growth-partners.example"), ("Crypto Ride", "team@cryptoride.example"),
            ("Mail Admin", "admin@mail-verify.example"), ("Prize Desk", "win@prize-claim.example")]


def fmt_date(d, lang, rng):
    m = MONTHS[lang].split()[d.month - 1]
    styles = {"en": [f"{d.day} {m}", f"{d.day:02d}/{d.month:02d}", f"{m} {d.day}{'th' if 10 < d.day < 14 else {1: 'st', 2: 'nd', 3: 'rd'}.get(d.day % 10, 'th')}"],
              "de": [f"{d.day}. {m}", f"{d.day:02d}.{d.month:02d}."], "es": [f"{d.day} de {m}", f"{d.day}/{d.month}"],
              "fr": [f"{d.day} {m}", f"{d.day:02d}/{d.month:02d}/{d.year}"]}[lang]
    return rng.choice(styles) if rng else styles[0]


def fmt_money(gbp, lang, rng):
    styles = {"en": [f"£{gbp:,}", f"£{gbp:,}.00", f"{gbp} pounds"], "de": [f"{gbp:,} £".replace(",", "."), f"{gbp} Pfund"],
              "es": [f"{gbp:,} libras".replace(",", "."), f"{gbp} £"], "fr": [f"{gbp:,} £".replace(",", " "), f"{gbp} livres"]}[lang]
    return rng.choice(styles) if rng else styles[0]


def typo(word, rng):
    i = rng.randrange(1, len(word) - 1)
    return rng.choice([word[:i] + word[i + 1] + word[i] + word[i + 2:], word[:i] + word[i + 1:], word[:i] + word[i] + word[i:]])


print(sum(len(v) for v in T.values()), "wordings,", sum("de" in t for v in T.values() for t in v), "of them in all four languages")

added to inbox/data.py

%%ensure -a inbox/data.py
import html, unicodedata

POOLS = {"PRODUCT": ["bike", "ebike", "accessory"], "BIKE": ["bike", "ebike"], "EBIKE": ["ebike"],
         "ACCESSORY": ["accessory"], "SERVICE": ["service"], "SERVICE2": ["service"], "KIDBIKE": ["kids"]}
EXTRA = {"greet": {"en": ["Hi,", "Hello,", "Hello Tidewell,", "Dear Sir or Madam,"], "de": ["Hallo,", "Guten Tag,"],
                   "es": ["Hola,", "Buenos días,"], "fr": ["Bonjour,", "Madame, Monsieur,"]},
         "sign": {"en": ["Thanks,", "Many thanks,", "Cheers,"], "de": ["Danke,", "Viele Grüße,"],
                  "es": ["Gracias,", "Un saludo,"], "fr": ["Merci,", "Cordialement,"]},
         "phone": {"en": "You can call me on ", "de": "Sie erreichen mich unter ", "es": "Me podéis llamar al ",
                   "fr": "Vous pouvez m'appeler au "},
         "quote": {"en": ("On ", ", Tidewell Cycles <help@tidewell.example> wrote:"), "de": ("Am ", " schrieb Tidewell Cycles <help@tidewell.example>:"),
                   "es": ("El ", ", Tidewell Cycles <help@tidewell.example> escribió:"), "fr": ("Le ", ", Tidewell Cycles <help@tidewell.example> a écrit :")},
         "again": {"en": "Sorry, sending this again as I have not heard back. ", "de": "Entschuldigung, ich schicke das noch einmal. ",
                   "es": "Perdón, lo envío otra vez porque no he recibido respuesta. ", "fr": "Désolé, je renvoie ce message car je n'ai pas eu de réponse. "}}
LABEL_OF = {"PRODUCT": "PRODUCT", "BIKE": "PRODUCT", "EBIKE": "PRODUCT", "ACCESSORY": "PRODUCT", "SERVICE": "PRODUCT",
            "SERVICE2": "PRODUCT", "KIDBIKE": "PRODUCT", "ORDER_ID": "ORDER_ID", "DATE": "DATE", "MONEY": "MONEY",
            "LOC": "LOC", "PERSON": "PERSON"}


def fill(template, values, lang, rng):
    """Template text -> list of (text, label or None) pieces, formatting dates and money for `lang`."""
    pieces = []
    for part in re.split(r"(\{\w+\})", template):
        if not (part.startswith("{") and part.endswith("}")):
            pieces.append((part, None)) if part else None
            continue
        slot = part[1:-1]
        v = values[slot]
        if slot == "DATE":
            v = fmt_date(v, lang, rng)
        elif slot == "MONEY":
            v = fmt_money(v, lang, rng)
        elif slot in ("fault", "wear", "damage"):
            v = PHRASES[slot][v][LANGS.index(lang)]
        pieces.append((v, LABEL_OF.get(slot)))
    return pieces


def email_of(first, last):
    """'Mehmet', 'Yılmaz' -> 'mehmet.yilmaz@...'; one fixed address per customer, on a reserved .example domain."""
    plain = unicodedata.normalize("NFKD", f"{first}.{last}".lower().replace("ı", "i").replace(" ", "").replace("'", ""))
    plain = "".join(ch for ch in plain if ch.isascii())
    return plain + "@" + ["post.example", "mailbox.example", "inbox.example"][len(first + last) % 3]


def make_messages(n=1800, seed=SEED):
    rng = random.Random(seed)
    days = [WINDOW[0] + timedelta(i) for i in range((WINDOW[1] - WINDOW[0]).days + 1)]
    base = {"order_status": 26, "returns_refund": 16, "repair_booking": 17, "product_question": 18,
            "warranty_fault": 11, "praise": 6, "spam": 6}

    def weights(d):
        w = dict(base)
        w["spam"] *= 8 if EVENTS["spam_wave"][0] <= d <= EVENTS["spam_wave"][1] else 1
        w["warranty_fault"] *= 3.5 if EVENTS["battery_notice"] <= d <= date(2026, 7, 31) else 2 if d.month == 8 else 1
        w["order_status"] *= (2.5 if EVENTS["summer_sale"][0] <= d <= EVENTS["summer_sale"][1] else 1) * (1.6 if d >= EVENTS["carrier_change"] else 1)
        w["returns_refund"] *= 1.3 if date(2026, 7, 29) <= d <= date(2026, 8, 15) else 1
        vol = (0.6 if d.weekday() >= 5 else 1.0) * (1.5 if d >= EVENTS["eu_shipping"] else 1.0)
        return [w[i] * vol for i in INTENTS]

    W = [weights(d) for d in days]
    n_dup = round(n * 0.03)
    picks = sorted(rng.choices(range(len(days)), weights=[sum(w) for w in W], k=n - n_dup))
    by_kind = {k: [p for p in PRODUCTS if p[3] == k] for k in ["bike", "ebike", "accessory", "service"]}
    by_kind["kids"] = [p for p in PRODUCTS if p[1] == "Harbour Kids 20"]
    out = []
    for di in picks:
        d = days[di]
        intent = rng.choices(INTENTS, weights=W[di])[0]
        lang = "en" if intent == "spam" else rng.choices(LANGS, weights=[95, 2, 1.5, 1.5] if d < EVENTS["eu_shipping"] else [58, 16, 14, 12])[0]
        options = [t for t in T[intent] if lang in t and date.fromisoformat(t.get("from", "2026-01-01")) <= d]
        tmpl = rng.choices(options, weights=[t.get("w", 1) for t in options])[0]
        if intent == "spam":
            cust, email = rng.choice(SPAMMERS)
            first, last, gender, town = cust, "", None, None
        else:
            first, last, gender, _, town = rng.choice([c for c in CUSTOMERS if c[3] == lang])
            email = email_of(first, last)
        v = {}
        for slot, kinds in POOLS.items():
            pool = [p for k in kinds for p in by_kind[k]]
            pick = rng.choice([p for p in pool if p[1] != v.get("SERVICE")])
            v[slot] = pick[1]
            v[slot + "_sku"], v[slot + "_price"] = pick[0], pick[2]
        main = next((s for s in POOLS if "{" + s + "}" in tmpl["en"]), None)
        future = intent == "repair_booking" or "before {DATE}" in tmpl["en"]
        v["DATE"] = d + timedelta(rng.randint(2, 14)) if future else d - timedelta(rng.randint(3, 20))
        v["MONEY"] = v[main + "_price"] if main else rng.choice([50, 100, 250, 500])
        v["ORDER_ID"] = f"TW-{100000 + di * 120 + rng.randrange(120)}"
        lives_there = "live in {LOC}" in tmpl["en"] or "shop in {LOC}" in tmpl["en"]
        v["LOC"] = town if lives_there and town else rng.choice(TOWNS[lang])
        v["PERSON"] = rng.choice(["Dev", "Dev", "Ines", "Callum"]) if intent == "praise" else first
        v.update(carrier="SwiftDrop" if d >= EVENTS["carrier_change"] else "ParcelNorth", email=email, batch=BATCH,
                 ticket=f"RJ-2026-{300 + di * 2 + rng.randrange(3):04d}", fault=rng.randrange(4), wear=rng.randrange(4),
                 damage=rng.randrange(3))
        core = fill(tmpl[lang], v, lang, rng)
        en_ref = "".join(t for t, _ in fill(tmpl["en"], v, "en", None)) if lang != "en" else None
        channel = rng.choices(["email", "web form", "review"], weights=CHANNELS[intent])[0]
        channel = "web form" if channel == "review" and tmpl["s"] == "neu" else channel   # reviews praise or complain
        pieces, g = list(core), 0
        if channel == "email":
            if rng.random() < 0.7:
                pieces, g = [(rng.choice(EXTRA["greet"][lang]) + "\n\n", None)] + pieces, 1
            if rng.random() < 0.15:
                pieces += [("\n" + EXTRA["phone"][lang] + rng.choice(["07700 900", "07700900", "+44 7700 900"]) + f"{rng.randrange(1000):03d}.", None)]
            if lang == "en" and intent in ("repair_booking", "order_status") and rng.random() < 0.3:
                pieces += [(f"\nMy postcode is BS{rng.randint(1, 16)} {rng.randint(1, 9)}{rng.choice('ABDEFGHJLNPQRSTUWXYZ')}{rng.choice('ABDEFGHJLNPQRSTUWXYZ')}.", None)]
            if rng.random() < 0.7 and intent != "spam":
                pieces += [("\n\n" + rng.choice(EXTRA["sign"][lang]) + "\n", None), (first if rng.random() < 0.6 else f"{first} {last}", "PERSON")]
                if rng.random() < 0.15:
                    pieces += [("\n" + {"en": "Sent from my phone", "de": "Von meinem Handy gesendet", "es": "Enviado desde mi móvil", "fr": "Envoyé de mon téléphone"}[lang], None)]
            if rng.random() < 0.25 and intent != "spam":
                q = EXTRA["quote"][lang]
                prev = fmt_date(d - timedelta(rng.randint(1, 6)), lang, None) + (" de 2026" if lang == "es" else " 2026")
                pieces += [("\n\n> " + q[0], None), (prev, "DATE"), (q[1] + "\n> Thanks for your message. We will reply within one working day.", None)]
        r = rng.random
        if r() < 0.15:                                          # typos
            for _ in range(rng.randint(1, 2)):
                i = rng.randrange(len(pieces))
                if pieces[i][1] is None:
                    words = pieces[i][0].split(" ")
                    j = rng.randrange(len(words))
                    if len(words[j]) >= 5 and words[j].isalpha():
                        words[j] = typo(words[j], rng)
                        pieces[i] = (" ".join(words), None)
        case = r()
        if case < 0.03:
            pieces = [(t.upper(), l) for t, l in pieces]
        elif case < 0.09:
            pieces = [(t.lower(), l) for t, l in pieces]
        if r() < 0.08:                                          # emoji after the main text
            pieces.insert(g + len(core), (" " + rng.choice(["😡", "😤", "🚲"] if tmpl["s"] == "neg" else ["🙂", "👍", "🚲"]), None))
        if r() < 0.05:                                          # odd whitespace
            i = next((i for i, (t, l) in enumerate(pieces) if l is None and " " in t), None)
            if i is not None:
                t = pieces[i][0]
                pieces[i] = (t.replace(" ", " ", 1) if r() < 0.5 else t.replace(" ", " ​", 1), None)
        if channel == "web form" and r() < 0.6:                 # HTML from the form
            pieces = [("<p>", None)] + [(html.escape(t).replace("\n", "<br>\n"), None) if l is None else (t, l) for t, l in pieces] + [("</p>", None)]
        if r() < 0.04:                                          # mojibake: UTF-8 bytes read as cp1252
            try:
                pieces = [(t.encode("utf-8").decode("cp1252"), l) for t, l in pieces]
            except UnicodeDecodeError:
                pass
        body, spans, pos = "", [], 0
        for t, l in pieces:
            if l:
                spans.append((pos, pos + len(t), l))
            body, pos = body + t, pos + len(t)
        sent = tmpl["s"]
        stars = None
        if channel == "review":
            stars = {"pos": rng.choice([4, 5]), "neu": 3, "neg": rng.choice([1, 2])}[sent]
            if r() < 0.08:
                stars = {"pos": rng.choice([1, 2]), "neu": rng.choice([1, 5]), "neg": 5}[sent]
        subject = {"email": SUBJECTS[intent][LANGS.index(lang)], "web form": "Web form: " + FORM_TOPIC[intent],
                   "review": "Review: " + (v[main] if main else "Tidewell Cycles")}[channel]
        out.append({"date": d.isoformat(), "channel": channel, "language": lang, "customer": f"{first} {last}".strip(),
                    "gender": gender, "email": email, "subject": subject, "body": body, "intent": intent, "sentiment": sent,
                    "stars": stars, "entities": spans, "sku": v[main + "_sku"] if main else None,
                    "order_id": v["ORDER_ID"] if "{ORDER_ID}" in tmpl["en"] else None, "duplicate_of": None,
                    "core": "".join(t for t, _ in core), "en_ref": en_ref, "label_flipped": False})
    originals = [m for m in out if m["intent"] != "spam"]
    for m in rng.sample(originals, n_dup):                      # near-duplicates: the same customer writes again
        again, cut = EXTRA["again"][m["language"]], 3 if m["body"].startswith("<p>") else 0   # inside the form's HTML
        dup = dict(m, body=m["body"][:cut] + again + m["body"][cut:], entities=[(a + len(again), b + len(again), l) for a, b, l in m["entities"]],
                   date=min(date.fromisoformat(m["date"]) + timedelta(rng.randint(0, 2)), WINDOW[1]).isoformat(), duplicate_of=m)
        out.append(dup)
    out.sort(key=lambda m: (m["date"], m["duplicate_of"] is not None))
    for i, m in enumerate(out, 1):
        m["id"] = f"TW-M{i:04d}"
    for m in out:
        if m["duplicate_of"] is not None:
            m["duplicate_of"] = m["duplicate_of"]["id"]
    for m in rng.sample(out, round(n * 0.02)):                  # deliberate wrong labels, for error analysis
        m["intent"], m["label_flipped"] = rng.choice([i for i in INTENTS if i != m["intent"]]), True
    split_of = {}
    for intent in INTENTS:                                      # stratified 70 / 15 / 15, duplicates follow their original
        ids = [m["id"] for m in out if m["intent"] == intent and m["duplicate_of"] is None]
        rng.shuffle(ids)
        a, b = round(len(ids) * 0.70), round(len(ids) * 0.85)
        split_of.update({x: "train" if k < a else "dev" if k < b else "test" for k, x in enumerate(ids)})
    for m in out:
        m["split"] = split_of[m["duplicate_of"] or m["id"]]
        m["period"] = "before_july" if m["date"] < "2026-07-01" else "from_july"
    return out


MESSAGES = make_messages()
print(len(MESSAGES), "messages;", MESSAGES[0]["date"], "to", MESSAGES[-1]["date"])
print(MESSAGES[0]["id"], "|", MESSAGES[0]["subject"], "|", MESSAGES[0]["body"][:80].replace("\n", " "))

added to inbox/data.py

%%ensure -a inbox/data.py
import re

def unmark(marked):
    """'[[Volt 2|PRODUCT]] died' -> ('Volt 2 died', [(0, 6, 'PRODUCT')])."""
    text, spans, pos = "", [], 0
    for m in re.finditer(r"\[\[(.+?)\|([A-Z_]+)\]\]|([^\[]+|\[)", marked):
        if m.group(1):
            spans.append((len(text), len(text) + len(m.group(1)), m.group(2)))
            text += m.group(1)
        else:
            text += m.group(3)
    return text, spans

# (language, kind, intent, second intent, sentiment, text with [[entity|LABEL]] marks, English reference or None)
HARD = [
 ("en", "sarcasm", "warranty_fault", None, "neg", "Oh brilliant, the [[Volt 2|PRODUCT]] battery died halfway up the hill again. Really loving my [[£2,199|MONEY]] paperweight.", None),
 ("en", "two_intents", "order_status", "returns_refund", "neu", "Two things: where is order [[TW-118734|ORDER_ID]]? And I want to send back the [[LockSure D-lock|PRODUCT]] from my last order, it's too heavy.", None),
 ("en", "very_short", "returns_refund", None, "neg", "refund??", None),
 ("en", "very_short", "order_status", None, "neu", "where's my bike", None),
 ("en", "slang", "repair_booking", None, "neu", "yo the gears on my [[Causeway|PRODUCT]] are proper knackered, can u lot sort it [[sat|DATE]]?", None),
 ("en", "plain", "praise", None, "pos", "[[Dev|PERSON]] fixed my wheel in ten minutes while I waited. Legend.", None),
 ("en", "sarcasm", "order_status", None, "neg", "Great to see my parcel has been 'out for delivery' for six days. Very scenic route, I assume. Order [[TW-121090|ORDER_ID]].", None),
 ("en", "negation", "product_question", None, "neu", "I don't want the [[Volt 2|PRODUCT]], I want something without a battery for my commute. Is the [[Harbour|PRODUCT]] any good on hills?", None),
 ("en", "plain", "warranty_fault", None, "neg", "The display on my [[Volt 2 Step-Through|PRODUCT]] keeps showing E07 and the motor won't engage. Bought it on [[14 March|DATE]].", None),
 ("en", "plain", "spam", None, "neu", "Hi [[Rosa|PERSON]], loved your shop! We help bike brands get 10x more reviews. Can I send over a quick proposal?", None),
 ("en", "very_long", "warranty_fault", None, "neu", "Hello, I hope you are well. I'm writing about the email you sent on [[9 June|DATE]] about battery batch VB-0426. I checked the label under my pack as the email said, and it does say VB-0426. To be honest the battery had been fine until about a fortnight ago, when it started dropping from 80% to nothing in a couple of miles. I use the bike every day to take my son to school and then ride on to work, so I can't really be without it for long. Could you tell me how the swap works, whether I need to bring the whole bike in or just the battery, and roughly how long it takes? My order number was [[TW-103552|ORDER_ID]] if that helps. Many thanks, [[Margaret Ellis|PERSON]]", None),
 ("en", "codes", "product_question", None, "neu", "is TW-BATT-500 the same pack as the one in the [[Volt 2|PRODUCT]] or a newer version?", None),
 ("en", "plain", "returns_refund", None, "neu", "I'd like to return the [[Commuter Pannier Set|PRODUCT]]. The clips don't fit my rack. Order [[TW-115902|ORDER_ID]].", None),
 ("en", "plain", "repair_booking", None, "neu", "Can I bring my [[Harbour Kids 20|PRODUCT]] in for a [[Basic Service|PRODUCT]] next week? Any day but [[Wednesday|DATE]].", None),
 ("en", "mixed", "praise", None, "neu", "Bike is lovely, delivery was a nightmare. Overall happy though, thanks [[Ines|PERSON]].", None),
 ("en", "slang", "order_status", None, "neg", "any news on my order mate? [[TW-122417|ORDER_ID]]. been ages", None),
 ("en", "plain", "spam", None, "neu", "Your Tidewell Cycles account has been suspended. Confirm your payment details at tidewell-secure.example within 24 hours.", None),
 ("en", "very_short", "warranty_fault", None, "neg", "battery dead. again.", None),
 ("en", "plain", "repair_booking", None, "neu", "Is the [[Full Service|PRODUCT]] still [[£129|MONEY]]? If so I'll book it for the [[2nd of September|DATE]].", None),
 ("en", "plain", "product_question", None, "neu", "Will a [[LockSure D-lock|PRODUCT]] fit the frame of the [[Causeway|PRODUCT]] or is the tube too fat?", None),
 ("en", "sarcasm", "returns_refund", None, "neg", "Thanks so much for refunding me in 'five working days'. It's been fifteen. Order [[TW-112006|ORDER_ID]], [[£89|MONEY]].", None),
 ("en", "two_intents", "warranty_fault", "repair_booking", "neg", "My [[Volt 2|PRODUCT]] motor is cutting out, and while it's in could you also do a [[Basic Service|PRODUCT]]?", None),
 ("en", "plain", "praise", None, "pos", "Five stars for the [[Harbour|PRODUCT]]. Comfy, fast enough, and the bell is very loud. Ha.", None),
 ("en", "plain", "order_status", None, "neg", "Ordered on [[3/7|DATE]], still waiting on [[TW-119880|ORDER_ID]]. SwiftDrop says 'exception'. What does that mean?", None),
 ("en", "plain", "product_question", None, "neu", "Do you price match? Saw the [[Volt 2|PRODUCT]] cheaper elsewhere.", None),
 ("en", "sarcasm", "order_status", None, "neg", "Love how SwiftDrop left my new bike in the bin store. Really secure. Order [[TW-124118|ORDER_ID]].", None),
 ("en", "very_long", "returns_refund", None, "neu", "Hello, I bought the [[Harbour Kids 20|PRODUCT]] for my nephew's birthday on [[18 July|DATE]] during your summer sale and paid [[£279|MONEY]]. Unfortunately my sister had already bought him a bike, which nobody told me about until the party. The bike is still in its box, completely unused, and I have the receipt. I understand that sale items can sometimes be treated differently, so I wanted to check before I post it back whether I can still get a full refund, and whether you send a returns label or I need to arrange the courier myself. I would also like to know how long the refund normally takes to appear on my card. Thank you for your help. [[Peter Lowe|PERSON]]", None),
 ("en", "slang", "repair_booking", None, "neu", "brakes squealing like a banshee lol. book me in?", None),
 ("en", "plain", "spam", None, "neu", "We are a logistics company offering cheap shipping from overseas factories. Minimum order 500 units. Message us for prices.", None),
 ("en", "plain", "warranty_fault", None, "neg", "Box arrived soaked and the [[Causeway|PRODUCT]] rear mech is bent. Photos attached.", None),
 ("en", "very_short", "product_question", None, "neu", "kids helmets?", None),
 ("en", "plain", "praise", None, "pos", "Shout out to [[Callum|PERSON]] for talking me out of the expensive model. Saved me [[£500|MONEY]] and I love the bike.", None),
 ("en", "two_intents", "order_status", "product_question", "neu", "Has order [[TW-120577|ORDER_ID]] shipped yet? Also, does the [[Beacon Light Set|PRODUCT]] come with batteries?", None),
 ("en", "negation", "repair_booking", None, "neu", "This is not a complaint, the bike is great, I just need to know when my [[Full Service|PRODUCT]] slot is.", None),
 ("en", "very_short", "returns_refund", None, "neg", "Charged twice. [[£45|MONEY]] and [[£45|MONEY]]. Fix it.", None),
 ("en", "sarcasm", "warranty_fault", None, "neg", "Fourth puncture this month on the 'puncture resistant' tyres. Impressive.", None),
 ("en", "plain", "spam", None, "neu", "Earn passive income from your bikes! Tokenise your fleet with BikeCoin today.", None),
 ("en", "plain", "praise", None, "pos", "cheers for sorting the refund so quick, really appreciate it", None),
 ("en", "very_long", "order_status", None, "neg", "Hi there, I'm getting a bit worried about my order [[TW-123390|ORDER_ID]]. I ordered the [[Volt 2 Step-Through|PRODUCT]] on [[22 July|DATE]] in the sale, and the confirmation said it would arrive within five working days. On [[12 August|DATE]] I got a text from a courier I've never heard of, SwiftDrop, saying they had tried to deliver it, but I was at home all day and nobody knocked. Since then the tracking page just says 'awaiting next attempt'. I've tried calling the number on the text but it goes to a recorded message. Could you check with them what is going on and when I can expect it? I need it for my new job, which starts on [[1 September|DATE]]. Thanks, [[Nadia Hussein|PERSON]]", None),
 ("en", "slang", "repair_booking", None, "neu", "can i get a puncture fixed same day if i drop it in before 9?", None),
 ("de", "plain", "order_status", None, "neg", "Meine Bestellung [[TW-121554|ORDER_ID]] ist seit einer Woche unterwegs. Wo bleibt sie?", "My order TW-121554 has been on its way for a week. Where is it?"),
 ("de", "plain", "warranty_fault", None, "neg", "Der Akku von meinem [[Volt 2|PRODUCT]] lädt nicht mehr. Das Rad habe ich erst seit [[Juli|DATE]].", "The battery on my Volt 2 no longer charges. I have only had the bike since July."),
 ("de", "sarcasm", "order_status", None, "neg", "Toll, drei Zustellversuche und ich war jedes Mal zu Hause. Super Service von SwiftDrop.", "Great, three delivery attempts and I was at home every time. Super service from SwiftDrop."),
 ("de", "plain", "returns_refund", None, "neu", "Ich möchte das [[Commuter Pannier Set|PRODUCT]] zurückschicken. Wie bekomme ich ein Rücksendeetikett?", "I would like to send back the Commuter Pannier Set. How do I get a returns label?"),
 ("de", "plain", "product_question", None, "neu", "Kann man das [[Volt 2|PRODUCT]] auch in [[Deutschland|LOC]] reparieren lassen, oder muss es nach [[Bristol|LOC]]?", "Can the Volt 2 also be repaired in Germany, or does it have to go to Bristol?"),
 ("de", "code_switch", "repair_booking", None, "neu", "Hallo, mein [[Causeway|PRODUCT]] hat einen flat tyre, kann ich das bei euch fixen lassen oder soll ich es selbst machen?", "Hello, my Causeway has a flat tyre, can I get it fixed by you or should I do it myself?"),
 ("de", "plain", "praise", None, "pos", "Vielen Dank, das Rad ist heute angekommen und es ist wunderschön!", "Thank you very much, the bike arrived today and it is beautiful!"),
 ("de", "two_intents", "order_status", "warranty_fault", "neg", "Wo ist meine Bestellung [[TW-122960|ORDER_ID]]? Und ich will das [[LockSure D-lock|PRODUCT]] umtauschen, der Schlüssel klemmt.", "Where is my order TW-122960? And I want to exchange the LockSure D-lock, the key sticks."),
 ("de", "very_short", "order_status", None, "neu", "Rechnung?", "Invoice?"),
 ("de", "plain", "repair_booking", None, "neu", "Können Sie mein E-Bike am [[Samstag|DATE]] durchsehen? Die Bremsen schleifen.", "Can you look over my e-bike on Saturday? The brakes are rubbing."),
 ("de", "very_long", "warranty_fault", None, "neu", "Guten Tag, ich habe am [[14. Juli|DATE]] ein [[Volt 2 Step-Through|PRODUCT]] bei Ihnen gekauft. Seit gestern zeigt das Display den Fehler E07 und der Motor unterstützt nicht mehr. Ich habe den Akku vollständig geladen und das Rad neu gestartet, aber nichts hat geholfen. Bitte teilen Sie mir mit, wie die Garantie funktioniert und ob ich das Rad nach [[Großbritannien|LOC]] schicken muss. Mit freundlichen Grüßen, [[Sabine Krause|PERSON]]", "Good afternoon, on 14 July I bought a Volt 2 Step-Through from you. Since yesterday the display has shown error E07 and the motor no longer assists. I have fully charged the battery and restarted the bike, but nothing has helped. Please tell me how the warranty works and whether I have to send the bike to Great Britain. Kind regards, Sabine Krause"),
 ("de", "plain", "spam", None, "neu", "Sehr geehrte Damen und Herren, wir bieten Ihnen günstige Werbung auf 500 Fahrrad-Blogs an. Interesse?", "Dear Sir or Madam, we offer you cheap advertising on 500 cycling blogs. Interested?"),
 ("de", "slang", "warranty_fault", None, "neg", "Moin, Rad is da, aber der Lenker ist total schief montiert. Was nu?", "Hi, the bike is here, but the handlebars are fitted completely crooked. What now?"),
 ("de", "sarcasm", "returns_refund", None, "neg", "Super, die Erstattung von [[549 £|MONEY]] dauert jetzt schon drei Wochen. Ganz toll.", "Great, the refund of £549 has now taken three weeks. Just wonderful."),
 ("es", "plain", "order_status", None, "neu", "Hola, ¿cuándo llega mi pedido [[TW-121871|ORDER_ID]]? Ya ha pasado una semana.", "Hello, when will my order TW-121871 arrive? A week has already gone by."),
 ("es", "plain", "warranty_fault", None, "neg", "La batería de mi [[Volt 2|PRODUCT]] se agota en diez minutos. ¿Está en garantía?", "The battery of my Volt 2 runs out in ten minutes. Is it under warranty?"),
 ("es", "code_switch", "order_status", None, "neu", "Hi, my order llegó pero falta el cargador. Can you send it?", "Hi, my order arrived but the charger is missing. Can you send it?"),
 ("es", "plain", "returns_refund", None, "neu", "Quiero devolver la [[Harbour|PRODUCT]], es demasiado grande para mí. ¿Me devolvéis el dinero?", "I want to return the Harbour, it is too big for me. Will you give me my money back?"),
 ("es", "sarcasm", "order_status", None, "neg", "Genial, el repartidor dejó la bici en la calle bajo la lluvia. Gracias, de verdad.", "Great, the delivery driver left the bike in the street in the rain. Thanks, really."),
 ("es", "plain", "product_question", None, "neu", "¿La [[Volt 2 Step-Through|PRODUCT]] sirve para alguien de 1,55 m?", "Is the Volt 2 Step-Through suitable for someone who is 1.55 m tall?"),
 ("es", "plain", "repair_booking", None, "neu", "Necesito cambiar las pastillas de freno. ¿Tenéis hueco el [[jueves|DATE]]?", "I need to change the brake pads. Do you have a slot on Thursday?"),
 ("es", "plain", "praise", None, "pos", "¡Muchísimas gracias a [[Ines|PERSON]] por responder en español! Todo resuelto.", "Many thanks to Ines for replying in Spanish! All sorted."),
 ("es", "two_intents", "order_status", "returns_refund", "neg", "Mi pedido [[TW-122204|ORDER_ID]] no ha llegado y además quiero cancelar el [[Beacon Light Set|PRODUCT]] que pedí aparte.", "My order TW-122204 has not arrived and I also want to cancel the Beacon Light Set that I ordered separately."),
 ("es", "very_short", "product_question", None, "neu", "¿Hacéis envíos a [[Canarias|LOC]]?", "Do you ship to the Canary Islands?"),
 ("es", "plain", "spam", None, "neu", "Gana [[5.000 euros|MONEY]] al mes desde casa. Haz clic aquí.", "Earn 5,000 euros a month from home. Click here."),
 ("es", "very_long", "warranty_fault", None, "neg", "Buenas tardes. Compré una [[Causeway|PRODUCT]] el [[20 de julio|DATE]] y me llegó el [[29 de julio|DATE]]. Desde el primer día el cambio trasero hace un ruido muy raro y las marchas saltan cuando subo cuestas. La he llevado a una tienda de bicis aquí en [[Valencia|LOC]] y me dicen que el desviador viene doblado de fábrica. No quiero pagar una reparación por algo que vino mal. ¿Me podéis decir qué hago? ¿La tengo que enviar a [[Inglaterra|LOC]]? Un saludo, [[Marta Gil|PERSON]]", "Good afternoon. I bought a Causeway on 20 July and it arrived on 29 July. From the first day the rear gears have made a very strange noise and the gears slip when I ride uphill. I have taken it to a bike shop here in Valencia and they tell me the derailleur came bent from the factory. I don't want to pay for a repair for something that arrived faulty. Can you tell me what to do? Do I have to send it to England? Best wishes, Marta Gil"),
 ("es", "slang", "praise", None, "neu", "Tío, la bici mola mazo, pero el sillín es durísimo jaja", "Mate, the bike is really cool, but the saddle is rock hard haha"),
 ("fr", "plain", "order_status", None, "neg", "Bonjour, ma commande [[TW-122691|ORDER_ID]] est bloquée en transit depuis six jours. Pouvez-vous vérifier ?", "Hello, my order TW-122691 has been stuck in transit for six days. Can you check?"),
 ("fr", "plain", "warranty_fault", None, "neg", "Mon [[Volt 2|PRODUCT]] affiche l'erreur E07 et le moteur ne marche plus.", "My Volt 2 shows error E07 and the motor no longer works."),
 ("fr", "sarcasm", "order_status", None, "neg", "Bravo, livré chez le voisin du dessus sans me prévenir. Très pro.", "Well done, delivered to the upstairs neighbour without telling me. Very professional."),
 ("fr", "plain", "returns_refund", None, "neu", "Je souhaite renvoyer le [[LockSure D-lock|PRODUCT]] : il est trop lourd pour moi. Comment faire ?", "I would like to send back the LockSure D-lock: it is too heavy for me. How do I do that?"),
 ("fr", "plain", "product_question", None, "neu", "Le [[Harbour|PRODUCT]] est-il livré monté ou faut-il l'assembler ?", "Does the Harbour come assembled or does it need to be put together?"),
 ("fr", "code_switch", "warranty_fault", None, "neu", "Salut, the bike est arrivé mais the front light ne s'allume pas. C'est normal ?", "Hi, the bike has arrived but the front light does not switch on. Is that normal?"),
 ("fr", "plain", "repair_booking", None, "neu", "Est-ce que je peux prendre rendez-vous pour un [[Full Service|PRODUCT]] le [[3 septembre|DATE]] ?", "Can I make an appointment for a Full Service on 3 September?"),
 ("fr", "plain", "praise", None, "pos", "Merci à toute l'équipe, le vélo est parfait et la livraison à [[Lyon|LOC]] a été rapide.", "Thanks to the whole team, the bike is perfect and the delivery to Lyon was quick."),
 ("fr", "very_short", "returns_refund", None, "neg", "Remboursement ???", "Refund ???"),
 ("fr", "two_intents", "warranty_fault", "product_question", "neg", "Ma batterie ne charge plus et je voudrais aussi savoir si vous vendez une deuxième batterie ([[Volt 2 battery pack (500 Wh)|PRODUCT]]).", "My battery no longer charges and I would also like to know whether you sell a second battery (Volt 2 battery pack (500 Wh))."),
 ("fr", "plain", "spam", None, "neu", "Félicitations ! Vous avez gagné un téléphone neuf. Cliquez ici pour le recevoir.", "Congratulations! You have won a new phone. Click here to receive it."),
 ("fr", "slang", "praise", None, "neu", "Franchement, le vélo est trop bien, mais le livreur a galéré pour trouver chez moi lol", "Honestly, the bike is great, but the delivery driver struggled to find my place lol"),
 ("fr", "very_long", "warranty_fault", None, "neu", "Bonjour, j'ai acheté un [[Volt 2 Step-Through|PRODUCT]] le [[16 juillet|DATE]] pendant les soldes, pour [[1 999 £|MONEY]]. Depuis une semaine, la batterie se vide très vite : je fais à peine dix kilomètres alors qu'avant j'en faisais cinquante. J'ai lu sur votre site qu'il y avait un problème avec certains lots de batteries, mais le numéro sur la mienne n'est pas VB-0426. Est-ce que ma batterie est quand même couverte par la garantie, et comment faire pour l'échanger depuis la [[France|LOC]] ? Cordialement, [[Hélène Rousseau|PERSON]]", "Hello, I bought a Volt 2 Step-Through on 16 July during the sale, for £1,999. For a week the battery has been running down very quickly: I barely manage ten kilometres when before I did fifty. I read on your website that there was a problem with some battery batches, but the number on mine is not VB-0426. Is my battery still covered by the warranty, and how do I exchange it from France? Kind regards, Hélène Rousseau"),
]
HARD_TEST = []
for k, (lang, kind, intent, second, sent, marked, en) in enumerate(HARD, 1):
    text, spans = unmark(marked)
    HARD_TEST.append({"id": f"HT{k:02d}", "language": lang, "kind": kind, "intent": intent, "second_intent": second,
                      "sentiment": sent, "text": text, "entities": spans, "en_ref": en})

from collections import Counter
print(len(HARD_TEST), "hand-written messages:", dict(Counter(h["language"] for h in HARD_TEST)))
print(dict(Counter(h["intent"] for h in HARD_TEST)))
print(dict(Counter(h["kind"] for h in HARD_TEST)))
print(HARD_TEST[0]["text"], HARD_TEST[0]["entities"])

added to inbox/data.py

import hashlib, json
from inbox.data import MESSAGES, HARD_TEST, LABELS


def fingerprint(*datasets):
    """The first 12 hex digits of a SHA-256 hash of the data: equal fingerprints mean identical data."""
    blob = json.dumps(datasets, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8")
    return hashlib.sha256(blob).hexdigest()[:12]


print(len(MESSAGES), "messages,", len(HARD_TEST), "hand-written,", len(LABELS), "intents")
print("messages ", fingerprint(MESSAGES))
print("hard_test", fingerprint(HARD_TEST))

14 products, 60 customers, 7 intents
women: 30 men: 30
{'en': 36, 'de': 8, 'es': 8, 'fr': 8}
182 days from the first message day to DATA_AS_OF
35 wordings, 18 of them in all four languages
1800 messages; 2026-03-02 to 2026-08-30
TW-M0001 | Web form: Workshop | <p>Mon vélo (Causeway) a besoin d&#x27;une réparation : les plaquettes de frei
80 hand-written messages: {'en': 40, 'de': 14, 'es': 13, 'fr': 13}
{'warranty_fault': 16, 'order_status': 18, 'returns_refund': 10, 'repair_booking': 10, 'praise': 10, 'product_question': 9, 'spam': 7}
{'sarcasm': 9, 'two_intents': 6, 'very_short': 8, 'slang': 7, 'plain': 37, 'negation': 2, 'very_long': 6, 'codes': 1, 'mixed': 1, 'code_switch': 3}
Oh brilliant, the Volt 2 battery died halfway up the hill again. Really loving my £2,199 paperweight. [(18, 24, 'PRODUCT'), (82, 88, 'MONEY')]
1800 messages, 80 hand-written, 7 intents
messages  294611361162
hard_test 63c80b0396bc