Lesson 2: Characters, encodings and clean inbox text

%pip install -q ftfy==6.3.1 pytest==9.1.1

from datetime import date, timedelta

COURSE_DATE = date(2026, 10, 14)   # facts, versions and model names checked on this day
DATA_AS_OF = date(2026, 8, 31)     # the inbox snapshot: count days to this, never to the clock
WINDOW = (date(2026, 3, 2), date(2026, 8, 30))   # 26 weeks of messages, Monday to Sunday
SEED = 20260831

EVENTS = {   # fixed dates the data and the lessons share
    "spam_wave": (date(2026, 4, 6), date(2026, 4, 19)),
    "battery_notice": date(2026, 6, 9),      # email to Volt 2 owners about pack batch VB-0426
    "eu_shipping": date(2026, 7, 1),          # Germany, Spain and France
    "summer_sale": (date(2026, 7, 15), date(2026, 7, 28)),
    "carrier_change": date(2026, 8, 10),      # ParcelNorth -> SwiftDrop
}
BATCH = "VB-0426"
STAFF = ["Rosa", "Ines", "Dev", "Callum", "Amara"]

PRODUCTS = [   # (sku, name, price in GBP, kind)
    ("TW-VOLT2-BLK", "Volt 2", 2199, "ebike"),
    ("TW-VOLT2-STP", "Volt 2 Step-Through", 2249, "ebike"),
    ("TW-HARB-GRN", "Harbour", 649, "bike"),
    ("TW-HKID-20R", "Harbour Kids 20", 329, "bike"),
    ("TW-CAUS-GRY", "Causeway", 1149, "bike"),
    ("TW-PANN-SET", "Commuter Pannier Set", 89, "accessory"),
    ("TW-BATT-500", "Volt 2 battery pack (500 Wh)", 549, "accessory"),
    ("TW-LOCK-DLK", "LockSure D-lock", 45, "accessory"),
    ("TW-LITE-BCN", "Beacon Light Set", 39, "accessory"),
    ("TW-SERV-BAS", "Basic Service", 65, "service"),
    ("TW-SERV-FUL", "Full Service", 129, "service"),
    ("TW-REPR-PUN", "Puncture Repair", 18, "service"),
    ("TW-DIAG-EBK", "E-bike Diagnostic", 45, "service"),
    ("TW-GIFT-CRD", "Gift Card", 50, "gift"),          # sold from 25 to 200 GBP; 50 is the usual
]
INTENTS = ["order_status", "returns_refund", "repair_booking", "product_question",
           "warranty_fault", "praise", "spam"]
LABELS = {   # Ines's written definitions, used for every label in the course
    "order_status": "Where an order is, delivery problems, missing items, invoices and order documents.",
    "returns_refund": "Sending something back, exchanges without a fault, cancellations, refunds and wrong charges.",
    "repair_booking": "Booking, moving or asking the price of a service or repair for wear and tear.",
    "product_question": "Questions before or after buying: sizes, stock, range, compatibility, delivery areas.",
    "warranty_fault": "A product that is faulty or arrived damaged, including the battery batch VB-0426.",
    "praise": "Thanks, compliments and good reviews, with no request that needs action.",
    "spam": "Unsolicited selling, phishing and scams.",
}

CUSTOMERS = [   # (first, last, gender, language, home town); all fictional
    ("Joan", "Hartley", "f", "en", "Clifton"), ("Oliver", "Bennett", "m", "en", "Bristol"),
    ("Amelia", "Clarke", "f", "en", "Bath"), ("Mohammed", "Khan", "m", "en", "Bristol"),
    ("Aisha", "Begum", "f", "en", "Bedminster"), ("Daniel", "O'Connor", "m", "en", "Bristol"),
    ("Priya", "Patel", "f", "en", "Swindon"), ("Arjun", "Sharma", "m", "en", "Bristol"),
    ("Grace", "Okafor", "f", "en", "Cardiff"), ("Emeka", "Eze", "m", "en", "Bristol"),
    ("Hannah", "Lewis", "f", "en", "Stroud"), ("Tom", "Hughes", "m", "en", "Gloucester"),
    ("Siobhan", "Murphy", "f", "en", "Bristol"), ("Liam", "Walsh", "m", "en", "Bath"),
    ("Mei", "Chen", "f", "en", "Bristol"), ("Wei", "Zhang", "m", "en", "Clifton"),
    ("Fatima", "Hussain", "f", "en", "Bristol"), ("Yusuf", "Ali", "m", "en", "Bedminster"),
    ("Zara", "Ahmed", "f", "en", "Taunton"), ("Jack", "Robinson", "m", "en", "Bristol"),
    ("Ella", "Morgan", "f", "en", "Cardiff"), ("Rhys", "Evans", "m", "en", "Cardiff"),
    ("Megan", "Price", "f", "en", "Weston-super-Mare"), ("Ben", "Carter", "m", "en", "Bristol"),
    ("Lucy", "Ward", "f", "en", "Bath"), ("Kofi", "Asante", "m", "en", "Bristol"),
    ("Ama", "Boateng", "f", "en", "Bristol"), ("Olusegun", "Adeyemi", "m", "en", "Swindon"),
    ("Funmi", "Adebayo", "f", "en", "Bristol"), ("Raj", "Gupta", "m", "en", "Gloucester"),
    ("Neha", "Kapoor", "f", "en", "Bristol"), ("Tomasz", "Nowak", "m", "en", "Bristol"),
    ("Agnieszka", "Kowalska", "f", "en", "Stroud"), ("Stefan", "Ionescu", "m", "en", "Bristol"),
    ("Ioana", "Popescu", "f", "en", "Bath"), ("Hiroshi", "Tanaka", "m", "en", "Bristol"),
    ("Lukas", "Becker", "m", "de", "Berlin"), ("Anna", "Schmidt", "f", "de", "Hamburg"),
    ("Jonas", "Weber", "m", "de", "München"), ("Lea", "Fischer", "f", "de", "Köln"),
    ("Mehmet", "Yılmaz", "m", "de", "Berlin"), ("Elif", "Demir", "f", "de", "Leipzig"),
    ("Felix", "Wagner", "m", "de", "Hamburg"), ("Sophie", "Müller", "f", "de", "München"),
    ("Javier", "García", "m", "es", "Madrid"), ("Lucía", "Fernández", "f", "es", "Sevilla"),
    ("Alejandro", "Martín", "m", "es", "Valencia"), ("Carmen", "Ruiz", "f", "es", "Madrid"),
    ("Mateo", "López", "m", "es", "Barcelona"), ("Valentina", "Torres", "f", "es", "Bilbao"),
    ("Youssef", "El Amrani", "m", "es", "Barcelona"), ("Nuria", "Puig", "f", "es", "Barcelona"),
    ("Hugo", "Martin", "m", "fr", "Paris"), ("Camille", "Dubois", "f", "fr", "Lyon"),
    ("Lucas", "Bernard", "m", "fr", "Nantes"), ("Chloé", "Lefèvre", "f", "fr", "Bordeaux"),
    ("Karim", "Benali", "m", "fr", "Lille"), ("Amina", "Diallo", "f", "fr", "Paris"),
    ("Théo", "Moreau", "m", "fr", "Lyon"), ("Léa", "Laurent", "f", "fr", "Nantes"),
]

print(len(PRODUCTS), "products,", len(CUSTOMERS), "customers,", len(INTENTS), "intents")
print("women:", sum(c[2] == "f" for c in CUSTOMERS), "men:", sum(c[2] == "m" for c in CUSTOMERS))
print({lang: sum(c[3] == lang for c in CUSTOMERS) for lang in ["en", "de", "es", "fr"]})
print((DATA_AS_OF - WINDOW[0]).days, "days from the first message day to DATA_AS_OF")

14 products, 60 customers, 7 intents
women: 30 men: 30
{'en': 36, 'de': 8, 'es': 8, 'fr': 8}
182 days from the first message day to DATA_AS_OF

import random, re

# Templates: one dict per wording. Keys en, de, es, fr hold the same message in each language
# (English-only wordings have just "en"). {UPPER} slots become gold entities; {lower} slots do not.
T = {
 "order_status": [
  {"s": "neu", "en": "Hi, I placed an order on {DATE} (order {ORDER_ID}, item: {PRODUCT}) and it still says processing. When will it ship?",
   "de": "Hallo, ich habe am {DATE} bestellt (Bestellung {ORDER_ID}, Artikel: {PRODUCT}) und der Status steht immer noch auf in Bearbeitung. Wann wird sie verschickt?",
   "es": "Hola, hice un pedido el {DATE} (pedido {ORDER_ID}, artículo: {PRODUCT}) y sigue en preparación. ¿Cuándo lo enviaréis?",
   "fr": "Bonjour, j'ai passé commande le {DATE} (commande {ORDER_ID}, article : {PRODUCT}) et elle est toujours en préparation. Quand sera-t-elle expédiée ?"},
  {"s": "neg", "en": "Order {ORDER_ID} was due on {DATE} and nothing has arrived. The {carrier} tracking has not changed for three days. Where is my parcel ({PRODUCT})?",
   "de": "Bestellung {ORDER_ID} sollte am {DATE} ankommen, aber es ist nichts da. Die Sendungsverfolgung von {carrier} hat sich seit drei Tagen nicht geändert. Wo ist mein Paket ({PRODUCT})?",
   "es": "El pedido {ORDER_ID} tenía que llegar el {DATE} y no ha llegado nada. El seguimiento de {carrier} no cambia desde hace tres días. ¿Dónde está mi paquete ({PRODUCT})?",
   "fr": "La commande {ORDER_ID} devait arriver le {DATE} et rien n'est arrivé. Le suivi {carrier} n'a pas bougé depuis trois jours. Où est mon colis ({PRODUCT}) ?"},
  {"s": "neu", "en": "Could you send me the tracking link for order {ORDER_ID}? The confirmation email never came through to {email}.",
   "de": "Könnten Sie mir den Tracking-Link für Bestellung {ORDER_ID} schicken? Die Bestätigung ist nie bei {email} angekommen.",
   "es": "¿Me podéis enviar el enlace de seguimiento del pedido {ORDER_ID}? El correo de confirmación nunca llegó a {email}.",
   "fr": "Pouvez-vous m'envoyer le lien de suivi de la commande {ORDER_ID} ? L'e-mail de confirmation n'est jamais arrivé sur {email}."},
  {"s": "neg", "from": "2026-08-10", "w": 3, "en": "Third time {carrier} has said 'attempted delivery' when I was in all day. Order {ORDER_ID}. This never happened with your old courier."},
  {"s": "neu", "en": "any update on {ORDER_ID}? need the {PRODUCT} before {DATE} if poss"},
 ],
 "returns_refund": [
  {"s": "neu", "en": "I'd like to return the {PRODUCT} from order {ORDER_ID}. It's unused and still in the box. How do I send it back?",
   "de": "Ich möchte {PRODUCT} aus Bestellung {ORDER_ID} zurückgeben. Unbenutzt und noch originalverpackt. Wie schicke ich es zurück?",
   "es": "Quiero devolver {PRODUCT} del pedido {ORDER_ID}. Está sin usar y en su caja. ¿Cómo lo envío?",
   "fr": "Je voudrais retourner {PRODUCT} de la commande {ORDER_ID}. Il n'a pas servi et il est encore dans sa boîte. Comment le renvoyer ?"},
  {"s": "neg", "en": "I sent the return for {ORDER_ID} back on {DATE} and still no refund of {MONEY}. How long does this take?",
   "de": "Ich habe die Rücksendung für {ORDER_ID} am {DATE} abgeschickt und noch keine Erstattung über {MONEY} bekommen. Wie lange dauert das?",
   "es": "Envié la devolución del pedido {ORDER_ID} el {DATE} y todavía no me habéis devuelto {MONEY}. ¿Cuánto tarda?",
   "fr": "J'ai renvoyé la commande {ORDER_ID} le {DATE} et je n'ai toujours pas reçu le remboursement de {MONEY}. Combien de temps faut-il ?"},
  {"s": "neu", "en": "The {KIDBIKE} is the wrong size for my daughter. Can I swap it or get a refund? Order {ORDER_ID}.",
   "de": "{KIDBIKE} ist für meine Tochter die falsche Größe. Kann ich umtauschen oder bekomme ich das Geld zurück? Bestellung {ORDER_ID}.",
   "es": "{KIDBIKE} es de una talla equivocada para mi hija. ¿Puedo cambiarlo o me devolvéis el dinero? Pedido {ORDER_ID}.",
   "fr": "{KIDBIKE} n'est pas à la bonne taille pour ma fille. Puis-je l'échanger ou être remboursé ? Commande {ORDER_ID}."},
  {"s": "neg", "en": "You charged me {MONEY} twice for order {ORDER_ID}. Please refund the duplicate payment."},
  {"s": "neu", "en": "want to cancel order {ORDER_ID} before it ships, changed my mind about the {PRODUCT}. refund to my card pls"},
 ],
 "repair_booking": [
  {"s": "neu", "en": "Can I book a {SERVICE} for {DATE}? It's for my bike ({BIKE}). I can drop it at the workshop in the morning.",
   "de": "Kann ich für den {DATE} einen Termin für {SERVICE} buchen? Es geht um mein Rad ({BIKE}). Ich kann es morgens in die Werkstatt bringen.",
   "es": "¿Puedo reservar {SERVICE} para el {DATE}? Es para mi bici ({BIKE}). Puedo dejarla en el taller por la mañana.",
   "fr": "Puis-je réserver {SERVICE} pour le {DATE} ? C'est pour mon vélo ({BIKE}). Je peux le déposer à l'atelier le matin."},
  {"s": "neu", "en": "My bike ({BIKE}) needs a repair: {wear}. Can I bring it in or send it to you? I live in {LOC}.",
   "de": "Mein Rad ({BIKE}) muss repariert werden: {wear}. Kann ich es vorbeibringen oder einschicken? Ich wohne in {LOC}.",
   "es": "Mi bici ({BIKE}) necesita una reparación: {wear}. ¿Puedo llevarla o enviárosla? Vivo en {LOC}.",
   "fr": "Mon vélo ({BIKE}) a besoin d'une réparation : {wear}. Puis-je l'apporter ou vous l'envoyer ? J'habite à {LOC}."},
  {"s": "neu", "en": "Please can I move my {SERVICE} booking ({ticket}) from {DATE} to the following week? Something came up.",
   "de": "Kann ich meinen Termin für {SERVICE} ({ticket}) vom {DATE} auf die Woche danach verschieben? Mir ist etwas dazwischengekommen.",
   "es": "¿Puedo cambiar mi cita de {SERVICE} ({ticket}) del {DATE} a la semana siguiente? Me ha surgido algo.",
   "fr": "Puis-je déplacer mon rendez-vous {SERVICE} ({ticket}) du {DATE} à la semaine suivante ? J'ai un empêchement."},
  {"s": "neu", "en": "hiya, how much is a {SERVICE} and can u fit me in {DATE}? bike is a {BIKE}, cheers"},
  {"s": "neu", "en": "I'd like to book the {SERVICE} for my {BIKE}. Is {MONEY} still the price? Any day after {DATE} works."},
 ],
 "product_question": [
  {"s": "neu", "en": "Does the {BIKE} come in a smaller frame? I'm 158 cm tall.",
   "de": "Gibt es {BIKE} auch mit einem kleineren Rahmen? Ich bin 1,58 m groß.",
   "es": "¿{BIKE} viene en una talla de cuadro más pequeña? Mido 1,58 m.",
   "fr": "{BIKE} existe-t-il avec un cadre plus petit ? Je mesure 1,58 m."},
  {"s": "neu", "en": "What range does the {EBIKE} get on one charge, and is the {ACCESSORY} compatible with it?",
   "de": "Welche Reichweite hat {EBIKE} mit einer Akkuladung, und passt {ACCESSORY} dazu?",
   "es": "¿Qué autonomía tiene {EBIKE} con una carga, y es compatible con {ACCESSORY}?",
   "fr": "Quelle autonomie a {EBIKE} avec une charge, et {ACCESSORY} est-il compatible ?"},
  {"s": "neu", "en": "Is the {PRODUCT} in stock? The website says {MONEY} but I can't add it to the basket.",
   "de": "Ist {PRODUCT} auf Lager? Auf der Website steht {MONEY}, aber ich kann es nicht in den Warenkorb legen.",
   "es": "¿Tenéis {PRODUCT} en stock? En la web pone {MONEY} pero no puedo añadirlo a la cesta.",
   "fr": "Est-ce que {PRODUCT} est en stock ? Le site indique {MONEY} mais je n'arrive pas à l'ajouter au panier."},
  {"s": "neu", "en": "Do you deliver to {LOC}? And do bikes arrive fully built or do I need to assemble them?"},
  {"s": "neu", "en": "quick q: whats the difference between the {SERVICE} and the {SERVICE2}?"},
 ],
 "warranty_fault": [
  {"s": "neg", "en": "My e-bike ({EBIKE}) is only four months old and {fault}. Is this covered by the warranty?",
   "de": "Mein E-Bike ({EBIKE}) ist erst vier Monate alt und {fault}. Ist das von der Garantie abgedeckt?",
   "es": "Mi bici eléctrica ({EBIKE}) solo tiene cuatro meses y {fault}. ¿Lo cubre la garantía?",
   "fr": "Mon vélo électrique ({EBIKE}) n'a que quatre mois et {fault}. Est-ce couvert par la garantie ?"},
  {"s": "neg", "from": "2026-06-09", "w": 4, "en": "I got your email about battery batch {batch}. My pack has that code and {fault}. What happens now?",
   "de": "Ich habe Ihre E-Mail zur Akku-Charge {batch} bekommen. Mein Akku hat diesen Code und {fault}. Wie geht es jetzt weiter?",
   "es": "Recibí vuestro correo sobre el lote de baterías {batch}. Mi batería tiene ese código y {fault}. ¿Qué hago ahora?",
   "fr": "J'ai reçu votre e-mail sur le lot de batteries {batch}. Ma batterie porte ce code et {fault}. Que se passe-t-il maintenant ?"},
  {"s": "neg", "en": "The {PRODUCT} arrived with {damage}. I've attached photos. Can you send a replacement?",
   "de": "{PRODUCT} kam mit {damage} an. Fotos sind im Anhang. Können Sie Ersatz schicken?",
   "es": "{PRODUCT} llegó con {damage}. Adjunto fotos. ¿Podéis enviarme uno nuevo?",
   "fr": "{PRODUCT} est arrivé avec {damage}. Je joins des photos. Pouvez-vous m'en envoyer un autre ?"},
  {"s": "neg", "en": "Spoke snapped on my {BIKE} after two rides. That's not normal wear, surely? Bought {DATE}, order {ORDER_ID}."},
  {"s": "neg", "en": "{EBIKE} battery dead again. Second pack in a month. I want this sorted properly this time."},
 ],
 "praise": [
  {"s": "pos", "en": "Just wanted to say thank you to {PERSON} in the workshop. My bike ({BIKE}) rides like new after the {SERVICE}.",
   "de": "Ich wollte mich nur bei {PERSON} in der Werkstatt bedanken. Mein Rad ({BIKE}) fährt sich nach {SERVICE} wie neu.",
   "es": "Solo quería dar las gracias a {PERSON} del taller. Mi bici ({BIKE}) va como nueva después de {SERVICE}.",
   "fr": "Je voulais juste remercier {PERSON} de l'atelier. Mon vélo ({BIKE}) roule comme neuf après {SERVICE}."},
  {"s": "pos", "en": "Love my new bike ({BIKE}). Fast delivery to {LOC} and it was easy to set up. Five stars.",
   "de": "Ich liebe mein neues Rad ({BIKE}). Schnelle Lieferung nach {LOC} und leicht aufzubauen. Fünf Sterne.",
   "es": "Me encanta mi nueva bici ({BIKE}). Entrega rápida a {LOC} y fácil de montar. Cinco estrellas.",
   "fr": "J'adore mon nouveau vélo ({BIKE}). Livraison rapide à {LOC} et facile à monter. Cinq étoiles."},
  {"s": "pos", "en": "Great service from {PERSON}, who sorted my problem in one email. Thank you!",
   "de": "Super Service von {PERSON}, das Problem war mit einer E-Mail gelöst. Vielen Dank!",
   "es": "Un servicio genial de {PERSON}, que me lo resolvió en un solo correo. ¡Gracias!",
   "fr": "Super service de la part de {PERSON}, qui a réglé mon problème en un seul e-mail. Merci !"},
  {"s": "pos", "en": "the {ACCESSORY} is brilliant, fits perfectly. cheers"},
  {"s": "pos", "en": "Honestly the best bike shop in {LOC}. Friendly, quick and fair prices."},
 ],
 "spam": [
  {"s": "neu", "en": "Dear partner, we offer guaranteed first-page search ranking for your online store. Reply today for a {MONEY} discount."},
  {"s": "neu", "en": "Exclusive partnership opportunity: invest in our crypto bike-sharing token and earn 300% returns. Contact {email} now."},
  {"s": "neu", "en": "Your mailbox storage is full. Verify your account within 24 hours at http://mail-verify.example/login to avoid suspension."},
  {"s": "neu", "en": "Hello, I am an influencer with 50k followers. Send me a free {BIKE} and I will post about your brand."},
  {"s": "neu", "en": "Congratulations! You have been selected for a {MONEY} gift voucher. Click http://prize-claim.example to claim."},
 ],
}
PHRASES = {   # paired fillers for the {lower} slots: (en, de, es, fr)
 "fault": [("the battery will not charge", "der Akku lädt nicht mehr", "la batería no carga", "la batterie ne se charge plus"),
           ("the motor cuts out on hills", "der Motor setzt am Berg aus", "el motor se para en las cuestas", "le moteur se coupe dans les montées"),
           ("the display shows error E07", "das Display zeigt Fehler E07", "la pantalla muestra el error E07", "l'écran affiche l'erreur E07"),
           ("the battery runs flat very quickly", "der Akku ist sehr schnell leer", "la batería se agota muy rápido", "la batterie se vide très vite")],
 "wear": [("the back tyre has a puncture", "der Hinterreifen hat einen Platten", "la rueda trasera está pinchada", "le pneu arrière est crevé"),
          ("the gears keep slipping", "die Gangschaltung springt ständig", "los cambios saltan todo el rato", "les vitesses sautent sans arrêt"),
          ("the brake pads are worn out", "die Bremsbeläge sind abgenutzt", "las pastillas de freno están gastadas", "les plaquettes de frein sont usées"),
          ("the chain squeaks", "die Kette quietscht", "la cadena chirría", "la chaîne grince")],
 "damage": [("a cracked mudguard", "einem gebrochenen Schutzblech", "un guardabarros roto", "un garde-boue fendu"),
            ("a bent front wheel", "einem verbogenen Vorderrad", "la rueda delantera doblada", "une roue avant voilée"),
            ("a broken clip", "einem kaputten Clip", "un clip roto", "un clip cassé")],
}
SUBJECTS = {"order_status": ["Where is my order?", "Wo bleibt meine Bestellung?", "¿Dónde está mi pedido?", "Où est ma commande ?"],
            "returns_refund": ["Return and refund", "Rücksendung", "Devolución", "Retour"],
            "repair_booking": ["Booking a service", "Werkstatttermin", "Cita en el taller", "Rendez-vous atelier"],
            "product_question": ["Product question", "Frage zum Produkt", "Pregunta sobre un producto", "Question sur un produit"],
            "warranty_fault": ["Fault with my bike", "Defekt", "Avería", "Panne"],
            "praise": ["Thank you!", "Danke!", "¡Gracias!", "Merci !"],
            "spam": ["Partnership opportunity"] * 4}
FORM_TOPIC = {"order_status": "Order enquiry", "returns_refund": "Returns", "repair_booking": "Workshop",
              "product_question": "Product question", "warranty_fault": "Faulty item", "praise": "Feedback", "spam": "Other"}
CHANNELS = {"order_status": (5, 4, 1), "returns_refund": (6, 4, 0), "repair_booking": (4, 6, 0),   # email, web form, review
            "product_question": (4, 6, 0), "warranty_fault": (5, 3, 2), "praise": (3, 1, 6), "spam": (1, 0, 0)}
LANGS = ["en", "de", "es", "fr"]
MONTHS = {"en": "January February March April May June July August September October November December",
          "de": "Januar Februar März April Mai Juni Juli August September Oktober November Dezember",
          "es": "enero febrero marzo abril mayo junio julio agosto septiembre octubre noviembre diciembre",
          "fr": "janvier février mars avril mai juin juillet août septembre octobre novembre décembre"}
TOWNS = {"en": ["Bristol", "Bath", "Clifton", "Cardiff", "Stroud"], "de": ["Berlin", "Hamburg", "München", "Köln"],
         "es": ["Madrid", "Barcelona", "Valencia", "Sevilla"], "fr": ["Paris", "Lyon", "Nantes", "Lille"]}
SPAMMERS = [("Growth Partners", "offers@growth-partners.example"), ("Crypto Ride", "team@cryptoride.example"),
            ("Mail Admin", "admin@mail-verify.example"), ("Prize Desk", "win@prize-claim.example")]


def fmt_date(d, lang, rng):
    m = MONTHS[lang].split()[d.month - 1]
    styles = {"en": [f"{d.day} {m}", f"{d.day:02d}/{d.month:02d}", f"{m} {d.day}{'th' if 10 < d.day < 14 else {1: 'st', 2: 'nd', 3: 'rd'}.get(d.day % 10, 'th')}"],
              "de": [f"{d.day}. {m}", f"{d.day:02d}.{d.month:02d}."], "es": [f"{d.day} de {m}", f"{d.day}/{d.month}"],
              "fr": [f"{d.day} {m}", f"{d.day:02d}/{d.month:02d}/{d.year}"]}[lang]
    return rng.choice(styles) if rng else styles[0]


def fmt_money(gbp, lang, rng):
    styles = {"en": [f"£{gbp:,}", f"£{gbp:,}.00", f"{gbp} pounds"], "de": [f"{gbp:,} £".replace(",", "."), f"{gbp} Pfund"],
              "es": [f"{gbp:,} libras".replace(",", "."), f"{gbp} £"], "fr": [f"{gbp:,} £".replace(",", " "), f"{gbp} livres"]}[lang]
    return rng.choice(styles) if rng else styles[0]


def typo(word, rng):
    i = rng.randrange(1, len(word) - 1)
    return rng.choice([word[:i] + word[i + 1] + word[i] + word[i + 2:], word[:i] + word[i + 1:], word[:i] + word[i] + word[i:]])


print(sum(len(v) for v in T.values()), "wordings,", sum("de" in t for v in T.values() for t in v), "of them in all four languages")

35 wordings, 18 of them in all four languages

import html, unicodedata

POOLS = {"PRODUCT": ["bike", "ebike", "accessory"], "BIKE": ["bike", "ebike"], "EBIKE": ["ebike"],
         "ACCESSORY": ["accessory"], "SERVICE": ["service"], "SERVICE2": ["service"], "KIDBIKE": ["kids"]}
EXTRA = {"greet": {"en": ["Hi,", "Hello,", "Hello Tidewell,", "Dear Sir or Madam,"], "de": ["Hallo,", "Guten Tag,"],
                   "es": ["Hola,", "Buenos días,"], "fr": ["Bonjour,", "Madame, Monsieur,"]},
         "sign": {"en": ["Thanks,", "Many thanks,", "Cheers,"], "de": ["Danke,", "Viele Grüße,"],
                  "es": ["Gracias,", "Un saludo,"], "fr": ["Merci,", "Cordialement,"]},
         "phone": {"en": "You can call me on ", "de": "Sie erreichen mich unter ", "es": "Me podéis llamar al ",
                   "fr": "Vous pouvez m'appeler au "},
         "quote": {"en": ("On ", ", Tidewell Cycles <help@tidewell.example> wrote:"), "de": ("Am ", " schrieb Tidewell Cycles <help@tidewell.example>:"),
                   "es": ("El ", ", Tidewell Cycles <help@tidewell.example> escribió:"), "fr": ("Le ", ", Tidewell Cycles <help@tidewell.example> a écrit :")},
         "again": {"en": "Sorry, sending this again as I have not heard back. ", "de": "Entschuldigung, ich schicke das noch einmal. ",
                   "es": "Perdón, lo envío otra vez porque no he recibido respuesta. ", "fr": "Désolé, je renvoie ce message car je n'ai pas eu de réponse. "}}
LABEL_OF = {"PRODUCT": "PRODUCT", "BIKE": "PRODUCT", "EBIKE": "PRODUCT", "ACCESSORY": "PRODUCT", "SERVICE": "PRODUCT",
            "SERVICE2": "PRODUCT", "KIDBIKE": "PRODUCT", "ORDER_ID": "ORDER_ID", "DATE": "DATE", "MONEY": "MONEY",
            "LOC": "LOC", "PERSON": "PERSON"}


def fill(template, values, lang, rng):
    """Template text -> list of (text, label or None) pieces, formatting dates and money for `lang`."""
    pieces = []
    for part in re.split(r"(\{\w+\})", template):
        if not (part.startswith("{") and part.endswith("}")):
            pieces.append((part, None)) if part else None
            continue
        slot = part[1:-1]
        v = values[slot]
        if slot == "DATE":
            v = fmt_date(v, lang, rng)
        elif slot == "MONEY":
            v = fmt_money(v, lang, rng)
        elif slot in ("fault", "wear", "damage"):
            v = PHRASES[slot][v][LANGS.index(lang)]
        pieces.append((v, LABEL_OF.get(slot)))
    return pieces


def email_of(first, last):
    """'Mehmet', 'Yılmaz' -> 'mehmet.yilmaz@...'; one fixed address per customer, on a reserved .example domain."""
    plain = unicodedata.normalize("NFKD", f"{first}.{last}".lower().replace("ı", "i").replace(" ", "").replace("'", ""))
    plain = "".join(ch for ch in plain if ch.isascii())
    return plain + "@" + ["post.example", "mailbox.example", "inbox.example"][len(first + last) % 3]


def make_messages(n=1800, seed=SEED):
    rng = random.Random(seed)
    days = [WINDOW[0] + timedelta(i) for i in range((WINDOW[1] - WINDOW[0]).days + 1)]
    base = {"order_status": 26, "returns_refund": 16, "repair_booking": 17, "product_question": 18,
            "warranty_fault": 11, "praise": 6, "spam": 6}

    def weights(d):
        w = dict(base)
        w["spam"] *= 8 if EVENTS["spam_wave"][0] <= d <= EVENTS["spam_wave"][1] else 1
        w["warranty_fault"] *= 3.5 if EVENTS["battery_notice"] <= d <= date(2026, 7, 31) else 2 if d.month == 8 else 1
        w["order_status"] *= (2.5 if EVENTS["summer_sale"][0] <= d <= EVENTS["summer_sale"][1] else 1) * (1.6 if d >= EVENTS["carrier_change"] else 1)
        w["returns_refund"] *= 1.3 if date(2026, 7, 29) <= d <= date(2026, 8, 15) else 1
        vol = (0.6 if d.weekday() >= 5 else 1.0) * (1.5 if d >= EVENTS["eu_shipping"] else 1.0)
        return [w[i] * vol for i in INTENTS]

    W = [weights(d) for d in days]
    n_dup = round(n * 0.03)
    picks = sorted(rng.choices(range(len(days)), weights=[sum(w) for w in W], k=n - n_dup))
    by_kind = {k: [p for p in PRODUCTS if p[3] == k] for k in ["bike", "ebike", "accessory", "service"]}
    by_kind["kids"] = [p for p in PRODUCTS if p[1] == "Harbour Kids 20"]
    out = []
    for di in picks:
        d = days[di]
        intent = rng.choices(INTENTS, weights=W[di])[0]
        lang = "en" if intent == "spam" else rng.choices(LANGS, weights=[95, 2, 1.5, 1.5] if d < EVENTS["eu_shipping"] else [58, 16, 14, 12])[0]
        options = [t for t in T[intent] if lang in t and date.fromisoformat(t.get("from", "2026-01-01")) <= d]
        tmpl = rng.choices(options, weights=[t.get("w", 1) for t in options])[0]
        if intent == "spam":
            cust, email = rng.choice(SPAMMERS)
            first, last, gender, town = cust, "", None, None
        else:
            first, last, gender, _, town = rng.choice([c for c in CUSTOMERS if c[3] == lang])
            email = email_of(first, last)
        v = {}
        for slot, kinds in POOLS.items():
            pool = [p for k in kinds for p in by_kind[k]]
            pick = rng.choice([p for p in pool if p[1] != v.get("SERVICE")])
            v[slot] = pick[1]
            v[slot + "_sku"], v[slot + "_price"] = pick[0], pick[2]
        main = next((s for s in POOLS if "{" + s + "}" in tmpl["en"]), None)
        future = intent == "repair_booking" or "before {DATE}" in tmpl["en"]
        v["DATE"] = d + timedelta(rng.randint(2, 14)) if future else d - timedelta(rng.randint(3, 20))
        v["MONEY"] = v[main + "_price"] if main else rng.choice([50, 100, 250, 500])
        v["ORDER_ID"] = f"TW-{100000 + di * 120 + rng.randrange(120)}"
        lives_there = "live in {LOC}" in tmpl["en"] or "shop in {LOC}" in tmpl["en"]
        v["LOC"] = town if lives_there and town else rng.choice(TOWNS[lang])
        v["PERSON"] = rng.choice(["Dev", "Dev", "Ines", "Callum"]) if intent == "praise" else first
        v.update(carrier="SwiftDrop" if d >= EVENTS["carrier_change"] else "ParcelNorth", email=email, batch=BATCH,
                 ticket=f"RJ-2026-{300 + di * 2 + rng.randrange(3):04d}", fault=rng.randrange(4), wear=rng.randrange(4),
                 damage=rng.randrange(3))
        core = fill(tmpl[lang], v, lang, rng)
        en_ref = "".join(t for t, _ in fill(tmpl["en"], v, "en", None)) if lang != "en" else None
        channel = rng.choices(["email", "web form", "review"], weights=CHANNELS[intent])[0]
        channel = "web form" if channel == "review" and tmpl["s"] == "neu" else channel   # reviews praise or complain
        pieces, g = list(core), 0
        if channel == "email":
            if rng.random() < 0.7:
                pieces, g = [(rng.choice(EXTRA["greet"][lang]) + "\n\n", None)] + pieces, 1
            if rng.random() < 0.15:
                pieces += [("\n" + EXTRA["phone"][lang] + rng.choice(["07700 900", "07700900", "+44 7700 900"]) + f"{rng.randrange(1000):03d}.", None)]
            if lang == "en" and intent in ("repair_booking", "order_status") and rng.random() < 0.3:
                pieces += [(f"\nMy postcode is BS{rng.randint(1, 16)} {rng.randint(1, 9)}{rng.choice('ABDEFGHJLNPQRSTUWXYZ')}{rng.choice('ABDEFGHJLNPQRSTUWXYZ')}.", None)]
            if rng.random() < 0.7 and intent != "spam":
                pieces += [("\n\n" + rng.choice(EXTRA["sign"][lang]) + "\n", None), (first if rng.random() < 0.6 else f"{first} {last}", "PERSON")]
                if rng.random() < 0.15:
                    pieces += [("\n" + {"en": "Sent from my phone", "de": "Von meinem Handy gesendet", "es": "Enviado desde mi móvil", "fr": "Envoyé de mon téléphone"}[lang], None)]
            if rng.random() < 0.25 and intent != "spam":
                q = EXTRA["quote"][lang]
                prev = fmt_date(d - timedelta(rng.randint(1, 6)), lang, None) + (" de 2026" if lang == "es" else " 2026")
                pieces += [("\n\n> " + q[0], None), (prev, "DATE"), (q[1] + "\n> Thanks for your message. We will reply within one working day.", None)]
        r = rng.random
        if r() < 0.15:                                          # typos
            for _ in range(rng.randint(1, 2)):
                i = rng.randrange(len(pieces))
                if pieces[i][1] is None:
                    words = pieces[i][0].split(" ")
                    j = rng.randrange(len(words))
                    if len(words[j]) >= 5 and words[j].isalpha():
                        words[j] = typo(words[j], rng)
                        pieces[i] = (" ".join(words), None)
        case = r()
        if case < 0.03:
            pieces = [(t.upper(), l) for t, l in pieces]
        elif case < 0.09:
            pieces = [(t.lower(), l) for t, l in pieces]
        if r() < 0.08:                                          # emoji after the main text
            pieces.insert(g + len(core), (" " + rng.choice(["😡", "😤", "🚲"] if tmpl["s"] == "neg" else ["🙂", "👍", "🚲"]), None))
        if r() < 0.05:                                          # odd whitespace
            i = next((i for i, (t, l) in enumerate(pieces) if l is None and " " in t), None)
            if i is not None:
                t = pieces[i][0]
                pieces[i] = (t.replace(" ", " ", 1) if r() < 0.5 else t.replace(" ", " ​", 1), None)
        if channel == "web form" and r() < 0.6:                 # HTML from the form
            pieces = [("<p>", None)] + [(html.escape(t).replace("\n", "<br>\n"), None) if l is None else (t, l) for t, l in pieces] + [("</p>", None)]
        if r() < 0.04:                                          # mojibake: UTF-8 bytes read as cp1252
            try:
                pieces = [(t.encode("utf-8").decode("cp1252"), l) for t, l in pieces]
            except UnicodeDecodeError:
                pass
        body, spans, pos = "", [], 0
        for t, l in pieces:
            if l:
                spans.append((pos, pos + len(t), l))
            body, pos = body + t, pos + len(t)
        sent = tmpl["s"]
        stars = None
        if channel == "review":
            stars = {"pos": rng.choice([4, 5]), "neu": 3, "neg": rng.choice([1, 2])}[sent]
            if r() < 0.08:
                stars = {"pos": rng.choice([1, 2]), "neu": rng.choice([1, 5]), "neg": 5}[sent]
        subject = {"email": SUBJECTS[intent][LANGS.index(lang)], "web form": "Web form: " + FORM_TOPIC[intent],
                   "review": "Review: " + (v[main] if main else "Tidewell Cycles")}[channel]
        out.append({"date": d.isoformat(), "channel": channel, "language": lang, "customer": f"{first} {last}".strip(),
                    "gender": gender, "email": email, "subject": subject, "body": body, "intent": intent, "sentiment": sent,
                    "stars": stars, "entities": spans, "sku": v[main + "_sku"] if main else None,
                    "order_id": v["ORDER_ID"] if "{ORDER_ID}" in tmpl["en"] else None, "duplicate_of": None,
                    "core": "".join(t for t, _ in core), "en_ref": en_ref, "label_flipped": False})
    originals = [m for m in out if m["intent"] != "spam"]
    for m in rng.sample(originals, n_dup):                      # near-duplicates: the same customer writes again
        again, cut = EXTRA["again"][m["language"]], 3 if m["body"].startswith("<p>") else 0   # inside the form's HTML
        dup = dict(m, body=m["body"][:cut] + again + m["body"][cut:], entities=[(a + len(again), b + len(again), l) for a, b, l in m["entities"]],
                   date=min(date.fromisoformat(m["date"]) + timedelta(rng.randint(0, 2)), WINDOW[1]).isoformat(), duplicate_of=m)
        out.append(dup)
    out.sort(key=lambda m: (m["date"], m["duplicate_of"] is not None))
    for i, m in enumerate(out, 1):
        m["id"] = f"TW-M{i:04d}"
    for m in out:
        if m["duplicate_of"] is not None:
            m["duplicate_of"] = m["duplicate_of"]["id"]
    for m in rng.sample(out, round(n * 0.02)):                  # deliberate wrong labels, for error analysis
        m["intent"], m["label_flipped"] = rng.choice([i for i in INTENTS if i != m["intent"]]), True
    split_of = {}
    for intent in INTENTS:                                      # stratified 70 / 15 / 15, duplicates follow their original
        ids = [m["id"] for m in out if m["intent"] == intent and m["duplicate_of"] is None]
        rng.shuffle(ids)
        a, b = round(len(ids) * 0.70), round(len(ids) * 0.85)
        split_of.update({x: "train" if k < a else "dev" if k < b else "test" for k, x in enumerate(ids)})
    for m in out:
        m["split"] = split_of[m["duplicate_of"] or m["id"]]
        m["period"] = "before_july" if m["date"] < "2026-07-01" else "from_july"
    return out


MESSAGES = make_messages()
print(len(MESSAGES), "messages;", MESSAGES[0]["date"], "to", MESSAGES[-1]["date"])
print(MESSAGES[0]["id"], "|", MESSAGES[0]["subject"], "|", MESSAGES[0]["body"][:80].replace("\n", " "))

1800 messages; 2026-03-02 to 2026-08-30
TW-M0001 | Web form: Workshop | <p>Mon vélo (Causeway) a besoin d&#x27;une réparation : les plaquettes de frei

BY_ID = {m["id"]: m for m in MESSAGES}
chloe = BY_ID["TW-M1553"]
raw = chloe["body"]
print(chloe["date"], chloe["channel"], chloe["language"], chloe["intent"])
print(raw)
print(len(raw), "characters")

2026-08-14 email fr returns_refund
Harbour Kids 20 n'est pas à la bonne taille pour ma fille. Puis-je l'échanger ou être remboursé ? Commande TW-119891.
Vous pouvez m'appeler au +44 7700 900466.

Merci,
Chloé Lefèvre

> Le 9 août 2026, Tidewell Cycles <help@tidewell.example> a écrit :
> Thanks for your message. We will reply within one working day.
323 characters

for s in ["Chloé", "£", "€", "🚲"]:
    points = " ".join(f"U+{ord(ch):04X}" for ch in s)
    data = s.encode("utf-8")
    print(f"{s!r:8} code points: {points:38} UTF-8: {data.hex(' '):18} {len(s)} chars, {len(data)} bytes")

print(chr(0xE9), ord("é"), hex(ord("é")))
print(b"Chlo\xc3\xa9".decode("utf-8"))

'Chloé'  code points: U+0043 U+0068 U+006C U+006F U+00E9     UTF-8: 43 68 6c 6f c3 a9  5 chars, 6 bytes
'£'      code points: U+00A3                                 UTF-8: c2 a3              1 chars, 2 bytes
'€'      code points: U+20AC                                 UTF-8: e2 82 ac           1 chars, 3 bytes
'🚲'      code points: U+1F6B2                                UTF-8: f0 9f 9a b2        1 chars, 4 bytes
é 233 0xe9
Chloé

for ch in "飀":
    row = []
    for enc in ["ascii", "latin-1", "cp1252", "utf-8"]:
        try:
            row.append(f"{enc}: {ch.encode(enc).hex(' ')}")
        except UnicodeEncodeError:
            row.append(f"{enc}: (none)")
    print(ch, " | ".join(row))

utf8_bytes = "Chloé Lefèvre".encode("utf-8")
print(utf8_bytes)
print(utf8_bytes.decode("cp1252"))

é ascii: (none) | latin-1: e9 | cp1252: e9 | utf-8: c3 a9
£ ascii: (none) | latin-1: a3 | cp1252: a3 | utf-8: c2 a3
€ ascii: (none) | latin-1: (none) | cp1252: 80 | utf-8: e2 82 ac
b'Chlo\xc3\xa9 Lef\xc3\xa8vre'
Chloé Lefèvre

cp1252_bytes = "Chloé Lefèvre".encode("cp1252")
print(cp1252_bytes)
cp1252_bytes.decode("utf-8")

b'Chlo\xe9 Lef\xe8vre'
Traceback (most recent call last):
  File "<cell>", line 3, in <module>
    cp1252_bytes.decode("utf-8")
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe9 in position 4: invalid continuation byte

import re
import ftfy

def looks_garbled(text):
    """True when the text is UTF-8 that was read as Windows-1252."""
    try:
        return text.encode("cp1252").decode("utf-8") != text
    except UnicodeError:
        return False

garbled = [m for m in MESSAGES if looks_garbled(m["body"])]
markers = [m for m in MESSAGES if "Ã" in m["body"] or "Â" in m["body"]]
missed = [m for m in garbled if m not in markers]
print(len(garbled), "garbled messages;", len(markers), "contain à or Â")
print(sorted({re.search(r"â€.|ðŸ..", m["body"]).group() for m in missed}))

print(raw.encode("cp1252").decode("utf-8")[:60])
print(ftfy.fix_encoding(raw)[:60])
print(looks_garbled("SÃO PAULO"), ftfy.fix_encoding("SÃO PAULO"))

29 garbled messages; 19 contain à or Â
['​', '🚲', '😡', '🙂']
Harbour Kids 20 n'est pas à la bonne taille pour ma fille. P
Harbour Kids 20 n'est pas à la bonne taille pour ma fille. P
False SÃO PAULO

tidied = raw.replace("\xa0", " ")
tidied.encode("cp1252").decode("utf-8")

Traceback (most recent call last):
  File "<cell>", line 2, in <module>
    tidied.encode("cp1252").decode("utf-8")
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xc3 in position 26: invalid continuation byte

tidied = raw.replace("\xa0", " ")
print(repr(raw[24:29]), repr(tidied[24:29]))
print(repr(ftfy.fix_encoding(tidied)[24:30]))

's Ã\xa0 ' 's à  '
's à la'

import unicodedata

typed = "Chloe\u0301 Lefèvre"
print(typed, "|", chloe["customer"])
print(typed == chloe["customer"], len(typed), len(chloe["customer"]))
print([f"U+{ord(ch):04X}" for ch in typed[3:6]])
nfc = unicodedata.normalize("NFC", typed)
print(nfc == chloe["customer"], len(nfc))
print([unicodedata.normalize(form, "é").encode("utf-8") for form in ["NFC", "NFD"]])
print(sum(unicodedata.normalize("NFC", m["body"]) != m["body"] for m in MESSAGES), "messages change under NFC")

Chloé Lefèvre | Chloé Lefèvre
False 14 13
['U+006F', 'U+0065', 'U+0301']
True 13
[b'\xc3\xa9', b'e\xcc\x81']
0 messages change under NFC

samples = ["ＴＷ－１１９８９１", "fit", "Harbour\xa0Kids", "½ price", "m²"]
for s in samples:
    print(f"{s!r:24} NFC: {unicodedata.normalize('NFC', s)!r:24} NFKC: {unicodedata.normalize('NFKC', s)!r}")

ORDER = re.compile(r"TW-\d{6}")
wide = "Commande ＴＷ－１１９８９１"
print(ORDER.findall(wide), ORDER.findall(unicodedata.normalize("NFKC", wide)))

'ＴＷ－１１９８９１'              NFC: 'ＴＷ－１１９８９１'              NFKC: 'TW-119891'
'fit'                     NFC: 'fit'                     NFKC: 'fit'
'Harbour\xa0Kids'        NFC: 'Harbour\xa0Kids'        NFKC: 'Harbour Kids'
'½ price'                NFC: '½ price'                NFKC: '1⁄2 price'
'm²'                     NFC: 'm²'                     NFKC: 'm2'
[] ['TW-119891']

for a, b in [("Viele Grüße", "VIELE GRÜSSE"), ("Lefèvre", "LEFÈVRE"), ("Yılmaz", "YILMAZ")]:
    print(f"{a} / {b}: lower {a.lower() == b.lower()}, casefold {a.casefold() == b.casefold()}")

print("Grüße".lower(), "Grüße".casefold())
print("Yılmaz".casefold(), "YILMAZ".casefold())
print(len("İ"), len("İ".lower()), [f"U+{ord(c):04X}" for c in "İ".lower()])

shouting = [m for m in MESSAGES if m["body"].isupper()]
print(len(shouting), "messages in capitals, e.g.", BY_ID["TW-M0404"]["body"][3:40])

Viele Grüße / VIELE GRÜSSE: lower False, casefold True
Lefèvre / LEFÈVRE: lower True, casefold True
Yılmaz / YILMAZ: lower False, casefold False
grüße grüsse
yılmaz yilmaz
1 2 ['U+0069', 'U+0307']
37 messages in capitals, e.g. YOU CHARGED ME £100 TWICE FOR ORDER 

fatima = ftfy.fix_encoding(BY_ID["TW-M1639"]["body"])
print(fatima[:16])
print(repr(fatima[:16]))
print("Hello Tidewell" in fatima)

for ch in "\u200b\xa0\u200d\ufeff\u00ad":
    print(f"U+{ord(ch):04X} {unicodedata.name(ch):26} {unicodedata.category(ch)} isspace={ch.isspace()}")

fixed = [ftfy.fix_encoding(m["body"]) for m in MESSAGES]
print(sum("\u200b" in t for t in fixed), "with a zero-width space;", sum("\xa0" in t for t in fixed), "with a no-break space")

Hello ​Tidewell,
'Hello \u200bTidewell,'
False
U+200B ZERO WIDTH SPACE           Cf isspace=False
U+00A0 NO-BREAK SPACE             Zs isspace=True
U+200D ZERO WIDTH JOINER          Cf isspace=False
U+FEFF ZERO WIDTH NO-BREAK SPACE  Cf isspace=False
U+00AD SOFT HYPHEN                Cf isspace=False
49 with a zero-width space; 54 with a no-break space

INVISIBLE = dict.fromkeys(map(ord, "\u200b\u200c\u2060\ufeff\u00ad"))   # ZWJ U+200D is kept: it joins emoji

def tidy_spaces(text):
    text = text.translate(INVISIBLE)
    text = text.replace("\r\n", "\n").replace("\r", "\n")
    text = re.sub(r"[^\S\n]+", " ", text)                 # any run of spaces, tabs, no-break spaces -> one space
    text = "\n".join(line.strip() for line in text.split("\n"))
    return re.sub(r"\n{3,}", "\n\n", text).strip()

print(repr(tidy_spaces(fatima)[:16]), "Hello Tidewell" in tidy_spaces(fatima))
print(repr(tidy_spaces("Hi,\xa0\xa0I placed\t an order\r\n\r\n\r\n\r\nThanks,")))

'Hello Tidewell,\n' True
'Hi, I placed an order\n\nThanks,'

import html

form = BY_ID["TW-M0001"]["body"]
print(form[:70])
TAG = re.compile(r"</?[a-zA-Z][a-zA-Z0-9]*\s*/?>")    # <p>, </p>, <br>, <br/>: a tag name, nothing else
step = ftfy.fix_encoding(form)
step = TAG.sub("", step.replace("<br>", ""))
step = html.unescape(step)
print(step[:70])
print(sum(bool(TAG.search(m["body"])) for m in MESSAGES), "messages carry HTML tags")

<p>Mon vélo (Causeway) a besoin d&#x27;une réparation : les plaquett
Mon vélo (Causeway) a besoin d'une réparation : les plaquettes de frei
459 messages carry HTML tags

quote_line = "> Le 9 août 2026, Tidewell Cycles <help@tidewell.example> a écrit :"
print(re.sub(r"<[^>]+>", "", quote_line))
print(TAG.sub("", quote_line))

> Le 9 août 2026, Tidewell Cycles  a écrit :
> Le 9 août 2026, Tidewell Cycles <help@tidewell.example> a écrit :

SIGNOFFS = {"thanks,", "many thanks,", "cheers,", "danke,", "viele grüße,",
            "gracias,", "un saludo,", "merci,", "cordialement,"}

def split_reply(text):
    """Cut an email into the new text, the sign-off with the name, and the quoted reply."""
    lines = text.split("\n")
    quoted = [line for line in lines if line.startswith(">")]
    lines = [line for line in lines if not line.startswith(">")]
    cut = max((i for i, line in enumerate(lines) if line.casefold() in SIGNOFFS), default=len(lines))
    return {"body": "\n".join(lines[:cut]).strip(),
            "signature": "\n".join(lines[cut:]).strip(),
            "quoted": "\n".join(quoted)}

parts = split_reply(tidy_spaces(ftfy.fix_encoding(raw)))
for key, value in parts.items():
    print(f"{key:9}| {value!r}")

body     | "Harbour Kids 20 n'est pas à la bonne taille pour ma fille. Puis-je l'échanger ou être remboursé ? Commande TW-119891.\nVous pouvez m'appeler au +44 7700 900466."
signature| 'Merci,\nChloé Lefèvre'
quoted   | '> Le 9 août 2026, Tidewell Cycles <help@tidewell.example> a écrit :\n> Thanks for your message. We will reply within one working day.'