Tokenization

import re
from nltk.tokenize import sent_tokenize, word_tokenize   # nltk.download("punkt_tab") once

text = "Dr. Rao paid ₹1,250 via UPI. It didn't go through! Can I retry?"

print("Sentences:", sent_tokenize(text))
print("split():  ", text.split())
print("NLTK words:", word_tokenize(text))
print("Characters:", list("UPI paid")[:8])

from collections import Counter

# word frequencies in a tiny corpus; each word starts as characters + end marker
corpus = {"low": 5, "lower": 2, "newest": 6, "widest": 3}
vocab = {tuple(w) + ("</w>",): f for w, f in corpus.items()}

def pair_counts(vocab):
    pairs = Counter()
    for word, f in vocab.items():
        for a, b in zip(word, word[1:]):
            pairs[(a, b)] += f
    return pairs

def merge(vocab, pair):
    out = {}
    for word, f in vocab.items():
        w, i = [], 0
        while i < len(word):
            if i < len(word) - 1 and (word[i], word[i + 1]) == pair:
                w.append(word[i] + word[i + 1]); i += 2
            else:
                w.append(word[i]); i += 1
        out[tuple(w)] = f
    return out

for step in range(1, 6):
    best, n = pair_counts(vocab).most_common(1)[0]
    vocab = merge(vocab, best)
    print(f"merge {step}: {best[0]} + {best[1]} -> {best[0] + best[1]}  (seen {n} times)")

print("\nFinal pieces:")
for word in vocab: print("  ", " ".join(word))

# pip install tokenizers
from tokenizers import Tokenizer, models, trainers, pre_tokenizers

corpus = [
    "UPI payments are instant and free for customers.",
    "The payment failed, please retry the UPI payment.",
    "Fixed deposit interest is paid every quarter.",
    "Interest rates for senior citizens are higher.",
    "Payments, repayments and prepayments are tracked daily.",
] * 20

def train(model, trainer):
    tok = Tokenizer(model)
    tok.pre_tokenizer = pre_tokenizers.Whitespace()
    tok.train_from_iterator(corpus, trainer)
    return tok

bpe = train(models.BPE(unk_token="[UNK]"), trainers.BpeTrainer(vocab_size=120, special_tokens=["[UNK]"]))
wp = train(models.WordPiece(unk_token="[UNK]"), trainers.WordPieceTrainer(vocab_size=120, special_tokens=["[UNK]"]))

for word in ["payments", "repayment", "interested", "UPI"]:
    print(f"{word:11s} BPE: {bpe.encode(word).tokens}   WordPiece: {wp.encode(word).tokens}")

# Same greeting, different scripts: byte-level tokenizers start from UTF-8 bytes
for text in ["hello", "नमस्ते", "வணக்கம்"]:
    print(f"{text:8s} characters={len(text):2d}  utf8_bytes={len(text.encode('utf-8'))}")

# Cost estimate: tokens x price.  Example rates only: check your provider's price page.
price_in, price_out = 0.15, 0.60          # US$ per 1 million tokens
calls_per_day, tokens_in, tokens_out = 10_000, 800, 200
daily = calls_per_day * (tokens_in * price_in + tokens_out * price_out) / 1_000_000
print(f"\n{calls_per_day:,} calls/day x ({tokens_in} in + {tokens_out} out) tokens")
print(f"= ${daily:.2f} per day, about ${daily * 30:.0f} per month")