Lab 01 - Tokenization¶

Goal: compare token-like counts under a fixed budget.

Info: Token budgeting for IT

Token constraints are critical in production. Reserve output tokens and account for system instructions before calculating available space.

Info: Budget allocation

Longer tickets force trade-offs: truncate history, summarize context, or reject requests. This is a daily operational decision.

In [1]:

sample_tickets = {
    "password_reset": "Need reset for AD account",
    "vpn_issue": "VPN disconnects every 5 minutes after patch",
    "long_outage": " ".join(["Service"] * 180)
}

def pseudo_token_count(text):
    return len(text.split())

for name, text in sample_tickets.items():
    print(name, pseudo_token_count(text))

sample_tickets = { "password_reset": "Need reset for AD account", "vpn_issue": "VPN disconnects every 5 minutes after patch", "long_outage": " ".join(["Service"] * 180) } def pseudo_token_count(text): return len(text.split()) for name, text in sample_tickets.items(): print(name, pseudo_token_count(text))

password_reset 5
vpn_issue 7
long_outage 180

In [2]:

token_budget = 120
system_tokens = 20
history_tokens = 55

for name, text in sample_tickets.items():
    user_tokens = pseudo_token_count(text)
    output_reserve = 30
    total = system_tokens + history_tokens + user_tokens + output_reserve
    print(f"{name}: total={total}, within_budget={total <= token_budget}")

token_budget = 120 system_tokens = 20 history_tokens = 55 for name, text in sample_tickets.items(): user_tokens = pseudo_token_count(text) output_reserve = 30 total = system_tokens + history_tokens + user_tokens + output_reserve print(f"{name}: total={total}, within_budget={total <= token_budget}")

password_reset: total=110, within_budget=True
vpn_issue: total=112, within_budget=True
long_outage: total=285, within_budget=False

In [3]:

def trim_history_to_fit(user_tokens, budget, system_tokens=20, output_reserve=30):
    max_history = max(0, budget - system_tokens - user_tokens - output_reserve)
    return max_history

for name, text in sample_tickets.items():
    user_tokens = pseudo_token_count(text)
    allowed_history = trim_history_to_fit(user_tokens, token_budget)
    print(f"{name}: allow up to {allowed_history} history tokens")

def trim_history_to_fit(user_tokens, budget, system_tokens=20, output_reserve=30): max_history = max(0, budget - system_tokens - user_tokens - output_reserve) return max_history for name, text in sample_tickets.items(): user_tokens = pseudo_token_count(text) allowed_history = trim_history_to_fit(user_tokens, token_budget) print(f"{name}: allow up to {allowed_history} history tokens")

password_reset: allow up to 65 history tokens
vpn_issue: allow up to 63 history tokens
long_outage: allow up to 0 history tokens