Lab 11 - Guardrails and Thresholds¶

Goal: simulate three-band policy decisions using confidence thresholds.

Info: Three-band policy

Auto-action (high), human-review (middle), abstain (low). Reduces both over-automation and over-escalation noise.

Info: Thresholds by risk class

Security deserves lower thresholds (escalate more). Password-reset can have higher thresholds (automate more). One size does not fit all.

In [ ]:

scores = [0.95, 0.82, 0.74, 0.68, 0.61, 0.49, 0.32]

def action(p, low=0.55, high=0.80):
    if p >= high:
        return 'auto'
    if p <= low:
        return 'abstain'
    return 'human-review'

for s in scores:
    print(s, '->', action(s))

scores = [0.95, 0.82, 0.74, 0.68, 0.61, 0.49, 0.32] def action(p, low=0.55, high=0.80): if p >= high: return 'auto' if p <= low: return 'abstain' return 'human-review' for s in scores: print(s, '->', action(s))

In [ ]:

def policy_counts(values, low=0.55, high=0.80):
    counts = {'auto': 0, 'human-review': 0, 'abstain': 0}
    for v in values:
        counts[action(v, low=low, high=high)] += 1
    return counts

print('Default policy:', policy_counts(scores))
print('Stricter policy:', policy_counts(scores, low=0.60, high=0.85))

def policy_counts(values, low=0.55, high=0.80): counts = {'auto': 0, 'human-review': 0, 'abstain': 0} for v in values: counts[action(v, low=low, high=high)] += 1 return counts print('Default policy:', policy_counts(scores)) print('Stricter policy:', policy_counts(scores, low=0.60, high=0.85))

Visualization: confidence bands and actions¶

Left plot shows thresholds on confidence scores. Right plot shows resulting action counts.

In [ ]:

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
scores = [0.95, 0.82, 0.74, 0.68, 0.61, 0.49, 0.32]
low, high = 0.55, 0.80
def action(p):
    if p >= high: return 'auto'
    if p <= low: return 'abstain'
    return 'review'
counts = {'auto': 0, 'review': 0, 'abstain': 0}
for s in scores:
    counts[action(s)] += 1
fig, ax = plt.subplots(1, 2, figsize=(11,4))
ax[0].hist(scores, bins=[0.0, low, high, 1.0], color='#72B7B2', edgecolor='black')
ax[0].axvline(low, linestyle='--', color='orange', label='low')
ax[0].axvline(high, linestyle='--', color='crimson', label='high')
ax[0].set_title('Confidence banding')
ax[0].legend()
ax[1].bar(list(counts.keys()), list(counts.values()), color=['#4C78A8','#F58518','#54A24B'])
ax[1].set_title('Policy action counts')
plt.tight_layout()
plt.show()

import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt scores = [0.95, 0.82, 0.74, 0.68, 0.61, 0.49, 0.32] low, high = 0.55, 0.80 def action(p): if p >= high: return 'auto' if p <= low: return 'abstain' return 'review' counts = {'auto': 0, 'review': 0, 'abstain': 0} for s in scores: counts[action(s)] += 1 fig, ax = plt.subplots(1, 2, figsize=(11,4)) ax[0].hist(scores, bins=[0.0, low, high, 1.0], color='#72B7B2', edgecolor='black') ax[0].axvline(low, linestyle='--', color='orange', label='low') ax[0].axvline(high, linestyle='--', color='crimson', label='high') ax[0].set_title('Confidence banding') ax[0].legend() ax[1].bar(list(counts.keys()), list(counts.values()), color=['#4C78A8','#F58518','#54A24B']) ax[1].set_title('Policy action counts') plt.tight_layout() plt.show()