LAB 17 · 软目标与坍塌：三个数值检查 · 实现代码

# Writable ephemeral workspace for detached readonly cloud runs.
import os, tempfile
os.chdir(tempfile.mkdtemp(prefix='lec03_lab17_'))

import numpy as np
rng=np.random.default_rng(17)
X=rng.normal(size=(384,2)); X=np.c_[X,np.ones(len(X))]
teacher=np.array([[2,-2,0],[0,1,-1],[.2,-.1,0.]])
def softmax(z):
    e=np.exp(z-z.max(1,keepdims=True)); return e/e.sum(1,keepdims=True)
T=2.; target=softmax(X@teacher/T); W=np.zeros((3,3))
def kl(q,p): return float((q*np.log((q+1e-12)/(p+1e-12))).sum(1).mean())
before=kl(target[256:],softmax(X[256:]@W/T))
for step in range(600):
    pred=softmax(X[:256]@W/T)
    W-=.3*X[:256].T@(pred-target[:256])/256/T
after=kl(target[256:],softmax(X[256:]@W/T))
constant=np.ones((8,4)); diverse=rng.normal(size=(8,4))
print('heldout teacher/student KL',round(before,5),'->',round(after,5))
print('collapsed consistency loss',float(((constant-constant[::-1])**2).mean()),'variance',constant.var(0).tolist())
print('noncollapsed variance',diverse.var(0).round(3).tolist())
ema=np.zeros_like(W)
for t in range(3): ema=.9*ema+.1*W
assert np.allclose(ema,(1-.9**3)*W)
print('3 EMA updates preserve factor',round(1-.9**3,3),'of fixed student parameters')
print('temperature teacher probabilities',softmax(np.array([[3.,1.,0.]])/T).round(4).tolist())