4. 推理只接自己的输出，再检查视觉依赖 · 实现代码

@torch.no_grad()
def greedy(pixels, prefix=(BOS,), max_new=8):
    model.eval(); seq=torch.tensor(prefix).repeat(len(pixels),1); finished=torch.zeros(len(pixels),dtype=torch.bool)
    for _ in range(max_new):
        nxt=model(pixels,seq)[:,-1].argmax(-1)
        nxt=torch.where(finished,torch.tensor(EOS),nxt)
        seq=torch.cat((seq,nxt[:,None]),1); finished|=nxt.eq(EOS)
        if bool(finished.all()): break
    return seq.tolist()

@torch.no_grad()
def beam_one(pixel, width=3, max_new=8):
    beams=[([BOS],0.0,False)]
    for _ in range(max_new):
        candidates=[]
        for seq,score,done in beams:
            if done: candidates.append((seq,score,True)); continue
            lp=F.log_softmax(model(pixel[None],torch.tensor([seq]))[0,-1],dim=-1)
            values,indices=lp.topk(width)
            for value,idx in zip(values.tolist(),indices.tolist()):
                candidates.append((seq+[idx],score+value,idx==EOS))
        beams=sorted(candidates,key=lambda b:b[1],reverse=True)[:width]
        if all(b[2] for b in beams): break
    # Ended hypotheses only if at least one ended; otherwise max-length fallback.
    ended=[b for b in beams if b[2]]
    return max(ended or beams,key=lambda b:b[1])

def words(ids):
    out=[]
    for idx in ids:
        if idx==EOS: break
        if idx not in [BOS,PAD]: out.append(VOCAB[idx])
    return ' '.join(out)

pred=[words(s) for s in greedy(Xtest)]
beam_pred=[words(beam_one(x)[0]) for x in Xtest]
zero_pred=[words(s) for s in greedy(torch.zeros_like(Xtest))]
# A deterministic derangement shifts test images; target captions stay fixed.
shuffle_index=torch.roll(torch.arange(len(Xtest)),1)
shuffle_pred=[words(s) for s in greedy(Xtest[shuffle_index])]
exact=lambda p: float(np.mean([a==b for a,b in zip(p,test_text)]))
metrics={'heldout_images':len(Xtest),'greedy_exact_match':exact(pred),'beam3_exact_match':exact(beam_pred),'zero_pixels_exact_match':exact(zero_pred),'shuffled_pixels_exact_match':exact(shuffle_pred),'final_train_token_ce':curve[-1]['train_token_ce'],'final_heldout_teacher_forced_ce':curve[-1]['heldout_teacher_forced_ce']}
print(json.dumps(metrics,indent=2))
# Hold the language prefix fixed; change only the input image.
rng=np.random.default_rng(999)
control_specs=[('red','square','left'),('blue','circle','right'),('green','square','center')]
control_pixels=torch.tensor(np.stack([render_object(*spec,rng) for spec in control_specs])).permute(0,3,1,2)
control_pred=[words(s) for s in greedy(control_pixels,prefix=(BOS,W['a']))]
controls=[{'condition':list(spec),'fixed_prefix':'<bos> a','generated':p} for spec,p in zip(control_specs,control_pred)]
print('same prefix, changed image:',json.dumps(controls,indent=2))