3. 正确分解目标、参照物和关系 · 实现代码

def connected_components(mask):
    seen=np.zeros(mask.shape,bool);components=[]
    for y,x in zip(*np.where(mask)):
        if seen[y,x]:continue
        stack=[(int(y),int(x))];seen[y,x]=True;points=[]
        while stack:
            yy,xx=stack.pop();points.append((yy,xx))
            for dy,dx in [(1,0),(-1,0),(0,1),(0,-1)]:
                ny,nx=yy+dy,xx+dx
                if 0<=ny<mask.shape[0] and 0<=nx<mask.shape[1] and mask[ny,nx] and not seen[ny,nx]:
                    seen[ny,nx]=True;stack.append((ny,nx))
        if len(points)>=30:
            yy,xx=np.array(points).T
            components.append([int(xx.min()),int(yy.min()),int(xx.max()+1),int(yy.max()+1)])
    return components

def pixel_candidates(rgb):
    candidates=[]
    for color,value in PALETTE.items():
        distance=np.linalg.norm(rgb.astype(float)-np.array(value),axis=-1)
        for box in connected_components(distance<20):
            candidates.append({'color':color,'category':'book' if color=='brown' else 'cup','box':box,'center_x':(box[0]+box[2])/2})
    return candidates

def ground(rgb,phrase):
    objects=pixel_candidates(rgb)
    if ' left of ' in phrase or ' right of ' in phrase:
        relation='left' if ' left of ' in phrase else 'right'
        target_noun,reference_phrase=phrase.split(' '+relation+' of ')
        ref_color=reference_phrase.split()[0]
        refs=[o for o in objects if o['category']=='cup' and o['color']==ref_color]
        if len(refs)!=1:return {'status':'reference missing/ambiguous','target':None,'reference':None,'candidates':objects}
        ref=refs[0]
        possible=[o for o in objects if o['category']==target_noun and o is not ref and ((o['center_x']<ref['center_x']) if relation=='left' else (o['center_x']>ref['center_x']))]
        return {'status':'unique target' if len(possible)==1 else 'no unique visible target','target':possible[0] if len(possible)==1 else None,'reference':ref,'relation':relation,'candidates':objects}
    color,noun=phrase.split();found=[o for o in objects if o['color']==color and o['category']==noun]
    return {'status':'unique target' if len(found)==1 else 'no unique visible target','target':found[0] if len(found)==1 else None,'reference':None,'candidates':objects}

cases=[('base',base,'cup left of blue cup'),('target occluded',occluded,'cup left of blue cup'),('colors exchanged',swapped,'cup left of blue cup'),('relation changed',swapped,'cup right of blue cup')]
results=[]
for condition,pixels,phrase in cases:
    row={'condition':condition,'phrase':phrase,**ground(pixels,phrase)};results.append(row)
    print(condition,phrase,'->',row['status'],'target',row['target'],'reference',row['reference'])
assert results[0]['target']['color']=='red'
assert results[0]['reference']['color']=='blue'
assert results[0]['target']['box']==[15,19,50,58]
assert results[1]['target'] is None
assert results[2]['target'] is None
assert results[3]['target']['color']=='red' and results[3]['target']['box'][0]==109