视觉 Agent 找对了按钮，为什么坐标却落在别处？

"""Invert an explicitly described screenshot crop/resize. Performs no UI action."""
from dataclasses import dataclass
import json
import math
from pathlib import Path


@dataclass(frozen=True)
class Observation:
    viewport_css: tuple
    raster_size: tuple
    crop_raster: tuple  # left, top, width, height, before resizing
    displayed_size: tuple
    scroll_css: tuple
    epoch: str

    def validate(self):
        vw, vh = self.viewport_css
        rw, rh = self.raster_size
        x, y, w, h = self.crop_raster
        dw, dh = self.displayed_size
        if min(vw, vh, rw, rh, w, h, dw, dh) <= 0:
            raise ValueError('Dimensions must be positive.')
        if x < 0 or y < 0 or x + w > rw or y + h > rh:
            raise ValueError('Crop must lie inside the viewport screenshot.')

    def invert(self, point, current_epoch):
        self.validate()
        if current_epoch != self.epoch:
            raise ValueError('Observation epoch is stale; obtain a new observation.')
        px, py = point
        dw, dh = self.displayed_size
        if not (0 <= px < dw and 0 <= py < dh):
            raise ValueError('Point must lie inside displayed image, excluding right/bottom edge.')
        left, top, width, height = self.crop_raster
        rx = left + px * width / dw
        ry = top + py * height / dh
        vw, vh = self.viewport_css
        rw, rh = self.raster_size
        cx, cy = rx * vw / rw, ry * vh / rh
        sx, sy = self.scroll_css
        return {'raster': [rx, ry], 'viewport_css':[cx, cy], 'document_css':[cx + sx, cy + sy]}

    def forward(self, viewport_point):
        cx, cy = viewport_point
        vw, vh = self.viewport_css
        rw, rh = self.raster_size
        left, top, width, height = self.crop_raster
        dw, dh = self.displayed_size
        return [(cx * rw / vw - left) * dw / width,
                (cy * rh / vh - top) * dh / height]


def main():
    device = Observation((1280,720),(2560,1440),(600,300,1200,800),(600,400),(0,900),'view-1')
    css = Observation((1280,720),(1280,720),(300,150,600,400),(600,400),(0,900),'view-1')
    detected = (210,135)
    result = device.invert(detected, 'view-1')
    assert result['viewport_css'] == [510.0,285.0]
    assert result['document_css'] == [510.0,1185.0]
    assert css.invert(detected, 'view-1')['viewport_css'] == result['viewport_css']
    assert device.forward(result['viewport_css']) == list(detected)
    naive_error = math.dist(detected, result['viewport_css'])
    print(f'detected in resized crop: {list(detected)}')
    print(f"full screenshot raster: {result['raster']}")
    print(f"mouse viewport CSS: {result['viewport_css']}")
    print(f"document CSS with scroll: {result['document_css']}")
    print(f'naive viewport error: {naive_error:.6f} CSS px')
    print('CSS-scale and device-scale screenshots: same recovered viewport point')
    rejected = []
    for name, point, epoch in [('stale',(210,135),'view-2'),('outside',(600,100),'view-1')]:
        try:
            device.invert(point, epoch)
        except ValueError:
            rejected.append(name)
            print(f'{name}: rejected')
        else:
            raise AssertionError(f'{name} was not rejected')
    for point in [(0,0),(1,1),(599.5,399.5),(210,135)]:
        recovered = device.invert(point, 'view-1')['viewport_css']
        assert all(math.isclose(a,b,abs_tol=1e-10) for a,b in zip(device.forward(recovered),point))
    report = {'detected':detected,'recovered':result,'naive_error_css_px':naive_error,'rejected':rejected,
        'scope':'Fixed viewport screenshot metadata and affine arithmetic; no screenshot captured and no click performed.'}
    Path('report.json').write_text(json.dumps(report, indent=2, allow_nan=False)+'\n', encoding='utf-8')
    print('checks passed: round trips, scale equivalence, stale epoch, image bounds')


if __name__ == '__main__':
    main()