944 lines
36 KiB
Python
944 lines
36 KiB
Python
"""
|
|
ESP32 camera viewer — local (on-device) HSV detection only.
|
|
|
|
Tunes colour / method / perf on the ESP over WebSocket; overlays blob from
|
|
X-Vision-Data (or /vision in detect-only mode).
|
|
|
|
Usage:
|
|
.\\.venv\\Scripts\\python.exe viewer.py --ip 192.168.1.178
|
|
|
|
Arrow keys / WASD also send directions. Enter sends the text box.
|
|
Commands go over ws://IP/ws (HTTP kept as fallback).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import threading
|
|
import time
|
|
import tkinter as tk
|
|
from tkinter import colorchooser, scrolledtext, ttk
|
|
|
|
import cv2
|
|
import numpy as np
|
|
import requests
|
|
from PIL import Image, ImageTk
|
|
from websocket import WebSocketApp
|
|
|
|
from targets import TargetDetector, create_target
|
|
from targets.orange_ball import OrangeBallDetector
|
|
|
|
ESP_METHODS = ("contour", "rect")
|
|
ESP_METHOD_LABELS = {"contour": "Contour", "rect": "Rectangle"}
|
|
|
|
|
|
class RobotApp:
|
|
def __init__(
|
|
self,
|
|
ip: str,
|
|
fps: float,
|
|
timeout: float,
|
|
detector: TargetDetector | None,
|
|
) -> None:
|
|
self.ip = ip
|
|
self.base = f"http://{ip}"
|
|
self.ws_url = f"ws://{ip}/ws"
|
|
self.period = 1.0 / fps if fps > 0 else 0.0
|
|
self.timeout = timeout
|
|
self.detector = detector
|
|
self.session = requests.Session()
|
|
self.session.headers.update(
|
|
{"Cache-Control": "no-cache", "Pragma": "no-cache"}
|
|
)
|
|
self._stop = threading.Event()
|
|
self._photo: ImageTk.PhotoImage | None = None
|
|
self._ws: WebSocketApp | None = None
|
|
self._ws_ready = threading.Event()
|
|
self._ws_lock = threading.Lock()
|
|
self._last_det_log = 0.0
|
|
self._raw_frame: np.ndarray | None = None
|
|
self._frame_lock = threading.Lock()
|
|
self._preview_scale = 1.0
|
|
self._preview_display_size = (640, 480)
|
|
self._preview_src_size = (640, 480)
|
|
self._vision_sync_after_id: str | None = None
|
|
self._last_vision_payload: str | None = None
|
|
self._pending_pick: tuple[int, int] | None = None
|
|
self._applying_remote = False
|
|
self._vision_settings_loaded = False
|
|
|
|
self.root = tk.Tk()
|
|
self.root.title(f"linefollower-eye — {ip}")
|
|
self.root.protocol("WM_DELETE_WINDOW", self.on_close)
|
|
self.show_mask = tk.BooleanVar(master=self.root, value=False)
|
|
self.mask_full = tk.BooleanVar(master=self.root, value=False)
|
|
self.morph_roi = tk.BooleanVar(master=self.root, value=True)
|
|
self.coarse_stride = tk.IntVar(master=self.root, value=4)
|
|
self.detect_only = tk.BooleanVar(master=self.root, value=False)
|
|
self.tol_var = tk.IntVar(master=self.root, value=10)
|
|
self.eyedropper_on = tk.BooleanVar(master=self.root, value=False)
|
|
|
|
if isinstance(self.detector, OrangeBallDetector):
|
|
# ESP only supports contour/rect
|
|
self.detector.set_method("contour")
|
|
|
|
main = ttk.Frame(self.root, padding=8)
|
|
main.grid(row=0, column=0, sticky="nsew")
|
|
self.root.rowconfigure(0, weight=1)
|
|
self.root.columnconfigure(0, weight=1)
|
|
main.rowconfigure(0, weight=1)
|
|
main.columnconfigure(0, weight=1)
|
|
|
|
self.video = ttk.Label(main, cursor="arrow")
|
|
self.video.grid(row=0, column=0, sticky="nsew", padx=(0, 8))
|
|
self.video.bind("<Button-1>", self._on_video_click)
|
|
|
|
side = ttk.Frame(main)
|
|
side.grid(row=0, column=1, sticky="ns")
|
|
|
|
ttk.Label(side, text="Drive").grid(row=0, column=0, columnspan=3, pady=(0, 4))
|
|
ttk.Button(side, text="↑", width=4, command=lambda: self.send_cmd("up")).grid(
|
|
row=1, column=1, padx=2, pady=2
|
|
)
|
|
ttk.Button(side, text="←", width=4, command=lambda: self.send_cmd("left")).grid(
|
|
row=2, column=0, padx=2, pady=2
|
|
)
|
|
ttk.Button(side, text="●", width=4, command=lambda: self.send_cmd("stop")).grid(
|
|
row=2, column=1, padx=2, pady=2
|
|
)
|
|
ttk.Button(side, text="→", width=4, command=lambda: self.send_cmd("right")).grid(
|
|
row=2, column=2, padx=2, pady=2
|
|
)
|
|
ttk.Button(side, text="↓", width=4, command=lambda: self.send_cmd("down")).grid(
|
|
row=3, column=1, padx=2, pady=2
|
|
)
|
|
|
|
row = 4
|
|
if isinstance(self.detector, OrangeBallDetector):
|
|
row = self._build_tune_panel(side, start_row=row)
|
|
|
|
ttk.Label(side, text="Terminal").grid(
|
|
row=row, column=0, columnspan=3, sticky="w", pady=(16, 4)
|
|
)
|
|
self.log = scrolledtext.ScrolledText(
|
|
side, width=36, height=12, state="disabled", wrap="word"
|
|
)
|
|
self.log.grid(row=row + 1, column=0, columnspan=3, sticky="nsew")
|
|
side.rowconfigure(row + 1, weight=1)
|
|
|
|
entry_row = ttk.Frame(side)
|
|
entry_row.grid(row=row + 2, column=0, columnspan=3, sticky="ew", pady=(6, 0))
|
|
self.entry = ttk.Entry(entry_row)
|
|
self.entry.pack(side="left", fill="x", expand=True)
|
|
self.entry.bind("<Return>", lambda _e: self.send_text())
|
|
ttk.Button(entry_row, text="Send", command=self.send_text).pack(
|
|
side="left", padx=(6, 0)
|
|
)
|
|
|
|
self.root.bind("<Up>", lambda _e: self.send_cmd("up"))
|
|
self.root.bind("<Down>", lambda _e: self.send_cmd("down"))
|
|
self.root.bind("<Left>", lambda _e: self.send_cmd("left"))
|
|
self.root.bind("<Right>", lambda _e: self.send_cmd("right"))
|
|
self.root.bind("<space>", lambda _e: self.send_cmd("stop"))
|
|
self.root.bind("w", lambda _e: self.send_cmd("up"))
|
|
self.root.bind("s", lambda _e: self.send_cmd("down"))
|
|
self.root.bind("a", lambda _e: self.send_cmd("left"))
|
|
self.root.bind("d", lambda _e: self.send_cmd("right"))
|
|
|
|
self.append_log(f"HTTP {self.base} WS {self.ws_url}")
|
|
if self.detector is not None:
|
|
self.append_log(f"Target: {self.detector.name} (ESP local vision)")
|
|
if isinstance(self.detector, OrangeBallDetector):
|
|
self.append_log("Eyedropper / Pick colour → sync HSV to ESP")
|
|
else:
|
|
self.append_log("Target: off (drive + stream only)")
|
|
self._ws_thread = threading.Thread(target=self._ws_loop, daemon=True)
|
|
self._ws_thread.start()
|
|
self._video_thread = threading.Thread(target=self._video_loop, daemon=True)
|
|
self._video_thread.start()
|
|
|
|
def _build_tune_panel(self, parent: ttk.Frame, start_row: int) -> int:
|
|
if not isinstance(self.detector, OrangeBallDetector):
|
|
return start_row
|
|
|
|
h_tol, _, _ = self.detector.get_tolerance()
|
|
self.tol_var.set(h_tol)
|
|
|
|
ttk.Label(parent, text="ESP local vision", font=("", 9, "bold")).grid(
|
|
row=start_row, column=0, columnspan=3, sticky="w", pady=(16, 4)
|
|
)
|
|
start_row += 1
|
|
|
|
ttk.Label(parent, text="Target colour").grid(
|
|
row=start_row, column=0, columnspan=3, sticky="w", pady=(8, 4)
|
|
)
|
|
start_row += 1
|
|
|
|
row = ttk.Frame(parent)
|
|
row.grid(row=start_row, column=0, columnspan=3, sticky="ew", pady=2)
|
|
self.swatch = tk.Canvas(row, width=36, height=24, highlightthickness=1)
|
|
self.swatch.pack(side="left")
|
|
ttk.Button(row, text="Pick colour…", command=self._pick_colour).pack(
|
|
side="left", padx=(8, 4)
|
|
)
|
|
self.eyedrop_btn = ttk.Checkbutton(
|
|
row,
|
|
text="Eyedropper",
|
|
variable=self.eyedropper_on,
|
|
command=self._on_eyedropper_toggle,
|
|
)
|
|
self.eyedrop_btn.pack(side="left")
|
|
start_row += 1
|
|
|
|
method_row = ttk.Frame(parent)
|
|
method_row.grid(row=start_row, column=0, columnspan=3, sticky="ew", pady=2)
|
|
ttk.Label(method_row, text="Method", width=11).pack(side="left")
|
|
self.method_var = tk.StringVar(
|
|
master=self.root,
|
|
value=ESP_METHOD_LABELS["contour"],
|
|
)
|
|
self.method_combo = ttk.Combobox(
|
|
method_row,
|
|
textvariable=self.method_var,
|
|
values=[ESP_METHOD_LABELS[m] for m in ESP_METHODS],
|
|
state="readonly",
|
|
width=16,
|
|
)
|
|
self.method_combo.pack(side="left", fill="x", expand=True, padx=4)
|
|
self.method_combo.bind("<<ComboboxSelected>>", self._on_method_change)
|
|
start_row += 1
|
|
|
|
self.show_mask_btn = ttk.Checkbutton(
|
|
parent,
|
|
text="Show mask",
|
|
variable=self.show_mask,
|
|
command=self._on_show_mask_change,
|
|
)
|
|
self.show_mask_btn.grid(row=start_row, column=0, columnspan=3, sticky="w")
|
|
start_row += 1
|
|
|
|
perf = ttk.LabelFrame(parent, text="ESP32 perf / detect", padding=4)
|
|
perf.grid(row=start_row, column=0, columnspan=3, sticky="ew", pady=(8, 4))
|
|
start_row += 1
|
|
|
|
stride_row = ttk.Frame(perf)
|
|
stride_row.pack(fill="x", pady=2)
|
|
ttk.Label(stride_row, text="Coarse stride", width=12).pack(side="left")
|
|
self.stride_combo = ttk.Combobox(
|
|
stride_row,
|
|
values=("2", "4"),
|
|
state="readonly",
|
|
width=4,
|
|
)
|
|
self.stride_combo.set(str(self.coarse_stride.get()))
|
|
self.stride_combo.pack(side="left", padx=4)
|
|
self.stride_combo.bind("<<ComboboxSelected>>", self._on_esp_perf_change)
|
|
|
|
opt_row = ttk.Frame(perf)
|
|
opt_row.pack(fill="x", pady=2)
|
|
ttk.Checkbutton(
|
|
opt_row,
|
|
text="Full-frame mask (slow)",
|
|
variable=self.mask_full,
|
|
command=self._on_esp_perf_change,
|
|
).pack(side="left")
|
|
ttk.Checkbutton(
|
|
opt_row,
|
|
text="ROI morph",
|
|
variable=self.morph_roi,
|
|
command=self._on_esp_perf_change,
|
|
).pack(side="left", padx=(8, 0))
|
|
|
|
detect_row = ttk.Frame(perf)
|
|
detect_row.pack(fill="x", pady=2)
|
|
ttk.Checkbutton(
|
|
detect_row,
|
|
text="Detect only (/vision, no video)",
|
|
variable=self.detect_only,
|
|
command=self._on_detect_only_change,
|
|
).pack(side="left")
|
|
|
|
self.esp_stats_label = ttk.Label(
|
|
perf,
|
|
text="timing: —",
|
|
wraplength=240,
|
|
justify="left",
|
|
)
|
|
self.esp_stats_label.pack(fill="x", pady=(4, 0))
|
|
|
|
tol_row = ttk.Frame(parent)
|
|
tol_row.grid(row=start_row, column=0, columnspan=3, sticky="ew", pady=2)
|
|
ttk.Label(tol_row, text="Tolerance", width=11).pack(side="left")
|
|
self.tol_label = ttk.Label(tol_row, width=4)
|
|
self.tol_scale = ttk.Scale(
|
|
tol_row,
|
|
from_=2,
|
|
to=40,
|
|
variable=self.tol_var,
|
|
orient="horizontal",
|
|
command=self._on_tol_change,
|
|
)
|
|
self.tol_scale.pack(side="left", fill="x", expand=True, padx=4)
|
|
self.tol_label.pack(side="left")
|
|
self._on_tol_change(str(self.tol_var.get()))
|
|
start_row += 1
|
|
|
|
self._update_swatch()
|
|
return start_row
|
|
|
|
def _method_key_from_label(self, label: str) -> str | None:
|
|
for key, text in ESP_METHOD_LABELS.items():
|
|
if text == label:
|
|
return key
|
|
return None
|
|
|
|
def _on_esp_perf_change(self, _event: object | None = None) -> None:
|
|
if self._applying_remote:
|
|
return
|
|
try:
|
|
self.coarse_stride.set(int(self.stride_combo.get()))
|
|
except ValueError:
|
|
self.coarse_stride.set(4)
|
|
self.stride_combo.set("4")
|
|
self._last_vision_payload = None
|
|
self._sync_vision_to_esp()
|
|
|
|
def _on_detect_only_change(self) -> None:
|
|
mode = "detect-only (/vision)" if self.detect_only.get() else "JPEG + overlay"
|
|
self.append_log(f"Stream mode → {mode}")
|
|
|
|
def _parse_vision_data_str(self, raw: str) -> dict[str, float | int | bool]:
|
|
out: dict[str, float | int | bool] = {}
|
|
if not raw or "found=" not in raw:
|
|
return out
|
|
if any(ord(ch) < 32 and ch not in "\t\r\n" for ch in raw):
|
|
return out
|
|
for part in raw.split(";"):
|
|
if "=" not in part:
|
|
continue
|
|
key, val = part.split("=", 1)
|
|
try:
|
|
if key == "found":
|
|
out["found"] = val == "1"
|
|
elif key == "area":
|
|
out[key] = int(float(val))
|
|
else:
|
|
out[key] = float(val)
|
|
except ValueError:
|
|
continue
|
|
return out
|
|
|
|
def _parse_vision_json(self, data: dict) -> dict[str, float | int | bool]:
|
|
return {
|
|
"found": bool(data.get("found")),
|
|
"cx": float(data.get("cx", 0)),
|
|
"cy": float(data.get("cy", 0)),
|
|
"r": float(data.get("radius", 0)),
|
|
"area": int(data.get("area", 0)),
|
|
"total": float(data.get("total_ms", 0)),
|
|
"detect": float(data.get("detect_ms", 0)),
|
|
"coarse": float(data.get("coarse_ms", 0)),
|
|
"refine": float(data.get("refine_ms", 0)),
|
|
"morph": float(data.get("morph_ms", 0)),
|
|
"mask": float(data.get("mask_ms", 0)),
|
|
"jpeg": float(data.get("jpeg_ms", 0)),
|
|
}
|
|
|
|
def _format_esp_stats_line(self, stats: dict[str, float | int | bool]) -> str:
|
|
parts: list[str] = []
|
|
for key, label in (
|
|
("total", "total"),
|
|
("detect", "det"),
|
|
("coarse", "coarse"),
|
|
("refine", "refine"),
|
|
("morph", "morph"),
|
|
("mask", "mask"),
|
|
("jpeg", "jpeg"),
|
|
):
|
|
val = stats.get(key)
|
|
if isinstance(val, (int, float)):
|
|
if key in ("morph", "mask") and val <= 0.05:
|
|
continue
|
|
if key == "total":
|
|
parts.append(f"{label} {val:.0f}ms")
|
|
else:
|
|
parts.append(f"{label} {val:.0f}")
|
|
|
|
blob = ""
|
|
if stats.get("found"):
|
|
cx = stats.get("cx", "?")
|
|
cy = stats.get("cy", "?")
|
|
r = stats.get("r", stats.get("radius", "?"))
|
|
blob = f" | blob @({cx},{cy}) r={r}"
|
|
return "timing: " + (" ".join(parts) if parts else "—") + blob
|
|
|
|
def _annotate_esp_target(
|
|
self, frame_bgr: np.ndarray, stats: dict[str, float | int | bool]
|
|
) -> np.ndarray:
|
|
if not stats.get("found"):
|
|
return frame_bgr
|
|
out = frame_bgr.copy()
|
|
cx = int(float(stats.get("cx", 0)))
|
|
cy = int(float(stats.get("cy", 0)))
|
|
radius = stats.get("r", stats.get("radius", 0))
|
|
r = max(4, int(float(radius)))
|
|
cv2.circle(out, (cx, cy), r, (0, 255, 0), 2)
|
|
cv2.drawMarker(out, (cx, cy), (0, 255, 0), cv2.MARKER_CROSS, 12, 2)
|
|
return out
|
|
|
|
def _seed_display_frame(self, capture_url: str) -> bool:
|
|
"""Fetch one JPEG for overlay when detect-only has no cached frame yet."""
|
|
try:
|
|
resp = self.session.get(capture_url, timeout=self._capture_timeout())
|
|
resp.raise_for_status()
|
|
buf = np.frombuffer(resp.content, dtype=np.uint8)
|
|
frame = cv2.imdecode(buf, cv2.IMREAD_COLOR)
|
|
if frame is None:
|
|
return False
|
|
with self._frame_lock:
|
|
self._raw_frame = frame.copy()
|
|
return True
|
|
except requests.RequestException:
|
|
return False
|
|
|
|
def _refresh_detect_overlay(self, esp_stats: dict[str, float | int | bool]) -> None:
|
|
with self._frame_lock:
|
|
frame = None if self._raw_frame is None else self._raw_frame.copy()
|
|
if frame is None:
|
|
return
|
|
display = self._annotate_esp_target(frame, esp_stats)
|
|
self.root.after(0, self._show_frame, display)
|
|
|
|
def _apply_esp_stats(self, stats: dict[str, float | int | bool]) -> None:
|
|
line = self._format_esp_stats_line(stats)
|
|
self.root.after(0, self._update_esp_stats_label, line)
|
|
|
|
now = time.perf_counter()
|
|
if stats.get("found") and now - self._last_det_log > 1.0:
|
|
self.append_log(
|
|
f"ESP blob @ ({stats.get('cx')},{stats.get('cy')}) "
|
|
f"r={stats.get('r', stats.get('radius'))} area={stats.get('area')}"
|
|
)
|
|
self._last_det_log = now
|
|
|
|
def _apply_vision_headers(self, headers: object) -> dict[str, float | int | bool] | None:
|
|
getter = getattr(headers, "get", None)
|
|
if getter is None:
|
|
return None
|
|
raw = getter("X-Vision-Data") or getter("x-vision-data")
|
|
if not raw:
|
|
return None
|
|
stats = self._parse_vision_data_str(str(raw))
|
|
if not stats:
|
|
return None
|
|
self._apply_esp_stats(stats)
|
|
return stats
|
|
|
|
def _update_esp_stats_label(self, text: str) -> None:
|
|
if hasattr(self, "esp_stats_label"):
|
|
self.esp_stats_label.configure(text=text)
|
|
|
|
def _request_vision_settings(self) -> None:
|
|
"""Pull current ESP vision params without writing (HTTP, WS fallback)."""
|
|
|
|
def _worker() -> None:
|
|
data: dict | None = None
|
|
try:
|
|
resp = self.session.get(
|
|
f"{self.base}/vision/settings",
|
|
timeout=self.timeout,
|
|
)
|
|
resp.raise_for_status()
|
|
parsed = resp.json()
|
|
if isinstance(parsed, dict) and parsed.get("ok"):
|
|
data = parsed
|
|
except (requests.RequestException, ValueError) as exc:
|
|
self.append_log(f"! vision settings HTTP: {exc}")
|
|
|
|
if data is None:
|
|
if not self._ws_send({"type": "vision", "get": True}):
|
|
self.append_log("! vision get failed (WS not connected)")
|
|
return
|
|
|
|
self.root.after(0, self._apply_vision_from_esp, data)
|
|
|
|
threading.Thread(target=_worker, daemon=True).start()
|
|
|
|
def _build_vision_payload(self, *, consume_pick: bool = True) -> dict | None:
|
|
if not isinstance(self.detector, OrangeBallDetector):
|
|
return None
|
|
if not hasattr(self, "method_var"):
|
|
return None
|
|
h, s, v = self.detector.get_hsv_center()
|
|
h_tol, s_tol, v_tol = self.detector.get_tolerance()
|
|
key = self._method_key_from_label(self.method_var.get()) or "contour"
|
|
if key not in ESP_METHODS:
|
|
key = "contour"
|
|
payload: dict = {
|
|
"type": "vision",
|
|
"local": True,
|
|
"show_mask": self.show_mask.get(),
|
|
"method": key,
|
|
"h": h,
|
|
"s": s,
|
|
"v": v,
|
|
"h_tol": h_tol,
|
|
"s_tol": s_tol,
|
|
"v_tol": v_tol,
|
|
"coarse_stride": int(self.coarse_stride.get()),
|
|
"min_area": int(self.detector.min_area),
|
|
"mask_full": self.mask_full.get(),
|
|
"morph_roi": self.morph_roi.get(),
|
|
}
|
|
if consume_pick and self._pending_pick is not None:
|
|
payload["pick_x"], payload["pick_y"] = self._pending_pick
|
|
self._pending_pick = None
|
|
return payload
|
|
|
|
def _set_swatch_bgr(self, b: int, g: int, r: int) -> None:
|
|
if not hasattr(self, "swatch"):
|
|
return
|
|
color = f"#{r:02x}{g:02x}{b:02x}"
|
|
self.swatch.configure(bg=color)
|
|
self.swatch.delete("all")
|
|
self.swatch.create_rectangle(0, 0, 36, 24, fill=color, outline="")
|
|
|
|
def _apply_vision_from_esp(self, data: dict) -> None:
|
|
"""Populate UI + detector from a vision ack JSON."""
|
|
if not isinstance(self.detector, OrangeBallDetector):
|
|
return
|
|
if not data.get("ok"):
|
|
err = data.get("error", "unknown")
|
|
self.append_log(f"! vision settings rejected: {err}")
|
|
return
|
|
|
|
self._applying_remote = True
|
|
try:
|
|
if "h" in data and "s" in data and "v" in data:
|
|
self.detector.set_hsv_center(
|
|
int(data["h"]), int(data["s"]), int(data["v"])
|
|
)
|
|
if "h_tol" in data or "s_tol" in data or "v_tol" in data:
|
|
self.detector.set_tolerance(
|
|
h_tol=int(data["h_tol"]) if "h_tol" in data else None,
|
|
s_tol=int(data["s_tol"]) if "s_tol" in data else None,
|
|
v_tol=int(data["v_tol"]) if "v_tol" in data else None,
|
|
)
|
|
if "h_tol" in data and hasattr(self, "tol_label"):
|
|
ht = int(data["h_tol"])
|
|
# Avoid Scale command overwriting s_tol/v_tol with h*6
|
|
if hasattr(self, "tol_scale"):
|
|
self.tol_scale.configure(command="")
|
|
self.tol_var.set(ht)
|
|
self.tol_label.configure(text=str(ht))
|
|
if hasattr(self, "tol_scale"):
|
|
self.tol_scale.configure(command=self._on_tol_change)
|
|
if "min_area" in data:
|
|
self.detector.min_area = float(int(data["min_area"]))
|
|
method = str(data.get("method", "contour"))
|
|
if method not in ESP_METHODS:
|
|
method = "contour"
|
|
self.detector.set_method(method)
|
|
if hasattr(self, "method_var"):
|
|
self.method_var.set(ESP_METHOD_LABELS[method])
|
|
if "show_mask" in data:
|
|
self.show_mask.set(bool(data["show_mask"]))
|
|
if "mask_full" in data:
|
|
self.mask_full.set(bool(data["mask_full"]))
|
|
if "morph_roi" in data:
|
|
self.morph_roi.set(bool(data["morph_roi"]))
|
|
if "coarse_stride" in data and hasattr(self, "stride_combo"):
|
|
stride = int(data["coarse_stride"])
|
|
if stride not in (2, 4):
|
|
stride = 4
|
|
self.coarse_stride.set(stride)
|
|
self.stride_combo.set(str(stride))
|
|
# Prefer ESP display BGR (eyedrop pixel / stored swatch) over HSV→BGR
|
|
if all(k in data for k in ("b", "g", "r")):
|
|
self._set_swatch_bgr(int(data["b"]), int(data["g"]), int(data["r"]))
|
|
elif hasattr(self, "swatch"):
|
|
self._update_swatch()
|
|
finally:
|
|
self._applying_remote = False
|
|
|
|
payload = self._build_vision_payload(consume_pick=False)
|
|
if payload is not None:
|
|
self._last_vision_payload = json.dumps(
|
|
payload, separators=(",", ":"), sort_keys=True
|
|
)
|
|
|
|
first = not self._vision_settings_loaded
|
|
self._vision_settings_loaded = True
|
|
if first:
|
|
h, s, v = self.detector.get_hsv_center()
|
|
ht, st, vt = self.detector.get_tolerance()
|
|
bgr = ""
|
|
if all(k in data for k in ("b", "g", "r")):
|
|
bgr = f" BGR=({int(data['b'])},{int(data['g'])},{int(data['r'])})"
|
|
self.append_log(
|
|
f"ESP settings loaded: HSV=({h},{s},{v}){bgr} "
|
|
f"tol=({ht},{st},{vt}) method={self.detector.get_method()} "
|
|
f"stride={self.coarse_stride.get()} mask={self.show_mask.get()} "
|
|
f"local={bool(data.get('local'))}"
|
|
)
|
|
|
|
def _sync_vision_to_esp(self, delay_ms: int = 150) -> None:
|
|
if self._applying_remote:
|
|
return
|
|
if not self._vision_settings_loaded:
|
|
return
|
|
if not isinstance(self.detector, OrangeBallDetector):
|
|
return
|
|
if self._vision_sync_after_id is not None:
|
|
self.root.after_cancel(self._vision_sync_after_id)
|
|
self._vision_sync_after_id = None
|
|
|
|
def _do() -> None:
|
|
self._vision_sync_after_id = None
|
|
if self._applying_remote or not self._vision_settings_loaded:
|
|
return
|
|
payload = self._build_vision_payload()
|
|
if payload is None:
|
|
return
|
|
blob = json.dumps(payload, separators=(",", ":"), sort_keys=True)
|
|
if blob == self._last_vision_payload:
|
|
return
|
|
self._last_vision_payload = blob
|
|
|
|
def _worker() -> None:
|
|
if not self._ws_send(payload):
|
|
self.append_log("! vision sync failed (WS not connected)")
|
|
|
|
threading.Thread(target=_worker, daemon=True).start()
|
|
|
|
self._vision_sync_after_id = self.root.after(delay_ms, _do)
|
|
|
|
def _capture_timeout(self) -> float:
|
|
if self.show_mask.get():
|
|
return max(self.timeout, 15.0)
|
|
return max(self.timeout, 6.0)
|
|
|
|
def _capture_period(self) -> float:
|
|
if self.show_mask.get():
|
|
return max(self.period, 0.5)
|
|
return max(self.period, 0.2)
|
|
|
|
def _on_show_mask_change(self) -> None:
|
|
if self._applying_remote:
|
|
return
|
|
self._last_vision_payload = None
|
|
self._sync_vision_to_esp()
|
|
|
|
def _on_method_change(self, _event: object | None = None) -> None:
|
|
if self._applying_remote:
|
|
return
|
|
if not isinstance(self.detector, OrangeBallDetector):
|
|
return
|
|
label = self.method_var.get()
|
|
key = self._method_key_from_label(label)
|
|
if key is None or key not in ESP_METHODS:
|
|
return
|
|
self.detector.set_method(key)
|
|
self.append_log(f"method → {label}")
|
|
self._sync_vision_to_esp()
|
|
|
|
def _on_eyedropper_toggle(self) -> None:
|
|
if self.eyedropper_on.get():
|
|
self.video.configure(cursor="crosshair")
|
|
self.append_log("Eyedropper on — click the video")
|
|
else:
|
|
self.video.configure(cursor="arrow")
|
|
|
|
def _update_swatch(self) -> None:
|
|
if not isinstance(self.detector, OrangeBallDetector):
|
|
return
|
|
b, g, r = self.detector.get_bgr_center()
|
|
color = f"#{r:02x}{g:02x}{b:02x}"
|
|
self.swatch.configure(bg=color)
|
|
self.swatch.delete("all")
|
|
self.swatch.create_rectangle(0, 0, 36, 24, fill=color, outline="")
|
|
|
|
def _pick_colour(self) -> None:
|
|
if not isinstance(self.detector, OrangeBallDetector):
|
|
return
|
|
self.eyedropper_on.set(False)
|
|
self._on_eyedropper_toggle()
|
|
b, g, r = self.detector.get_bgr_center()
|
|
result = colorchooser.askcolor(
|
|
color=(r, g, b), title="Target ball colour", parent=self.root
|
|
)
|
|
if not result or result[0] is None:
|
|
return
|
|
r2, g2, b2 = (int(v) for v in result[0])
|
|
self.detector.set_from_bgr(b2, g2, r2)
|
|
self._update_swatch()
|
|
self.append_log(f"colour set RGB=({r2},{g2},{b2})")
|
|
self._sync_vision_to_esp()
|
|
|
|
def _on_tol_change(self, _value: str) -> None:
|
|
if not isinstance(self.detector, OrangeBallDetector):
|
|
return
|
|
tol = int(round(float(self.tol_var.get())))
|
|
if hasattr(self, "tol_label"):
|
|
self.tol_label.configure(text=str(tol))
|
|
if self._applying_remote:
|
|
return
|
|
self.detector.set_tolerance(h_tol=tol, s_tol=tol * 6, v_tol=tol * 6)
|
|
self._sync_vision_to_esp()
|
|
|
|
def _video_click_to_frame(self, event: tk.Event) -> tuple[int, int] | None: # type: ignore[type-arg]
|
|
"""Map a click on the video label to pixel coords in the source frame."""
|
|
scale = self._preview_scale
|
|
disp_w, disp_h = self._preview_display_size
|
|
src_w, src_h = self._preview_src_size
|
|
if scale <= 0 or disp_w <= 0 or disp_h <= 0:
|
|
return None
|
|
|
|
widget_w = max(1, self.video.winfo_width())
|
|
widget_h = max(1, self.video.winfo_height())
|
|
offset_x = max(0, (widget_w - disp_w) // 2)
|
|
offset_y = max(0, (widget_h - disp_h) // 2)
|
|
|
|
ix = event.x - offset_x
|
|
iy = event.y - offset_y
|
|
if not (0 <= ix < disp_w and 0 <= iy < disp_h):
|
|
return None
|
|
|
|
x = int(round(ix / scale))
|
|
y = int(round(iy / scale))
|
|
x = max(0, min(x, src_w - 1))
|
|
y = max(0, min(y, src_h - 1))
|
|
return x, y
|
|
|
|
def _on_video_click(self, event: tk.Event) -> None: # type: ignore[type-arg]
|
|
if not isinstance(self.detector, OrangeBallDetector):
|
|
return
|
|
if not self.eyedropper_on.get():
|
|
return
|
|
if self.show_mask.get():
|
|
self.append_log("! turn off Show mask to pick colour from live image")
|
|
return
|
|
with self._frame_lock:
|
|
frame = None if self._raw_frame is None else self._raw_frame.copy()
|
|
if frame is None:
|
|
return
|
|
pt = self._video_click_to_frame(event)
|
|
if pt is None:
|
|
self.append_log("! eyedropper: click on the video image")
|
|
return
|
|
x, y = pt
|
|
b, g, r = (int(v) for v in frame[y, x])
|
|
self.detector.set_from_bgr(b, g, r)
|
|
self._update_swatch()
|
|
self.append_log(f"eyedrop @ ({x},{y}) RGB=({r},{g},{b})")
|
|
self._pending_pick = (x, y)
|
|
self._last_vision_payload = None
|
|
self._sync_vision_to_esp(delay_ms=0)
|
|
self.eyedropper_on.set(False)
|
|
self._on_eyedropper_toggle()
|
|
|
|
def append_log(self, line: str) -> None:
|
|
def _do() -> None:
|
|
self.log.configure(state="normal")
|
|
self.log.insert("end", line + "\n")
|
|
self.log.see("end")
|
|
self.log.configure(state="disabled")
|
|
|
|
self.root.after(0, _do)
|
|
|
|
def _ws_loop(self) -> None:
|
|
while not self._stop.is_set():
|
|
self._ws_ready.clear()
|
|
|
|
def on_open(_ws: WebSocketApp) -> None:
|
|
self._ws_ready.set()
|
|
self.append_log("WS connected — fetching ESP vision settings")
|
|
self._vision_settings_loaded = False
|
|
self.root.after(200, self._request_vision_settings)
|
|
|
|
def on_message(_ws: WebSocketApp, message: str) -> None:
|
|
self.append_log(f"< {message}")
|
|
try:
|
|
data = json.loads(message)
|
|
except (TypeError, json.JSONDecodeError):
|
|
return
|
|
if isinstance(data, dict) and data.get("type") == "vision":
|
|
self.root.after(0, self._apply_vision_from_esp, data)
|
|
|
|
def on_error(_ws: WebSocketApp, error: object) -> None:
|
|
self.append_log(f"! WS error: {error}")
|
|
|
|
def on_close(_ws: WebSocketApp, *_args: object) -> None:
|
|
self._ws_ready.clear()
|
|
self.append_log("WS closed")
|
|
|
|
ws = WebSocketApp(
|
|
self.ws_url,
|
|
on_open=on_open,
|
|
on_message=on_message,
|
|
on_error=on_error,
|
|
on_close=on_close,
|
|
)
|
|
with self._ws_lock:
|
|
self._ws = ws
|
|
try:
|
|
ws.run_forever(ping_interval=20, ping_timeout=10)
|
|
except Exception as exc: # noqa: BLE001
|
|
self.append_log(f"! WS run failed: {exc}")
|
|
with self._ws_lock:
|
|
self._ws = None
|
|
if not self._stop.is_set():
|
|
time.sleep(1.0)
|
|
|
|
def _ws_send(self, payload: dict) -> bool:
|
|
data = json.dumps(payload, separators=(",", ":"))
|
|
with self._ws_lock:
|
|
ws = self._ws
|
|
ready = self._ws_ready.is_set()
|
|
if not ws or not ready:
|
|
return False
|
|
try:
|
|
ws.send(data)
|
|
return True
|
|
except Exception as exc: # noqa: BLE001
|
|
self.append_log(f"! WS send failed: {exc}")
|
|
return False
|
|
|
|
def send_cmd(self, direction: str) -> None:
|
|
def _worker() -> None:
|
|
if self._ws_send({"type": "cmd", "dir": direction}):
|
|
return
|
|
try:
|
|
r = self.session.get(
|
|
f"{self.base}/cmd",
|
|
params={"dir": direction},
|
|
timeout=self.timeout,
|
|
)
|
|
self.append_log(f"< {r.text.strip()}")
|
|
except requests.RequestException as exc:
|
|
self.append_log(f"! cmd {direction} failed: {exc}")
|
|
|
|
threading.Thread(target=_worker, daemon=True).start()
|
|
|
|
def send_text(self) -> None:
|
|
text = self.entry.get().strip()
|
|
if not text:
|
|
return
|
|
self.entry.delete(0, "end")
|
|
|
|
def _worker() -> None:
|
|
if self._ws_send({"type": "msg", "text": text}):
|
|
return
|
|
try:
|
|
r = self.session.post(
|
|
f"{self.base}/msg",
|
|
data=text.encode("utf-8"),
|
|
headers={"Content-Type": "text/plain; charset=utf-8"},
|
|
timeout=self.timeout,
|
|
)
|
|
self.append_log(f"< {r.text.strip()}")
|
|
except requests.RequestException as exc:
|
|
self.append_log(f"! send failed: {exc}")
|
|
|
|
threading.Thread(target=_worker, daemon=True).start()
|
|
|
|
def _show_frame(self, frame_bgr: np.ndarray) -> None:
|
|
preview_w = 640
|
|
h, w = frame_bgr.shape[:2]
|
|
scale = preview_w / float(w)
|
|
preview_h = max(1, int(round(h * scale)))
|
|
self._preview_scale = scale
|
|
self._preview_display_size = (preview_w, preview_h)
|
|
self._preview_src_size = (w, h)
|
|
resized = cv2.resize(frame_bgr, (preview_w, preview_h), interpolation=cv2.INTER_AREA)
|
|
rgb = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB)
|
|
image = Image.fromarray(rgb)
|
|
photo = ImageTk.PhotoImage(image=image)
|
|
self._photo = photo
|
|
self.video.configure(image=photo)
|
|
|
|
def _video_loop(self) -> None:
|
|
capture_url = f"{self.base}/capture"
|
|
vision_url = f"{self.base}/vision"
|
|
while not self._stop.is_set():
|
|
t0 = time.perf_counter()
|
|
try:
|
|
esp_stats: dict[str, float | int | bool] | None = None
|
|
if self.detect_only.get():
|
|
with self._frame_lock:
|
|
need_seed = self._raw_frame is None
|
|
if need_seed:
|
|
self._seed_display_frame(capture_url)
|
|
resp = self.session.get(vision_url, timeout=self._capture_timeout())
|
|
resp.raise_for_status()
|
|
esp_stats = self._parse_vision_json(resp.json())
|
|
self._apply_esp_stats(esp_stats)
|
|
self._refresh_detect_overlay(esp_stats)
|
|
else:
|
|
resp = self.session.get(capture_url, timeout=self._capture_timeout())
|
|
resp.raise_for_status()
|
|
esp_stats = self._apply_vision_headers(resp.headers)
|
|
buf = np.frombuffer(resp.content, dtype=np.uint8)
|
|
frame = cv2.imdecode(buf, cv2.IMREAD_COLOR)
|
|
if frame is not None:
|
|
with self._frame_lock:
|
|
self._raw_frame = frame.copy()
|
|
display = frame
|
|
if esp_stats is not None:
|
|
display = self._annotate_esp_target(frame, esp_stats)
|
|
self.root.after(0, self._show_frame, display)
|
|
except requests.RequestException as exc:
|
|
self.append_log(f"! video: {exc}")
|
|
time.sleep(1.0)
|
|
|
|
period = self._capture_period()
|
|
if period > 0:
|
|
elapsed = time.perf_counter() - t0
|
|
sleep_s = period - elapsed
|
|
if sleep_s > 0:
|
|
time.sleep(sleep_s)
|
|
|
|
def on_close(self) -> None:
|
|
self._stop.set()
|
|
with self._ws_lock:
|
|
if self._ws is not None:
|
|
try:
|
|
self._ws.close()
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
self.session.close()
|
|
self.root.destroy()
|
|
|
|
def run(self) -> None:
|
|
self.root.mainloop()
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(
|
|
description="ESP32 camera + local (on-device) vision UI"
|
|
)
|
|
parser.add_argument("--ip", default="192.168.1.178")
|
|
parser.add_argument("--fps", type=float, default=10.0)
|
|
parser.add_argument("--timeout", type=float, default=2.0)
|
|
parser.add_argument(
|
|
"--target",
|
|
default="orange_ball",
|
|
help="Vision target module: orange_ball | none",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
detector = create_target(args.target)
|
|
app = RobotApp(
|
|
ip=args.ip,
|
|
fps=args.fps,
|
|
timeout=args.timeout,
|
|
detector=detector,
|
|
)
|
|
app.run()
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|