feat(graphics): DisplayMirror for screen mirroring and remote input

include/graphics/DeviceScreen.h

@@ -34,6 +34,9 @@ class DeviceScreen
     void task_handler(void);
     void toggleDisplay(void);
 
+    /** The display driver behind this screen, for hosts that drive it directly. */
+    DisplayDriver *getDisplayDriver(void) const { return gui->getDisplayDriver(); }
+
 #if defined(ARDUINO_ARCH_ESP32)
     int prepareSleep(void *);
     int wakeUp(esp_sleep_wakeup_cause_t cause);

include/graphics/DisplayMirror.h

@@ -0,0 +1,38 @@
+#pragma once
+
+#include <cstdint>
+
+class DisplayDriver;
+
+/**
+ * @brief Streams the rendered screen to a host and feeds the host's input back
+ *        into LVGL. Idle until start(); no driver knows this class exists.
+ */
+class DisplayMirror
+{
+  public:
+    // LVGL thread: pixels is the area's top-left, rows stride pixels apart, in the
+    // byte order pixelsByteSwapped() reports. Copy, never write, and return.
+    using FrameObserver = void (*)(int16_t x, int16_t y, uint16_t width, uint16_t height, const uint16_t *pixels,
+                                   uint16_t stride);
+
+    // once, after DeviceScreen::init() and before the UI task starts
+    static void start(DisplayDriver *driver);
+
+    // nullptr stops capture; a flush may still be in flight for one UI tick
+    static void setFrameObserver(FrameObserver observer);
+
+    static bool pixelsByteSwapped(void);
+
+    // any thread; repaints the screen and its overlay layers
+    static void requestFullRefresh(void);
+
+    // remote input, from a single producer thread; dropped when the queue is full
+    static void injectTouch(int16_t x, int16_t y);
+    // held past this device's long-press threshold, so it behaves like a finger
+    static void injectLongPress(int16_t x, int16_t y);
+    static void injectKey(uint32_t key); // LV_KEY_* or a printable character
+    static void injectLongPressKey(uint32_t key);
+    // moves group focus; negative is backwards, clamped to a byte
+    static void injectEncoder(int16_t steps);
+};

include/graphics/driver/DisplayDriver.h

@@ -3,6 +3,7 @@
 #include "graphics/DeviceGUI.h"
 #include "graphics/LVGL/LVGLGraphics.h"
 #include <cstdint>
+#include <functional>
 
 #define H_NORM_PX(h_scr_percent) ((int16_t)((screenWidth / 100.0) * (h_scr_percent)))
 #define V_NORM_PX(v_scr_percent) ((int16_t)((screenHeight / 100.0) * (v_scr_percent)))
@@ -42,7 +43,17 @@ class DisplayDriver
 
     lv_display_t *getDisplay(void) { return display; }
 
+    // pixels: top-left of the flushed area, rows stride pixels apart; LVGL thread
+    using FlushCallback =
+        std::function<void(int16_t x, int16_t y, uint16_t width, uint16_t height, const uint16_t *pixels, uint16_t stride)>;
+
+    // set once, before the UI task starts
+    void setFlushCB(FlushCallback cb) { flushCB = std::move(cb); }
+
   protected:
+    // subclasses call this from their LVGL flush callback
+    void flush(lv_display_t *disp, const lv_area_t *area, const uint8_t *px_map);
+
     LVGLGraphics lvgl;
     LVGLDisplay *display;
     LVGLTouch *touch;
@@ -52,4 +63,5 @@ class DisplayDriver
 
   private:
     static void displayToggleCb(void *displayDriver);
+    FlushCallback flushCB;
 };

include/graphics/driver/LGFXDriver.h

@@ -226,6 +226,7 @@ template <class LGFX> void LGFXDriver<LGFX>::display_flush(lv_display_t *disp, c
 {
     uint32_t w = lv_area_get_width(area);
     uint32_t h = lv_area_get_height(area);
+    static_cast<LGFXDriver *>(lv_display_get_driver_data(disp))->flush(disp, area, px_map);
     {
         ISpiLock::Guard bus;
         lgfx->startWrite();
@@ -248,12 +249,14 @@ template <class LGFX> void LGFXDriver<LGFX>::display_flush(lv_display_t *disp, c
     uint32_t h = lv_area_get_height(area);
     lgfx->setAddrWindow(area->x1, area->y1, w, h);
     lgfx->pushPixelsDMA((uint16_t *)px_map, w * h);
+    static_cast<LGFXDriver *>(lv_display_get_driver_data(disp))->flush(disp, area, px_map);
     lv_display_flush_ready(disp);
 }
 #elif defined(USE_FULL_DOUBLE_BUFFER)
 template <class LGFX> void LGFXDriver<LGFX>::display_flush(lv_display_t *disp, const lv_area_t *area, uint8_t *px_map)
 {
     auto driver = static_cast<LGFXDriver *>(lv_display_get_driver_data(disp));
+    driver->flush(disp, area, px_map);
     if (!driver->flushAreaValid) {
         driver->flushArea = *area;
         driver->flushAreaValid = true;
@@ -290,6 +293,7 @@ template <class LGFX> void LGFXDriver<LGFX>::display_flush(lv_display_t *disp, c
 {
     uint32_t w = lv_area_get_width(area);
     uint32_t h = lv_area_get_height(area);
+    static_cast<LGFXDriver *>(lv_display_get_driver_data(disp))->flush(disp, area, px_map);
     {
         ISpiLock::Guard bus;
         lgfx->pushImage(area->x1, area->y1, w, h, (uint16_t *)px_map);

source/graphics/DisplayMirror.cpp

@@ -0,0 +1,273 @@
+#include "graphics/DisplayMirror.h"
+#include "graphics/driver/DisplayDriver.h"
+#include "lvgl.h"
+#include "src/indev/lv_indev_private.h"
+#include "util/ILog.h"
+#include <atomic>
+
+namespace
+{
+struct Touch {
+    int16_t x, y;
+    uint16_t holdMs;
+};
+struct Key {
+    uint32_t key;
+    uint16_t holdMs;
+};
+constexpr uint8_t queueLen = 16; // SPSC ring; one slot is the full/empty marker
+// a hold must outlast the long-press threshold by at least one read, wherever in a read period it starts
+constexpr uint16_t longPressMarginMs = 2 * LV_DEF_REFR_PERIOD;
+
+std::atomic<DisplayMirror::FrameObserver> frameObserver{nullptr};
+std::atomic<bool> fullRefreshRequested{false};
+std::atomic<bool> wakeRequested{false};
+
+DisplayDriver *displaydriver = nullptr;
+bool byteSwapped = false;
+lv_indev_t *pointer = nullptr;
+lv_indev_t *keypad = nullptr;
+lv_indev_t *encoder = nullptr;
+
+Touch touchQueue[queueLen];
+std::atomic<uint8_t> touchHead{0}, touchTail{0};
+Key keyQueue[queueLen];
+std::atomic<uint8_t> keyHead{0}, keyTail{0};
+int8_t encoderQueue[queueLen];
+std::atomic<uint8_t> encoderHead{0}, encoderTail{0};
+
+// LVGL thread. Remote input must wake a slept panel and still act, or every event is spent as a wake.
+void service_requests(void)
+{
+    if (wakeRequested.exchange(false, std::memory_order_acquire) && displaydriver) {
+        if (displaydriver->isPowersaving())
+            displaydriver->forceWakeup();
+        if (lv_display_t *display = displaydriver->getDisplay())
+            lv_display_trigger_activity(display);
+    }
+    if (fullRefreshRequested.exchange(false, std::memory_order_acquire)) {
+        lv_obj_invalidate(lv_screen_active());
+        lv_obj_invalidate(lv_layer_top());
+        lv_obj_invalidate(lv_layer_sys());
+    }
+}
+
+// A queued touch is held PRESSED for holdMs (at least one read), then RELEASED once.
+void pointer_read(lv_indev_t *indev, lv_indev_data_t *data)
+{
+    static bool pressing = false;
+    static bool needRelease = false;
+    static uint32_t pressStart = 0;
+    static lv_point_t last = {0, 0};
+
+    service_requests();
+
+    if (pressing) {
+        const Touch &t = touchQueue[touchHead.load(std::memory_order_relaxed)];
+        if (lv_tick_elaps(pressStart) >= t.holdMs) {
+            pressing = false;
+            needRelease = true;
+            touchHead.store((touchHead.load(std::memory_order_relaxed) + 1) % queueLen, std::memory_order_release);
+        }
+        data->point = last;
+        data->state = LV_INDEV_STATE_PRESSED;
+        return;
+    }
+    if (needRelease) {
+        needRelease = false;
+        data->point = last;
+        data->state = LV_INDEV_STATE_RELEASED;
+        return;
+    }
+    if (touchHead.load(std::memory_order_relaxed) != touchTail.load(std::memory_order_acquire)) {
+        const Touch &t = touchQueue[touchHead.load(std::memory_order_relaxed)];
+        last.x = t.x;
+        last.y = t.y;
+        pressing = true;
+        pressStart = lv_tick_get();
+        data->point = last;
+        data->state = LV_INDEV_STATE_PRESSED;
+        return;
+    }
+    data->point = last;
+    data->state = LV_INDEV_STATE_RELEASED;
+}
+
+// A queued key is held PRESSED for holdMs (at least one read), then RELEASED once.
+void keypad_read(lv_indev_t *indev, lv_indev_data_t *data)
+{
+    static bool pressing = false;
+    static bool needRelease = false;
+    static uint32_t pressStart = 0;
+    static uint32_t lastKey = 0;
+
+    service_requests();
+
+    if (pressing) {
+        const Key &k = keyQueue[keyHead.load(std::memory_order_relaxed)];
+        if (lv_tick_elaps(pressStart) >= k.holdMs) {
+            pressing = false;
+            needRelease = true;
+            keyHead.store((keyHead.load(std::memory_order_relaxed) + 1) % queueLen, std::memory_order_release);
+        }
+        data->key = lastKey;
+        data->state = LV_INDEV_STATE_PRESSED;
+        return;
+    }
+    if (needRelease) {
+        needRelease = false;
+        data->key = lastKey;
+        data->state = LV_INDEV_STATE_RELEASED;
+        return;
+    }
+    if (keyHead.load(std::memory_order_relaxed) != keyTail.load(std::memory_order_acquire)) {
+        lastKey = keyQueue[keyHead.load(std::memory_order_relaxed)].key;
+        pressing = true;
+        pressStart = lv_tick_get();
+        data->key = lastKey;
+        data->state = LV_INDEV_STATE_PRESSED;
+        return;
+    }
+    data->key = lastKey;
+    data->state = LV_INDEV_STATE_RELEASED;
+}
+
+void encoder_read(lv_indev_t *indev, lv_indev_data_t *data)
+{
+    service_requests();
+
+    data->state = LV_INDEV_STATE_RELEASED;
+    data->enc_diff = 0;
+    uint8_t head = encoderHead.load(std::memory_order_relaxed);
+    if (head != encoderTail.load(std::memory_order_acquire)) {
+        data->enc_diff = encoderQueue[head];
+        encoderHead.store((head + 1) % queueLen, std::memory_order_release);
+    }
+}
+
+// The physical touch's long-press time, so a remote hold behaves like a finger.
+uint16_t touch_long_press_time(void)
+{
+    for (lv_indev_t *indev = lv_indev_get_next(nullptr); indev; indev = lv_indev_get_next(indev)) {
+        if (indev != pointer && lv_indev_get_type(indev) == LV_INDEV_TYPE_POINTER)
+            return indev->long_press_time;
+    }
+    return 0;
+}
+} // namespace
+
+void DisplayMirror::start(DisplayDriver *driver)
+{
+    if (displaydriver || !driver)
+        return;
+    displaydriver = driver;
+    lv_display_t *display = driver->getDisplay();
+    byteSwapped = display && lv_display_get_color_format(display) == LV_COLOR_FORMAT_RGB565_SWAPPED;
+
+    driver->setFlushCB([](int16_t x, int16_t y, uint16_t width, uint16_t height, const uint16_t *pixels, uint16_t stride) {
+        if (auto observer = frameObserver.load(std::memory_order_acquire))
+            observer(x, y, width, height, pixels, stride);
+    });
+
+    // every input driver makes its group the default in init(); a board with none gets one here
+    lv_group_t *group = lv_group_get_default();
+    if (!group) {
+        group = lv_group_create();
+        lv_group_set_default(group);
+    }
+
+    keypad = lv_indev_create();
+    lv_indev_set_type(keypad, LV_INDEV_TYPE_KEYPAD);
+    lv_indev_set_read_cb(keypad, keypad_read);
+    lv_indev_set_group(keypad, group);
+
+    encoder = lv_indev_create();
+    lv_indev_set_type(encoder, LV_INDEV_TYPE_ENCODER);
+    lv_indev_set_read_cb(encoder, encoder_read);
+    lv_indev_set_group(encoder, group);
+
+    const uint16_t longPressTime = touch_long_press_time();
+    pointer = lv_indev_create();
+    lv_indev_set_type(pointer, LV_INDEV_TYPE_POINTER);
+    lv_indev_set_read_cb(pointer, pointer_read);
+    if (longPressTime)
+        lv_indev_set_long_press_time(pointer, longPressTime);
+    ILOG_DEBUG("DisplayMirror: virtual input devices ready");
+}
+
+void DisplayMirror::setFrameObserver(FrameObserver observer)
+{
+    frameObserver.store(observer, std::memory_order_release);
+}
+
+bool DisplayMirror::pixelsByteSwapped(void)
+{
+    return byteSwapped;
+}
+
+void DisplayMirror::requestFullRefresh(void)
+{
+    fullRefreshRequested.store(true, std::memory_order_release);
+}
+
+namespace
+{
+void push_touch(int16_t x, int16_t y, uint16_t holdMs)
+{
+    uint8_t tail = touchTail.load(std::memory_order_relaxed);
+    uint8_t next = (tail + 1) % queueLen;
+    if (next == touchHead.load(std::memory_order_acquire))
+        return;
+    touchQueue[tail] = {x, y, holdMs};
+    wakeRequested.store(true, std::memory_order_release);
+    touchTail.store(next, std::memory_order_release);
+}
+
+void push_key(uint32_t key, uint16_t holdMs)
+{
+    uint8_t tail = keyTail.load(std::memory_order_relaxed);
+    uint8_t next = (tail + 1) % queueLen;
+    if (next == keyHead.load(std::memory_order_acquire))
+        return;
+    keyQueue[tail] = {key, holdMs};
+    wakeRequested.store(true, std::memory_order_release);
+    keyTail.store(next, std::memory_order_release);
+}
+
+// long_press_time is set once in start(), before any inject call
+uint16_t long_press_hold(const lv_indev_t *indev)
+{
+    return indev ? indev->long_press_time + longPressMarginMs : 0;
+}
+} // namespace
+
+void DisplayMirror::injectTouch(int16_t x, int16_t y)
+{
+    push_touch(x, y, 0);
+}
+
+void DisplayMirror::injectLongPress(int16_t x, int16_t y)
+{
+    push_touch(x, y, long_press_hold(pointer));
+}
+
+void DisplayMirror::injectKey(uint32_t key)
+{
+    push_key(key, 0);
+}
+
+void DisplayMirror::injectLongPressKey(uint32_t key)
+{
+    push_key(key, long_press_hold(keypad));
+}
+
+void DisplayMirror::injectEncoder(int16_t steps)
+{
+    uint8_t tail = encoderTail.load(std::memory_order_relaxed);
+    uint8_t next = (tail + 1) % queueLen;
+    if (next == encoderHead.load(std::memory_order_acquire))
+        return;
+    encoderQueue[tail] = (int8_t)(steps > 127 ? 127 : (steps < -127 ? -127 : steps));
+    wakeRequested.store(true, std::memory_order_release);
+    encoderTail.store(next, std::memory_order_release);
+}