From 2b4886039f4923050ea2d79a13c0bedf8c940ff7 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Mon, 2 Jun 2025 13:19:35 +1000 Subject: [PATCH 01/74] aiorepl: Fix Enter key handling in raw terminal mode. Handle both CR (0x0D) and LF (0x0A) for command execution to ensure compatibility with raw terminal mode where Enter sends CR instead of LF. This fixes the issue where aiorepl required Ctrl+Enter instead of just Enter to execute commands when used with MicroPython ports that put stdin in raw mode (such as the updated unix port using pyexec). Also improves handling of various newline sequences (CRLF, double-LF, double-CR) to prevent double-execution of commands. Signed-off-by: Andrew Leech --- micropython/aiorepl/aiorepl.py | 18 ++++++++++++------ micropython/aiorepl/manifest.py | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/micropython/aiorepl/aiorepl.py b/micropython/aiorepl/aiorepl.py index 15026e435..8d5d41342 100644 --- a/micropython/aiorepl/aiorepl.py +++ b/micropython/aiorepl/aiorepl.py @@ -121,16 +121,22 @@ async def task(g=None, prompt="--> "): pt = t # save previous time t = time.ticks_ms() if c < 0x20 or c > 0x7E: - if c == 0x0A: - # LF + if c == 0x0A or c == 0x0D: + # LF or CR (handle both for raw terminal mode compatibility) if paste: + # In paste mode, preserve the actual character sys.stdout.write(b) cmd += b continue - # If the previous character was also LF, and was less - # than 20 ms ago, this was likely due to CRLF->LFLF - # conversion, so ignore this linefeed. - if pc == 0x0A and time.ticks_diff(t, pt) < 20: + # Handle various newline sequences to avoid double-execution: + # - CR+LF (Windows style): ignore LF if it follows CR quickly + # - LF+LF (PTY double-newline): ignore second LF if it follows quickly + # - CR+CR (potential double-CR): ignore second CR if it follows quickly + if ( + (c == 0x0A and pc == 0x0D) # LF after CR (CRLF) + or (c == 0x0A and pc == 0x0A) # LF after LF (double LF) + or (c == 0x0D and pc == 0x0D) + ) and time.ticks_diff(t, pt) < 20: # CR after CR continue if curs: # move cursor to end of the line diff --git a/micropython/aiorepl/manifest.py b/micropython/aiorepl/manifest.py index 83802e1c0..32a20dbed 100644 --- a/micropython/aiorepl/manifest.py +++ b/micropython/aiorepl/manifest.py @@ -1,5 +1,5 @@ metadata( - version="0.2.2", + version="0.2.3", description="Provides an asynchronous REPL that can run concurrently with an asyncio, also allowing await expressions.", ) From 0b188385670c69b7d46fd84f76a734dee2f412e5 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Sun, 15 Feb 2026 13:40:14 +1100 Subject: [PATCH 02/74] usb-device-cdc: Fix default timeout in CDCInterface.init(). CDCInterface.__init__() sets self._timeout = 1000, then calls self.init(**kwargs). The init() method had timeout=None as default, which unconditionally overwrites self._timeout with None. This causes TypeError in read(), write(), readinto(), and ioctl() which all compare int >= self._timeout. Set the default timeout=1000 in init() to match the intended default, consistent with how other parameters (baudrate, bits, etc.) have their defaults specified directly in the init() signature. Signed-off-by: Andrew Leech --- micropython/usb/usb-device-cdc/manifest.py | 2 +- micropython/usb/usb-device-cdc/usb/device/cdc.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/micropython/usb/usb-device-cdc/manifest.py b/micropython/usb/usb-device-cdc/manifest.py index e844b6f01..3807dbee5 100644 --- a/micropython/usb/usb-device-cdc/manifest.py +++ b/micropython/usb/usb-device-cdc/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1.2") +metadata(version="0.1.3") require("usb-device") package("usb") diff --git a/micropython/usb/usb-device-cdc/usb/device/cdc.py b/micropython/usb/usb-device-cdc/usb/device/cdc.py index 4ec012bc5..a51941bc0 100644 --- a/micropython/usb/usb-device-cdc/usb/device/cdc.py +++ b/micropython/usb/usb-device-cdc/usb/device/cdc.py @@ -122,7 +122,7 @@ def __init__(self, **kwargs): self.init(**kwargs) def init( - self, baudrate=9600, bits=8, parity="N", stop=1, timeout=None, txbuf=256, rxbuf=256, flow=0 + self, baudrate=9600, bits=8, parity="N", stop=1, timeout=1000, txbuf=256, rxbuf=256, flow=0 ): # Configure the CDC serial port. Note that many of these settings like # baudrate, bits, parity, stop don't change the USB-CDC device behavior From 501daf47dcc5843ed09b1df50b522678e22047e0 Mon Sep 17 00:00:00 2001 From: Matt Trentini Date: Thu, 30 Sep 2021 22:30:07 +1000 Subject: [PATCH 03/74] colorsys: Add colorsys and test_colorsys from CPython. Copied from CPython v3.14.6 with no changes. Signed-off-by: Damien George --- python-stdlib/colorsys/colorsys.py | 169 ++++++++++++++++++++++++ python-stdlib/colorsys/manifest.py | 3 + python-stdlib/colorsys/test_colorsys.py | 113 ++++++++++++++++ tools/ci.sh | 1 + 4 files changed, 286 insertions(+) create mode 100644 python-stdlib/colorsys/colorsys.py create mode 100644 python-stdlib/colorsys/manifest.py create mode 100644 python-stdlib/colorsys/test_colorsys.py diff --git a/python-stdlib/colorsys/colorsys.py b/python-stdlib/colorsys/colorsys.py new file mode 100644 index 000000000..cae77b619 --- /dev/null +++ b/python-stdlib/colorsys/colorsys.py @@ -0,0 +1,169 @@ +# This file is copied verbatim from CPython v3.14.6. +# Its license is the standard Python License. + +"""Conversion functions between RGB and other color systems. + +This modules provides two functions for each color system ABC: + + rgb_to_abc(r, g, b) --> a, b, c + abc_to_rgb(a, b, c) --> r, g, b + +All inputs and outputs are triples of floats in the range [0.0...1.0] +(with the exception of I and Q, which covers a slightly larger range). +Inputs outside the valid range may cause exceptions or invalid outputs. + +Supported color systems: +RGB: Red, Green, Blue components +YIQ: Luminance, Chrominance (used by composite video signals) +HLS: Hue, Luminance, Saturation +HSV: Hue, Saturation, Value +""" + +# References: +# http://en.wikipedia.org/wiki/YIQ +# http://en.wikipedia.org/wiki/HLS_color_space +# http://en.wikipedia.org/wiki/HSV_color_space + +__all__ = ["rgb_to_yiq","yiq_to_rgb","rgb_to_hls","hls_to_rgb", + "rgb_to_hsv","hsv_to_rgb"] + +# Some floating-point constants + +ONE_THIRD = 1.0/3.0 +ONE_SIXTH = 1.0/6.0 +TWO_THIRD = 2.0/3.0 + +# YIQ: used by composite video signals (linear combinations of RGB) +# Y: perceived grey level (0.0 == black, 1.0 == white) +# I, Q: color components +# +# There are a great many versions of the constants used in these formulae. +# The ones in this library uses constants from the FCC version of NTSC. + +def rgb_to_yiq(r, g, b): + y = 0.30*r + 0.59*g + 0.11*b + i = 0.74*(r-y) - 0.27*(b-y) + q = 0.48*(r-y) + 0.41*(b-y) + return (y, i, q) + +def yiq_to_rgb(y, i, q): + # r = y + (0.27*q + 0.41*i) / (0.74*0.41 + 0.27*0.48) + # b = y + (0.74*q - 0.48*i) / (0.74*0.41 + 0.27*0.48) + # g = y - (0.30*(r-y) + 0.11*(b-y)) / 0.59 + + r = y + 0.9468822170900693*i + 0.6235565819861433*q + g = y - 0.27478764629897834*i - 0.6356910791873801*q + b = y - 1.1085450346420322*i + 1.7090069284064666*q + + if r < 0.0: + r = 0.0 + if g < 0.0: + g = 0.0 + if b < 0.0: + b = 0.0 + if r > 1.0: + r = 1.0 + if g > 1.0: + g = 1.0 + if b > 1.0: + b = 1.0 + return (r, g, b) + + +# HLS: Hue, Luminance, Saturation +# H: position in the spectrum +# L: color lightness +# S: color saturation + +def rgb_to_hls(r, g, b): + maxc = max(r, g, b) + minc = min(r, g, b) + sumc = (maxc+minc) + rangec = (maxc-minc) + l = sumc/2.0 + if minc == maxc: + return 0.0, l, 0.0 + if l <= 0.5: + s = rangec / sumc + else: + s = rangec / (2.0-maxc-minc) # Not always 2.0-sumc: gh-106498. + rc = (maxc-r) / rangec + gc = (maxc-g) / rangec + bc = (maxc-b) / rangec + if r == maxc: + h = bc-gc + elif g == maxc: + h = 2.0+rc-bc + else: + h = 4.0+gc-rc + h = (h/6.0) % 1.0 + return h, l, s + +def hls_to_rgb(h, l, s): + if s == 0.0: + return l, l, l + if l <= 0.5: + m2 = l * (1.0+s) + else: + m2 = l+s-(l*s) + m1 = 2.0*l - m2 + return (_v(m1, m2, h+ONE_THIRD), _v(m1, m2, h), _v(m1, m2, h-ONE_THIRD)) + +def _v(m1, m2, hue): + hue = hue % 1.0 + if hue < ONE_SIXTH: + return m1 + (m2-m1)*hue*6.0 + if hue < 0.5: + return m2 + if hue < TWO_THIRD: + return m1 + (m2-m1)*(TWO_THIRD-hue)*6.0 + return m1 + + +# HSV: Hue, Saturation, Value +# H: position in the spectrum +# S: color saturation ("purity") +# V: color brightness + +def rgb_to_hsv(r, g, b): + maxc = max(r, g, b) + minc = min(r, g, b) + rangec = (maxc-minc) + v = maxc + if minc == maxc: + return 0.0, 0.0, v + s = rangec / maxc + rc = (maxc-r) / rangec + gc = (maxc-g) / rangec + bc = (maxc-b) / rangec + if r == maxc: + h = bc-gc + elif g == maxc: + h = 2.0+rc-bc + else: + h = 4.0+gc-rc + h = (h/6.0) % 1.0 + return h, s, v + +def hsv_to_rgb(h, s, v): + if s == 0.0: + return v, v, v + i = int(h*6.0) # XXX assume int() truncates! + f = (h*6.0) - i + p = v*(1.0 - s) + q = v*(1.0 - s*f) + t = v*(1.0 - s*(1.0-f)) + i = i%6 + if i == 0: + return v, t, p + if i == 1: + return q, v, p + if i == 2: + return p, v, t + if i == 3: + return p, q, v + if i == 4: + return t, p, v + if i == 5: + return v, p, q + # Cannot get here diff --git a/python-stdlib/colorsys/manifest.py b/python-stdlib/colorsys/manifest.py new file mode 100644 index 000000000..55a3199e0 --- /dev/null +++ b/python-stdlib/colorsys/manifest.py @@ -0,0 +1,3 @@ +metadata(version="1.0.0") + +module("colorsys.py") diff --git a/python-stdlib/colorsys/test_colorsys.py b/python-stdlib/colorsys/test_colorsys.py new file mode 100644 index 000000000..68f9f6d67 --- /dev/null +++ b/python-stdlib/colorsys/test_colorsys.py @@ -0,0 +1,113 @@ +# This file is copied verbatim from CPython v3.14.6. +# Its license is the standard Python License. + +import unittest +import colorsys + +def frange(start, stop, step): + while start <= stop: + yield start + start += step + +class ColorsysTest(unittest.TestCase): + + def assertTripleEqual(self, tr1, tr2): + self.assertEqual(len(tr1), 3) + self.assertEqual(len(tr2), 3) + self.assertAlmostEqual(tr1[0], tr2[0]) + self.assertAlmostEqual(tr1[1], tr2[1]) + self.assertAlmostEqual(tr1[2], tr2[2]) + + def test_hsv_roundtrip(self): + for r in frange(0.0, 1.0, 0.2): + for g in frange(0.0, 1.0, 0.2): + for b in frange(0.0, 1.0, 0.2): + rgb = (r, g, b) + self.assertTripleEqual( + rgb, + colorsys.hsv_to_rgb(*colorsys.rgb_to_hsv(*rgb)) + ) + + def test_hsv_values(self): + values = [ + # rgb, hsv + ((0.0, 0.0, 0.0), ( 0 , 0.0, 0.0)), # black + ((0.0, 0.0, 1.0), (4./6., 1.0, 1.0)), # blue + ((0.0, 1.0, 0.0), (2./6., 1.0, 1.0)), # green + ((0.0, 1.0, 1.0), (3./6., 1.0, 1.0)), # cyan + ((1.0, 0.0, 0.0), ( 0 , 1.0, 1.0)), # red + ((1.0, 0.0, 1.0), (5./6., 1.0, 1.0)), # purple + ((1.0, 1.0, 0.0), (1./6., 1.0, 1.0)), # yellow + ((1.0, 1.0, 1.0), ( 0 , 0.0, 1.0)), # white + ((0.5, 0.5, 0.5), ( 0 , 0.0, 0.5)), # grey + ] + for (rgb, hsv) in values: + self.assertTripleEqual(hsv, colorsys.rgb_to_hsv(*rgb)) + self.assertTripleEqual(rgb, colorsys.hsv_to_rgb(*hsv)) + + def test_hls_roundtrip(self): + for r in frange(0.0, 1.0, 0.2): + for g in frange(0.0, 1.0, 0.2): + for b in frange(0.0, 1.0, 0.2): + rgb = (r, g, b) + self.assertTripleEqual( + rgb, + colorsys.hls_to_rgb(*colorsys.rgb_to_hls(*rgb)) + ) + + def test_hls_values(self): + values = [ + # rgb, hls + ((0.0, 0.0, 0.0), ( 0 , 0.0, 0.0)), # black + ((0.0, 0.0, 1.0), (4./6., 0.5, 1.0)), # blue + ((0.0, 1.0, 0.0), (2./6., 0.5, 1.0)), # green + ((0.0, 1.0, 1.0), (3./6., 0.5, 1.0)), # cyan + ((1.0, 0.0, 0.0), ( 0 , 0.5, 1.0)), # red + ((1.0, 0.0, 1.0), (5./6., 0.5, 1.0)), # purple + ((1.0, 1.0, 0.0), (1./6., 0.5, 1.0)), # yellow + ((1.0, 1.0, 1.0), ( 0 , 1.0, 0.0)), # white + ((0.5, 0.5, 0.5), ( 0 , 0.5, 0.0)), # grey + ] + for (rgb, hls) in values: + self.assertTripleEqual(hls, colorsys.rgb_to_hls(*rgb)) + self.assertTripleEqual(rgb, colorsys.hls_to_rgb(*hls)) + + def test_hls_nearwhite(self): # gh-106498 + values = ( + # rgb, hls: these do not work in reverse + ((0.9999999999999999, 1, 1), (0.5, 1.0, 1.0)), + ((1, 0.9999999999999999, 0.9999999999999999), (0.0, 1.0, 1.0)), + ) + for rgb, hls in values: + self.assertTripleEqual(hls, colorsys.rgb_to_hls(*rgb)) + self.assertTripleEqual((1.0, 1.0, 1.0), colorsys.hls_to_rgb(*hls)) + + def test_yiq_roundtrip(self): + for r in frange(0.0, 1.0, 0.2): + for g in frange(0.0, 1.0, 0.2): + for b in frange(0.0, 1.0, 0.2): + rgb = (r, g, b) + self.assertTripleEqual( + rgb, + colorsys.yiq_to_rgb(*colorsys.rgb_to_yiq(*rgb)) + ) + + def test_yiq_values(self): + values = [ + # rgb, yiq + ((0.0, 0.0, 0.0), (0.0, 0.0, 0.0)), # black + ((0.0, 0.0, 1.0), (0.11, -0.3217, 0.3121)), # blue + ((0.0, 1.0, 0.0), (0.59, -0.2773, -0.5251)), # green + ((0.0, 1.0, 1.0), (0.7, -0.599, -0.213)), # cyan + ((1.0, 0.0, 0.0), (0.3, 0.599, 0.213)), # red + ((1.0, 0.0, 1.0), (0.41, 0.2773, 0.5251)), # purple + ((1.0, 1.0, 0.0), (0.89, 0.3217, -0.3121)), # yellow + ((1.0, 1.0, 1.0), (1.0, 0.0, 0.0)), # white + ((0.5, 0.5, 0.5), (0.5, 0.0, 0.0)), # grey + ] + for (rgb, yiq) in values: + self.assertTripleEqual(yiq, colorsys.rgb_to_yiq(*rgb)) + self.assertTripleEqual(rgb, colorsys.yiq_to_rgb(*yiq)) + +if __name__ == "__main__": + unittest.main() diff --git a/tools/ci.sh b/tools/ci.sh index 2040185c0..efbd4d186 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -92,6 +92,7 @@ function ci_package_tests_run { for path in \ micropython/ucontextlib \ + python-stdlib/colorsys \ python-stdlib/contextlib \ python-stdlib/datetime \ python-stdlib/fnmatch \ From da7584d9116ac80fa8eed3d559ac1518a17786b3 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Mon, 15 Oct 2018 16:41:53 +1100 Subject: [PATCH 04/74] uuid: Provide UUID class and uuid4() implementation using os.urandom(). Includes unit test. Signed-off-by: Damien George --- python-stdlib/uuid/manifest.py | 3 +++ python-stdlib/uuid/test_uuid.py | 15 +++++++++++++++ python-stdlib/uuid/uuid.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+) create mode 100644 python-stdlib/uuid/manifest.py create mode 100644 python-stdlib/uuid/test_uuid.py create mode 100644 python-stdlib/uuid/uuid.py diff --git a/python-stdlib/uuid/manifest.py b/python-stdlib/uuid/manifest.py new file mode 100644 index 000000000..e59bdf2c0 --- /dev/null +++ b/python-stdlib/uuid/manifest.py @@ -0,0 +1,3 @@ +metadata(version="0.1.0") + +module("uuid.py") diff --git a/python-stdlib/uuid/test_uuid.py b/python-stdlib/uuid/test_uuid.py new file mode 100644 index 000000000..5e9c81483 --- /dev/null +++ b/python-stdlib/uuid/test_uuid.py @@ -0,0 +1,15 @@ +import uuid + +u1 = uuid.uuid4() +u2 = uuid.uuid4() + +assert str(u1) != str(u2), "Two uuid4 should not match" + +assert len(str(u1)) == len(str(u1)) == 36 + +assert str(repr(u1)).startswith("") + +assert len(u1.hex) == 32 + +print("OK") diff --git a/python-stdlib/uuid/uuid.py b/python-stdlib/uuid/uuid.py new file mode 100644 index 000000000..db16473db --- /dev/null +++ b/python-stdlib/uuid/uuid.py @@ -0,0 +1,28 @@ +import os +import ubinascii + + +class UUID: + def __init__(self, bytes): + if len(bytes) != 16: + raise ValueError('bytes arg must be 16 bytes long') + self._bytes = bytes + + @property + def hex(self): + return ubinascii.hexlify(self._bytes).decode() + + def __str__(self): + h = self.hex + return '-'.join((h[0:8], h[8:12], h[12:16], h[16:20], h[20:32])) + + def __repr__(self): + return "" % str(self) + + +def uuid4(): + """Generates a random UUID compliant to RFC 4122 pg.14""" + random = bytearray(os.urandom(16)) + random[6] = (random[6] & 0x0F) | 0x40 + random[8] = (random[8] & 0x3F) | 0x80 + return UUID(bytes=random) From 31a65f120df608daeb8cfc87f9e98ad08b7ed604 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 11 Jul 2026 15:02:12 +1000 Subject: [PATCH 05/74] uuid: Convert test to use unittest. And run it as part of CI. Signed-off-by: Damien George --- python-stdlib/uuid/test_uuid.py | 29 +++++++++++++++++++++-------- tools/ci.sh | 1 + 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/python-stdlib/uuid/test_uuid.py b/python-stdlib/uuid/test_uuid.py index 5e9c81483..25aa308c1 100644 --- a/python-stdlib/uuid/test_uuid.py +++ b/python-stdlib/uuid/test_uuid.py @@ -1,15 +1,28 @@ +import unittest import uuid -u1 = uuid.uuid4() -u2 = uuid.uuid4() -assert str(u1) != str(u2), "Two uuid4 should not match" +class TestUUID(unittest.TestCase): + def test_unique(self): + n = 10 + us = set(uuid.uuid4().bytes for _ in range(n)) + self.assertEqual(len(us), n) -assert len(str(u1)) == len(str(u1)) == 36 + def test_len(self): + u = uuid.uuid4() + self.assertEqual(len(str(u)), 36) + self.assertEqual(len(u.hex), 32) -assert str(repr(u1)).startswith("") + def test_repr(self): + u = str(repr(uuid.uuid4())) + self.assertTrue(u.startswith("")) -assert len(u1.hex) == 32 + def test_constructor(self): + u1 = uuid.uuid4() + u2 = uuid.UUID(bytes.fromhex(u1.hex)) + self.assertEqual(u1.hex, u2.hex) -print("OK") + +if __name__ == "__main__": + unittest.main() diff --git a/tools/ci.sh b/tools/ci.sh index efbd4d186..2fc7ba004 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -106,6 +106,7 @@ function ci_package_tests_run { python-stdlib/time \ python-stdlib/unittest/tests \ python-stdlib/unittest-discover/tests \ + python-stdlib/uuid \ ; do (cd $path && "${MICROPYTHON}" -m unittest) if [ $? -ne 0 ]; then false; return; fi From ff6c2d4fd61cfb615e138b0aca79f7991ff6a388 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 11 Jul 2026 15:02:38 +1000 Subject: [PATCH 06/74] uuid: Use bytes.hex instead of binascii.hexlify. Using `bytes.hex()` eliminates an import, and eliminates the call to `.decode()` to convert it to a str. Also run ruff format. Signed-off-by: Damien George --- python-stdlib/uuid/uuid.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/python-stdlib/uuid/uuid.py b/python-stdlib/uuid/uuid.py index db16473db..9790923c3 100644 --- a/python-stdlib/uuid/uuid.py +++ b/python-stdlib/uuid/uuid.py @@ -1,20 +1,19 @@ import os -import ubinascii class UUID: def __init__(self, bytes): if len(bytes) != 16: - raise ValueError('bytes arg must be 16 bytes long') + raise ValueError("bytes arg must be 16 bytes long") self._bytes = bytes @property def hex(self): - return ubinascii.hexlify(self._bytes).decode() + return self._bytes.hex() def __str__(self): h = self.hex - return '-'.join((h[0:8], h[8:12], h[12:16], h[16:20], h[20:32])) + return "-".join((h[0:8], h[8:12], h[12:16], h[16:20], h[20:32])) def __repr__(self): return "" % str(self) From 81fd893eb53ee377e9e1f5c19dac6ea8329e4bc4 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 20 Jul 2026 23:47:57 +1000 Subject: [PATCH 07/74] uuid: Add UUID.bytes attribute. Signed-off-by: Damien George --- python-stdlib/uuid/test_uuid.py | 3 ++- python-stdlib/uuid/uuid.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/python-stdlib/uuid/test_uuid.py b/python-stdlib/uuid/test_uuid.py index 25aa308c1..91e6cc53a 100644 --- a/python-stdlib/uuid/test_uuid.py +++ b/python-stdlib/uuid/test_uuid.py @@ -10,8 +10,9 @@ def test_unique(self): def test_len(self): u = uuid.uuid4() - self.assertEqual(len(str(u)), 36) + self.assertEqual(len(u.bytes), 16) self.assertEqual(len(u.hex), 32) + self.assertEqual(len(str(u)), 36) def test_repr(self): u = str(repr(uuid.uuid4())) diff --git a/python-stdlib/uuid/uuid.py b/python-stdlib/uuid/uuid.py index 9790923c3..0b3a278c9 100644 --- a/python-stdlib/uuid/uuid.py +++ b/python-stdlib/uuid/uuid.py @@ -5,11 +5,11 @@ class UUID: def __init__(self, bytes): if len(bytes) != 16: raise ValueError("bytes arg must be 16 bytes long") - self._bytes = bytes + self.bytes = bytes @property def hex(self): - return self._bytes.hex() + return self.bytes.hex() def __str__(self): h = self.hex @@ -24,4 +24,4 @@ def uuid4(): random = bytearray(os.urandom(16)) random[6] = (random[6] & 0x0F) | 0x40 random[8] = (random[8] & 0x3F) | 0x80 - return UUID(bytes=random) + return UUID(bytes(random)) From 00109e91b86526f28af7646a75f11f7705080ed3 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 20 Jul 2026 23:48:11 +1000 Subject: [PATCH 08/74] uuid: Make UUID.__repr__ match CPython. Signed-off-by: Damien George --- python-stdlib/uuid/test_uuid.py | 7 ++++--- python-stdlib/uuid/uuid.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/python-stdlib/uuid/test_uuid.py b/python-stdlib/uuid/test_uuid.py index 91e6cc53a..efd472296 100644 --- a/python-stdlib/uuid/test_uuid.py +++ b/python-stdlib/uuid/test_uuid.py @@ -15,9 +15,10 @@ def test_len(self): self.assertEqual(len(str(u)), 36) def test_repr(self): - u = str(repr(uuid.uuid4())) - self.assertTrue(u.startswith("")) + u = repr(uuid.uuid4()) + self.assertEqual(len(u), 44) + self.assertTrue(u.startswith("UUID('")) + self.assertTrue(u.endswith("')")) def test_constructor(self): u1 = uuid.uuid4() diff --git a/python-stdlib/uuid/uuid.py b/python-stdlib/uuid/uuid.py index 0b3a278c9..b131f7a7a 100644 --- a/python-stdlib/uuid/uuid.py +++ b/python-stdlib/uuid/uuid.py @@ -16,7 +16,7 @@ def __str__(self): return "-".join((h[0:8], h[8:12], h[12:16], h[16:20], h[20:32])) def __repr__(self): - return "" % str(self) + return "UUID('{}')".format(self) def uuid4(): From e13a8e8fea3cadeea2618055b91ff2b57601adc5 Mon Sep 17 00:00:00 2001 From: Pablo Ventura Date: Thu, 16 Jul 2026 13:02:22 -0300 Subject: [PATCH 09/74] umqtt.simple: Encode subscribe Remaining Length as VBI. Use the same variable-byte Remaining Length encoding as publish for subscribe and unsubscribe so long topics no longer overflow a single length byte. Fixes #969. Signed-off-by: Pablo Ventura --- micropython/umqtt.simple/test_umqtt_simple.py | 76 +++++++++++++++++++ micropython/umqtt.simple/umqtt/simple.py | 32 ++++++-- 2 files changed, 102 insertions(+), 6 deletions(-) create mode 100644 micropython/umqtt.simple/test_umqtt_simple.py diff --git a/micropython/umqtt.simple/test_umqtt_simple.py b/micropython/umqtt.simple/test_umqtt_simple.py new file mode 100644 index 000000000..c31efd17a --- /dev/null +++ b/micropython/umqtt.simple/test_umqtt_simple.py @@ -0,0 +1,76 @@ +import io +import sys + + +class Socket: + def __init__(self, read_data=b""): + self._write_buffer = io.BytesIO() + self._read_buffer = io.BytesIO(read_data) + + def write(self, buf, length=None): + if length is None: + length = len(buf) + self._write_buffer.write(buf[:length]) + + def read(self, n): + return self._read_buffer.read(n) + + def setblocking(self, blocking): + pass + + def close(self): + pass + + +sys.path.insert(0, "micropython/umqtt.simple") +# ruff: noqa: E402 +from umqtt.simple import MQTTClient + + +def make_client(read_data): + c = MQTTClient(b"cid", "127.0.0.1") + c.sock = Socket(read_data) + c.set_callback(lambda topic, msg: None) + return c + + +def test_subscribe_short_topic(): + # Remaining Length = 5 + 4 = 9 -> single VBI byte 0x09, pid=1 + c = make_client(b"\x90\x03\x00\x01\x00") + c.subscribe(b"abcd", qos=0) + out = c.sock._write_buffer.getvalue() + assert out[:2] == b"\x82\x09", out + assert out[2:4] == b"\x00\x01", out + assert out[4:6] == b"\x00\x04", out + assert out[6:10] == b"abcd", out + assert out[10:11] == b"\x00", out + + +def test_subscribe_long_topic(): + # Remaining Length = 5 + 123 = 128 -> VBI 0x80 0x01 + topic = b"a" * 123 + c = make_client(b"\x90\x03\x00\x01\x00") + c.subscribe(topic, qos=0) + out = c.sock._write_buffer.getvalue() + assert out[:3] == b"\x82\x80\x01", out + assert out[3:5] == b"\x00\x01", out + assert out[5:7] == b"\x00\x7b", out + assert out[7:130] == topic, out + assert out[130:131] == b"\x00", out + + +def test_unsubscribe_long_topic(): + # Remaining Length = 4 + 123 = 127 -> still one VBI byte; use 124 for 128 + topic = b"a" * 124 + c = make_client(b"\xb0\x02\x00\x01") + c.unsubscribe(topic) + out = c.sock._write_buffer.getvalue() + assert out[:3] == b"\xa2\x80\x01", out + assert out[3:5] == b"\x00\x01", out + assert out[5:7] == b"\x00\x7c", out + assert out[7:131] == topic, out + + +test_subscribe_short_topic() +test_subscribe_long_topic() +test_unsubscribe_long_topic() diff --git a/micropython/umqtt.simple/umqtt/simple.py b/micropython/umqtt.simple/umqtt/simple.py index bfdaf503c..68cf807a0 100644 --- a/micropython/umqtt.simple/umqtt/simple.py +++ b/micropython/umqtt.simple/umqtt/simple.py @@ -162,9 +162,19 @@ def subscribe(self, topic, qos=0): assert self.cb is not None, "Subscribe callback is not set" pkt = bytearray(b"\x82\0\0\0") self.pid += 1 - struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic) + 1, self.pid) + pid = self.pid + sz = 2 + 2 + len(topic) + 1 + assert sz < 2097152 + i = 1 + while sz > 0x7F: + pkt[i] = (sz & 0x7F) | 0x80 + sz >>= 7 + i += 1 + pkt[i] = sz # print(hex(len(pkt)), hexlify(pkt, ":")) - self.sock.write(pkt) + self.sock.write(pkt, i + 1) + struct.pack_into("!H", pkt, 0, pid) + self.sock.write(pkt, 2) self._send_str(topic) self.sock.write(qos.to_bytes(1, "little")) while 1: @@ -172,7 +182,7 @@ def subscribe(self, topic, qos=0): if op == 0x90: resp = self.sock.read(4) # print(resp) - assert resp[1] == pkt[2] and resp[2] == pkt[3] + assert resp[1] == pid >> 8 and resp[2] == (pid & 0xFF) if resp[3] == 0x80: raise MQTTException(resp[3]) return @@ -180,14 +190,24 @@ def subscribe(self, topic, qos=0): def unsubscribe(self, topic): pkt = bytearray(b"\xa2\0\0\0") self.pid += 1 - struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic), self.pid) - self.sock.write(pkt) + pid = self.pid + sz = 2 + 2 + len(topic) + assert sz < 2097152 + i = 1 + while sz > 0x7F: + pkt[i] = (sz & 0x7F) | 0x80 + sz >>= 7 + i += 1 + pkt[i] = sz + self.sock.write(pkt, i + 1) + struct.pack_into("!H", pkt, 0, pid) + self.sock.write(pkt, 2) self._send_str(topic) while 1: op = self.wait_msg() if op == 0xB0: resp = self.sock.read(3) - assert resp[1] == pkt[2] and resp[2] == pkt[3] + assert resp[1] == pid >> 8 and resp[2] == (pid & 0xFF) return # Wait for a single incoming MQTT message and process it. From e2a694990df0e476407ce983c628343e95fdd2b9 Mon Sep 17 00:00:00 2001 From: Pablo Ventura Date: Mon, 20 Jul 2026 10:49:39 -0300 Subject: [PATCH 10/74] umqtt.simple: Share subscribe/unsubscribe packet encoding. Factor Remaining Length VBI and ACK wait into _send_subunsub to reduce .mpy size after the long-topic fix. Signed-off-by: Pablo Ventura --- micropython/umqtt.simple/umqtt/simple.py | 45 ++++++++---------------- 1 file changed, 14 insertions(+), 31 deletions(-) diff --git a/micropython/umqtt.simple/umqtt/simple.py b/micropython/umqtt.simple/umqtt/simple.py index 68cf807a0..e7967f41e 100644 --- a/micropython/umqtt.simple/umqtt/simple.py +++ b/micropython/umqtt.simple/umqtt/simple.py @@ -158,12 +158,12 @@ def publish(self, topic, msg, retain=False, qos=0): elif qos == 2: assert 0 - def subscribe(self, topic, qos=0): - assert self.cb is not None, "Subscribe callback is not set" - pkt = bytearray(b"\x82\0\0\0") + def _send_subunsub(self, topic, typ, ack_op, ack_n, qos=None): + pkt = bytearray(4) + pkt[0] = typ self.pid += 1 pid = self.pid - sz = 2 + 2 + len(topic) + 1 + sz = 4 + len(topic) + (0 if qos is None else 1) assert sz < 2097152 i = 1 while sz > 0x7F: @@ -171,44 +171,27 @@ def subscribe(self, topic, qos=0): sz >>= 7 i += 1 pkt[i] = sz - # print(hex(len(pkt)), hexlify(pkt, ":")) self.sock.write(pkt, i + 1) struct.pack_into("!H", pkt, 0, pid) self.sock.write(pkt, 2) self._send_str(topic) - self.sock.write(qos.to_bytes(1, "little")) + if qos is not None: + self.sock.write(qos.to_bytes(1, "little")) while 1: op = self.wait_msg() - if op == 0x90: - resp = self.sock.read(4) - # print(resp) + if op == ack_op: + resp = self.sock.read(ack_n) assert resp[1] == pid >> 8 and resp[2] == (pid & 0xFF) - if resp[3] == 0x80: + if ack_n == 4 and resp[3] == 0x80: raise MQTTException(resp[3]) return + def subscribe(self, topic, qos=0): + assert self.cb is not None, "Subscribe callback is not set" + self._send_subunsub(topic, 0x82, 0x90, 4, qos) + def unsubscribe(self, topic): - pkt = bytearray(b"\xa2\0\0\0") - self.pid += 1 - pid = self.pid - sz = 2 + 2 + len(topic) - assert sz < 2097152 - i = 1 - while sz > 0x7F: - pkt[i] = (sz & 0x7F) | 0x80 - sz >>= 7 - i += 1 - pkt[i] = sz - self.sock.write(pkt, i + 1) - struct.pack_into("!H", pkt, 0, pid) - self.sock.write(pkt, 2) - self._send_str(topic) - while 1: - op = self.wait_msg() - if op == 0xB0: - resp = self.sock.read(3) - assert resp[1] == pid >> 8 and resp[2] == (pid & 0xFF) - return + self._send_subunsub(topic, 0xA2, 0xB0, 3) # Wait for a single incoming MQTT message and process it. # Subscribed messages are delivered to a callback previously From 315af6d30d386471fe719550e55c6ec194ba7c77 Mon Sep 17 00:00:00 2001 From: Pablo Ventura Date: Mon, 20 Jul 2026 10:55:00 -0300 Subject: [PATCH 11/74] umqtt.simple: Share Remaining Length VBI encoding. Factor the Variable Byte Integer encoder into a module helper and reuse it from connect, publish, and the subscribe/unsubscribe path to cut .mpy size. Signed-off-by: Pablo Ventura --- micropython/umqtt.simple/umqtt/simple.py | 41 ++++++++++-------------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/micropython/umqtt.simple/umqtt/simple.py b/micropython/umqtt.simple/umqtt/simple.py index e7967f41e..a8016f376 100644 --- a/micropython/umqtt.simple/umqtt/simple.py +++ b/micropython/umqtt.simple/umqtt/simple.py @@ -7,6 +7,16 @@ class MQTTException(Exception): pass +def _encode_len(pkt, sz): + i = 1 + while sz > 0x7F: + pkt[i] = (sz & 0x7F) | 0x80 + sz >>= 7 + i += 1 + pkt[i] = sz + return i + + class MQTTClient: def __init__( self, @@ -93,12 +103,7 @@ def connect(self, clean_session=True, timeout=None): msg[6] |= 0x4 | (self.lw_qos & 0x1) << 3 | (self.lw_qos & 0x2) << 3 msg[6] |= self.lw_retain << 5 - i = 1 - while sz > 0x7F: - premsg[i] = (sz & 0x7F) | 0x80 - sz >>= 7 - i += 1 - premsg[i] = sz + i = _encode_len(premsg, sz) self.sock.write(premsg, i + 2) self.sock.write(msg) @@ -130,12 +135,7 @@ def publish(self, topic, msg, retain=False, qos=0): if qos > 0: sz += 2 assert sz < 2097152 - i = 1 - while sz > 0x7F: - pkt[i] = (sz & 0x7F) | 0x80 - sz >>= 7 - i += 1 - pkt[i] = sz + i = _encode_len(pkt, sz) # print(hex(len(pkt)), hexlify(pkt, ":")) self.sock.write(pkt, i + 1) self._send_str(topic) @@ -163,26 +163,19 @@ def _send_subunsub(self, topic, typ, ack_op, ack_n, qos=None): pkt[0] = typ self.pid += 1 pid = self.pid - sz = 4 + len(topic) + (0 if qos is None else 1) - assert sz < 2097152 - i = 1 - while sz > 0x7F: - pkt[i] = (sz & 0x7F) | 0x80 - sz >>= 7 - i += 1 - pkt[i] = sz + i = _encode_len(pkt, 4 + len(topic) + (qos != None)) self.sock.write(pkt, i + 1) struct.pack_into("!H", pkt, 0, pid) self.sock.write(pkt, 2) self._send_str(topic) - if qos is not None: - self.sock.write(qos.to_bytes(1, "little")) + if qos != None: + self.sock.write(bytes((qos,))) while 1: op = self.wait_msg() if op == ack_op: resp = self.sock.read(ack_n) - assert resp[1] == pid >> 8 and resp[2] == (pid & 0xFF) - if ack_n == 4 and resp[3] == 0x80: + assert (resp[1] << 8 | resp[2]) == pid + if ack_n > 3 and resp[3] == 0x80: raise MQTTException(resp[3]) return From 327a15e930eacc5097f2cea9f1b9c4929a47b509 Mon Sep 17 00:00:00 2001 From: Pablo Ventura Date: Mon, 20 Jul 2026 11:06:03 -0300 Subject: [PATCH 12/74] umqtt.simple: Avoid None comparisons in _send_subunsub. Use ack_n to decide whether to include the QoS byte so ruff E711 passes without growing the compiled .mpy size. Signed-off-by: Pablo Ventura --- micropython/umqtt.simple/manifest.py | 2 +- micropython/umqtt.simple/umqtt/simple.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/micropython/umqtt.simple/manifest.py b/micropython/umqtt.simple/manifest.py index efbda68f5..2e4d33abb 100644 --- a/micropython/umqtt.simple/manifest.py +++ b/micropython/umqtt.simple/manifest.py @@ -1,4 +1,4 @@ -metadata(description="Lightweight MQTT client for MicroPython.", version="1.7.1") +metadata(description="Lightweight MQTT client for MicroPython.", version="1.8.0") # Originally written by Paul Sokolovsky. diff --git a/micropython/umqtt.simple/umqtt/simple.py b/micropython/umqtt.simple/umqtt/simple.py index a8016f376..7539711e3 100644 --- a/micropython/umqtt.simple/umqtt/simple.py +++ b/micropython/umqtt.simple/umqtt/simple.py @@ -158,17 +158,17 @@ def publish(self, topic, msg, retain=False, qos=0): elif qos == 2: assert 0 - def _send_subunsub(self, topic, typ, ack_op, ack_n, qos=None): + def _send_subunsub(self, topic, typ, ack_op, ack_n, qos=0): pkt = bytearray(4) pkt[0] = typ self.pid += 1 pid = self.pid - i = _encode_len(pkt, 4 + len(topic) + (qos != None)) + i = _encode_len(pkt, 4 + len(topic) + (ack_n > 3)) self.sock.write(pkt, i + 1) struct.pack_into("!H", pkt, 0, pid) self.sock.write(pkt, 2) self._send_str(topic) - if qos != None: + if ack_n > 3: self.sock.write(bytes((qos,))) while 1: op = self.wait_msg() From 64907fc6b3711c2c5fe3cfd893f1f9625c6c0bdf Mon Sep 17 00:00:00 2001 From: Pablo Ventura Date: Mon, 27 Jul 2026 10:49:00 -0300 Subject: [PATCH 13/74] tools/ci.sh: Add umqtt.simple mock test to package tests CI. Signed-off-by: Pablo Ventura --- tools/ci.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ci.sh b/tools/ci.sh index 2fc7ba004..4c56fb9c5 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -57,6 +57,7 @@ function ci_package_tests_run { export MICROPYPATH for test in \ micropython/drivers/storage/sdcard/sdtest.py \ + micropython/umqtt.simple/test_umqtt_simple.py \ micropython/xmltok/test_xmltok.py \ python-ecosys/requests/test_requests.py \ python-stdlib/argparse/test_argparse.py \ From 77b8ee72d024f50a77a10109cb1bcf88135422f5 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 15 Jul 2026 16:11:40 +1000 Subject: [PATCH 14/74] usb-device-cdc: Optimise CDCInterface constructor. Don't set instance attributes that will just be reset by `self.init()`. Saves 30 bytes of bytecode. Signed-off-by: Damien George --- micropython/usb/usb-device-cdc/manifest.py | 2 +- micropython/usb/usb-device-cdc/usb/device/cdc.py | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/micropython/usb/usb-device-cdc/manifest.py b/micropython/usb/usb-device-cdc/manifest.py index 3807dbee5..94e1f7847 100644 --- a/micropython/usb/usb-device-cdc/manifest.py +++ b/micropython/usb/usb-device-cdc/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1.3") +metadata(version="0.1.4") require("usb-device") package("usb") diff --git a/micropython/usb/usb-device-cdc/usb/device/cdc.py b/micropython/usb/usb-device-cdc/usb/device/cdc.py index a51941bc0..ca59b811c 100644 --- a/micropython/usb/usb-device-cdc/usb/device/cdc.py +++ b/micropython/usb/usb-device-cdc/usb/device/cdc.py @@ -107,18 +107,14 @@ def __init__(self, **kwargs): self.line_coding_cb = None self._line_state = 0 # DTR & RTS - # Set a default line coding of 115200/8N1 - self._line_coding = bytearray(b"\x00\xc2\x01\x00\x00\x00\x08") - - self._wb = () # Optional write Buffer (IN endpoint), set by CDC.init() - self._rb = () # Optional read Buffer (OUT endpoint), set by CDC.init() - self._timeout = 1000 # set from CDC.init() as well + self._line_coding = bytearray(7) # Will be populated by .init() # one control interface endpoint, two data interface endpoints self.ep_c_in = self.ep_d_in = self.ep_d_out = None self._c_itf = None # Number of control interface, data interface is one more + # The _timeout, _wb and _rb attributes will be set by this call to .init(). self.init(**kwargs) def init( From 769c8fcadc8a5d80d3d9e94733886a4032c335af Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 15 Jul 2026 16:13:53 +1000 Subject: [PATCH 15/74] usb/examples: Put timeout=0 in CDCInterface constructor. Makes the example a little bit simpler. Signed-off-by: Damien George --- micropython/usb/examples/device/cdc_repl_example.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/micropython/usb/examples/device/cdc_repl_example.py b/micropython/usb/examples/device/cdc_repl_example.py index 06dc9a76c..d27a89766 100644 --- a/micropython/usb/examples/device/cdc_repl_example.py +++ b/micropython/usb/examples/device/cdc_repl_example.py @@ -25,8 +25,8 @@ import usb.device from usb.device.cdc import CDCInterface -cdc = CDCInterface() -cdc.init(timeout=0) # zero timeout makes this non-blocking, suitable for os.dupterm() +# Zero timeout makes this non-blocking, suitable for os.dupterm(). +cdc = CDCInterface(timeout=0) # pass builtin_driver=True so that we get the built-in USB-CDC alongside, # if it's available. From 641e8b142c7edfec58fb0dd7e12dbe2ebb3150b5 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Thu, 23 Sep 2021 14:42:37 +1000 Subject: [PATCH 16/74] aioble/security: Only schedule save when needed. Was getting occasional: RuntimeError: schedule queue full Signed-off-by: Damien George --- micropython/bluetooth/aioble-security/manifest.py | 2 +- micropython/bluetooth/aioble/aioble/security.py | 9 +++------ micropython/bluetooth/aioble/manifest.py | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/micropython/bluetooth/aioble-security/manifest.py b/micropython/bluetooth/aioble-security/manifest.py index 5737d2a06..8a0d68b94 100644 --- a/micropython/bluetooth/aioble-security/manifest.py +++ b/micropython/bluetooth/aioble-security/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.2.0") +metadata(version="0.2.1") require("aioble-core") diff --git a/micropython/bluetooth/aioble/aioble/security.py b/micropython/bluetooth/aioble/aioble/security.py index 8e04d5b7b..b30681841 100644 --- a/micropython/bluetooth/aioble/aioble/security.py +++ b/micropython/bluetooth/aioble/aioble/security.py @@ -57,10 +57,6 @@ def _save_secrets(arg=None): _path = _path or _DEFAULT_PATH - if not _modified: - # Only save if the secrets changed. - return - with open(_path, "w") as f: # Convert bytes to hex strings (otherwise JSON will treat them like # strings). @@ -106,8 +102,9 @@ def _security_irq(event, data): _secrets[key] = value # Queue up a save (don't synchronously write to flash). - _modified = True - schedule(_save_secrets, None) + if not _modified: + _modified = True + schedule(_save_secrets, None) return True diff --git a/micropython/bluetooth/aioble/manifest.py b/micropython/bluetooth/aioble/manifest.py index 1cf06c4d8..1deefdf0e 100644 --- a/micropython/bluetooth/aioble/manifest.py +++ b/micropython/bluetooth/aioble/manifest.py @@ -3,7 +3,7 @@ # code. This allows (for development purposes) all the files to live in the # one directory. -metadata(version="0.6.1") +metadata(version="0.6.2") # Default installation gives you everything. Install the individual # components (or a combination of them) if you want a more minimal install. From a2e279cbead2d330e42b1a5e737cabc8d8cf2498 Mon Sep 17 00:00:00 2001 From: PermissionDenied7335 Date: Thu, 19 Oct 2023 08:24:25 +0800 Subject: [PATCH 17/74] requests: Add support for query and anchor chars directly after domain. Supports URLs like http://example.com?query and http://example.com#anchor. Fixes issue #978. Signed-off-by: Damien George --- python-ecosys/requests/manifest.py | 2 +- python-ecosys/requests/requests/__init__.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/python-ecosys/requests/manifest.py b/python-ecosys/requests/manifest.py index 4f76af397..e08c43018 100644 --- a/python-ecosys/requests/manifest.py +++ b/python-ecosys/requests/manifest.py @@ -1,3 +1,3 @@ -metadata(version="1.0.2", pypi="requests") +metadata(version="1.0.3", pypi="requests") package("requests") diff --git a/python-ecosys/requests/requests/__init__.py b/python-ecosys/requests/requests/__init__.py index 41f5cc47b..f9c43a740 100644 --- a/python-ecosys/requests/requests/__init__.py +++ b/python-ecosys/requests/requests/__init__.py @@ -111,6 +111,14 @@ def request( else: raise ValueError("Unsupported protocol: " + proto) + if "?" in host: + host, _path = host.split("?", 1) + path = "?" + _path + path + + if "#" in host: + host, _path = host.split("#", 1) + path = "#" + _path + path + if ":" in host: host, port = host.split(":", 1) port = int(port) From ab6047cc9debdb9f9bf27cb739962ce15be4118b Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 6 Aug 2026 14:13:32 +1000 Subject: [PATCH 18/74] requests: Add tests for query and anchor in URL. These new tests would have failed prior to the parent commit. Signed-off-by: Damien George --- python-ecosys/requests/test_requests.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/python-ecosys/requests/test_requests.py b/python-ecosys/requests/test_requests.py index 7b7a8f6c3..e7acba6c0 100644 --- a/python-ecosys/requests/test_requests.py +++ b/python-ecosys/requests/test_requests.py @@ -58,6 +58,18 @@ def test_simple_get(): ), format_message(response) +def test_get_query_anchor(): + response = requests.request("GET", "http://example.com?query") + assert response.raw._write_buffer.getvalue() == ( + b"GET /?query HTTP/1.1\r\nConnection: close\r\nHost: example.com\r\n\r\n" + ), format_message(response) + + response = requests.request("GET", "http://example.com#anchor") + assert response.raw._write_buffer.getvalue() == ( + b"GET /#anchor HTTP/1.1\r\nConnection: close\r\nHost: example.com\r\n\r\n" + ), format_message(response) + + def test_get_auth(): response = requests.request( "GET", "http://example.com", auth=("test-username", "test-password") @@ -312,6 +324,7 @@ def test_redirect_relative(): test_simple_get() +test_get_query_anchor() test_get_auth() test_get_custom_header() test_post_json() From 33c3822ef47a1ecec59b8d7f4cac10bb6e7c5ed1 Mon Sep 17 00:00:00 2001 From: Pablo Ventura Date: Fri, 3 Jul 2026 21:48:48 -0300 Subject: [PATCH 19/74] requests: Add chunked response body support. Decode Transfer-Encoding: chunked response bodies incrementally with a modified BodyStream wrapper, keeping .raw streaming (read/readinto) and .content lazy without buffering everything in request(). Chunk extensions are skipped and trailers are discarded. Signed-off-by: Pablo Ventura --- python-ecosys/requests/manifest.py | 2 +- python-ecosys/requests/requests/__init__.py | 48 ++++++++++++++------- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/python-ecosys/requests/manifest.py b/python-ecosys/requests/manifest.py index e08c43018..b49ae557e 100644 --- a/python-ecosys/requests/manifest.py +++ b/python-ecosys/requests/manifest.py @@ -1,3 +1,3 @@ -metadata(version="1.0.3", pypi="requests") +metadata(version="1.1.0", pypi="requests") package("requests") diff --git a/python-ecosys/requests/requests/__init__.py b/python-ecosys/requests/requests/__init__.py index f9c43a740..68f63d13c 100644 --- a/python-ecosys/requests/requests/__init__.py +++ b/python-ecosys/requests/requests/__init__.py @@ -4,28 +4,41 @@ class BodyStream: def __init__(self, sock, remaining): self._sock = sock + self._chunk = remaining < 0 self._remaining = remaining def read(self, n=-1): - if self._remaining == 0: - return b"" - if n < 0 or n > self._remaining: - n = self._remaining - data = self._sock.read(n) - self._remaining -= len(data) - if not data: - raise ValueError("Connection closed before Content-Length satisfied") - return data + buf = bytearray(n if n >= 0 else 256) + if n >= 0: + got = self.readinto(buf) + return buf[:got] if got else b"" + result = b"" + while True: + got = self.readinto(buf) + if not got: + return result + result += buf[:got] def readinto(self, buf): - if self._remaining == 0: - return 0 + s = self._sock + if self._remaining <= 0: + if self._remaining == 0: + return 0 + self._remaining = int(s.readline().split(b";")[0], 16) + if self._remaining == 0: + while True: + l = s.readline() + if not l or l == b"\r\n": + return 0 if len(buf) > self._remaining: buf = memoryview(buf)[: self._remaining] - got = self._sock.readinto(buf) - self._remaining -= got + got = s.readinto(buf) if not got: - raise ValueError("Connection closed before Content-Length satisfied") + raise ValueError("Connection closed before body complete") + self._remaining -= got + if self._remaining == 0 and self._chunk: + s.readline() + self._remaining = -1 return got def close(self): @@ -204,6 +217,7 @@ def request( if len(l) > 2: reason = l[2].rstrip() remaining = None + chunked = False while True: l = s.readline() if not l or l == b"\r\n": @@ -211,7 +225,7 @@ def request( # print(l) if l.startswith(b"Transfer-Encoding:"): if b"chunked" in l: - raise ValueError("Unsupported " + str(l, "utf-8")) + chunked = True elif l.startswith(b"Location:") and not 200 <= status <= 299: if status in [301, 302, 303, 307, 308]: redirect = str(l[10:-2], "utf-8") @@ -243,7 +257,9 @@ def request( else: return request(method, redirect, data, json, headers, stream) else: - if remaining is not None: + if chunked: + resp = Response(BodyStream(s, -1)) + elif remaining is not None: resp = Response(BodyStream(s, remaining)) else: resp = Response(s) From 75d3976f4af98bde8a062f44d912671d47697ae9 Mon Sep 17 00:00:00 2001 From: Pablo Ventura Date: Fri, 3 Jul 2026 21:49:19 -0300 Subject: [PATCH 20/74] requests: Test chunked response decoding. Replace the test that expected a ValueError for chunked responses with coverage of decoding via .content and incremental .raw.readinto(). Signed-off-by: Pablo Ventura --- python-ecosys/requests/test_requests.py | 34 ++++++++++++++++--------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/python-ecosys/requests/test_requests.py b/python-ecosys/requests/test_requests.py index e7acba6c0..38f5b8026 100644 --- a/python-ecosys/requests/test_requests.py +++ b/python-ecosys/requests/test_requests.py @@ -233,19 +233,28 @@ def test_content_length_via_content(): socket.socket = lambda *a, **k: Socket() -def test_chunked_response_raises(): +def test_chunked_response_via_content(): socket.socket = lambda *a, **k: Socket( - read_data=b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n" + read_data=b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n6\r\n world\r\n0\r\n\r\n" ) - raised = False - try: - requests.request("GET", "http://example.com") - except ValueError as e: - raised = True - if "Unsupported" not in str(e): - raise - if not raised: - raise AssertionError("expected ValueError for chunked response") + response = requests.request("GET", "http://example.com") + assert response.content == b"hello world" + socket.socket = lambda *a, **k: Socket() + + +def test_chunked_response_readinto(): + socket.socket = lambda *a, **k: Socket( + read_data=b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n3\r\nabc\r\n4\r\ndefg\r\n0\r\n\r\n" + ) + response = requests.request("GET", "http://example.com") + buf = bytearray(2) + result = b"" + while True: + n = response.raw.readinto(buf) + if n == 0: + break + result += buf if n == 2 else buf[:n] + assert result == b"abcdefg" socket.socket = lambda *a, **k: Socket() @@ -338,7 +347,8 @@ def test_redirect_relative(): test_overwrite_post_chunked_data_headers() test_do_not_modify_headers_argument() test_content_length_via_content() -test_chunked_response_raises() +test_chunked_response_via_content() +test_chunked_response_readinto() test_raw_open_before_content() test_raw_incremental_content_length() test_raw_readinto_content_length() From 3c0d0a5d47eb657f2271eed474b6f95da33a5842 Mon Sep 17 00:00:00 2001 From: Pablo Ventura Date: Fri, 3 Jul 2026 21:49:40 -0300 Subject: [PATCH 21/74] requests: Document chunked response support. Note chunked response decoding in the feature list and drop the outdated limitation and follow-up entry. Signed-off-by: Pablo Ventura --- python-ecosys/requests/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python-ecosys/requests/README.md b/python-ecosys/requests/README.md index a4a5bd12e..7d888e240 100644 --- a/python-ecosys/requests/README.md +++ b/python-ecosys/requests/README.md @@ -5,7 +5,8 @@ This module provides a lightweight version of the Python It includes support for all HTTP verbs, https, json decoding of responses, redirects, basic authentication, HTTP/1.1 requests, and reading response -bodies with Content-Length via streaming ``.raw`` or lazy ``.content``. +bodies with Content-Length or Transfer-Encoding: chunked via streaming +``.raw`` or lazy ``.content``. ### Limitations @@ -14,11 +15,9 @@ bodies with Content-Length via streaming ``.raw`` or lazy ``.content``. multipart-form encoding of post data (this can be done manually). * Compressed requests/responses are not currently supported. * File upload is not supported. -* Chunked encoding in responses is not supported. * HTTP keep-alive connection reuse is not supported (Connection: close by default). ### Follow-up work -* Chunked response bodies. * TLS certificate verification (see micropython-lib issue #838). * ``stream=True`` incremental body reads (see issue #777). From a747e80e04656fce357145629de548a6978b337b Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Wed, 11 Jun 2025 14:28:00 +1000 Subject: [PATCH 22/74] python-ecosys/debugpy: Add VS Code debugging support for MicroPython. This implementation provides a Debug Adapter Protocol (DAP) server that enables VS Code to debug MicroPython code with breakpoint, stepping and variable inspection support. Features: - Manual breakpoints via debugpy.breakpoint() - Line breakpoints set from VS Code - Stack trace inspection - Variable scopes (locals/globals) - Source code viewing - Stepping (into/over/out) - Non-blocking architecture for MicroPython's single-threaded environment - Conditional debug logging based on VS Code's logToFile setting Implementation highlights: - Uses MicroPython's sys.settrace() for execution monitoring - Handles path mapping between VS Code and MicroPython - Efficient O(n) fibonacci demo (was O(2^n) recursive) - Compatible with MicroPython's limited frame object attributes Files: - debugpy/: Core debugging implementation - test_vscode.py: VS Code integration test - VSCODE_TESTING_GUIDE.md: Setup and usage instructions - dap_monitor.py: Protocol debugging utility Usage: import debugpy debugpy.listen() # Start debug server debugpy.debug_this_thread() # Enable tracing debugpy.breakpoint() # Manual breakpoint Signed-off-by: Andrew Leech --- python-ecosys/debugpy/README.md | 172 +++++++ python-ecosys/debugpy/dap_monitor.py | 162 +++++++ python-ecosys/debugpy/debugpy/__init__.py | 20 + .../debugpy/debugpy/common/__init__.py | 1 + .../debugpy/debugpy/common/constants.py | 60 +++ .../debugpy/debugpy/common/messaging.py | 154 +++++++ python-ecosys/debugpy/debugpy/public_api.py | 126 ++++++ .../debugpy/debugpy/server/__init__.py | 1 + .../debugpy/debugpy/server/debug_session.py | 423 ++++++++++++++++++ .../debugpy/debugpy/server/pdb_adapter.py | 285 ++++++++++++ python-ecosys/debugpy/demo.py | 68 +++ python-ecosys/debugpy/development_guide.md | 84 ++++ python-ecosys/debugpy/manifest.py | 6 + python-ecosys/debugpy/test_vscode.py | 72 +++ .../debugpy/vscode_launch_example.json | 22 + 15 files changed, 1656 insertions(+) create mode 100644 python-ecosys/debugpy/README.md create mode 100644 python-ecosys/debugpy/dap_monitor.py create mode 100644 python-ecosys/debugpy/debugpy/__init__.py create mode 100644 python-ecosys/debugpy/debugpy/common/__init__.py create mode 100644 python-ecosys/debugpy/debugpy/common/constants.py create mode 100644 python-ecosys/debugpy/debugpy/common/messaging.py create mode 100644 python-ecosys/debugpy/debugpy/public_api.py create mode 100644 python-ecosys/debugpy/debugpy/server/__init__.py create mode 100644 python-ecosys/debugpy/debugpy/server/debug_session.py create mode 100644 python-ecosys/debugpy/debugpy/server/pdb_adapter.py create mode 100644 python-ecosys/debugpy/demo.py create mode 100644 python-ecosys/debugpy/development_guide.md create mode 100644 python-ecosys/debugpy/manifest.py create mode 100644 python-ecosys/debugpy/test_vscode.py create mode 100644 python-ecosys/debugpy/vscode_launch_example.json diff --git a/python-ecosys/debugpy/README.md b/python-ecosys/debugpy/README.md new file mode 100644 index 000000000..70859b974 --- /dev/null +++ b/python-ecosys/debugpy/README.md @@ -0,0 +1,172 @@ +# MicroPython debugpy + +A minimal implementation of debugpy for MicroPython, enabling remote debugging +such as VS Code debugging support. + +## Features + +- Debug Adapter Protocol (DAP) support for VS Code integration +- Basic debugging operations: + - Breakpoints + - Step over/into/out + - Stack trace inspection + - Variable inspection (globals, locals generally not supported) + - Expression evaluation + - Pause/continue execution + +## Requirements + +- MicroPython with `sys.settrace` support (enabled with `MICROPY_PY_SYS_SETTRACE`) +- Socket support for network communication +- JSON support for DAP message parsing + +## Usage + +### Basic Usage + +```python +import debugpy + +# Start listening for debugger connections +host, port = debugpy.listen() # Default: 127.0.0.1:5678 +print(f"Debugger listening on {host}:{port}") + +# Enable debugging for current thread +debugpy.debug_this_thread() + +# Your code here... +def my_function(): + x = 10 + y = 20 + result = x + y # Set breakpoint here in VS Code + return result + +result = my_function() +print(f"Result: {result}") + +# Manual breakpoint +debugpy.breakpoint() +``` + +### VS Code Configuration + +Create a `.vscode/launch.json` file in your project: + +```json +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Attach to MicroPython", + "type": "python", + "request": "attach", + "connect": { + "host": "127.0.0.1", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "." + } + ], + "justMyCode": false + } + ] +} +``` + +### Testing + +1. Build the MicroPython Unix coverage port: + ```bash + cd ports/unix + make CFLAGS_EXTRA="-DMICROPY_PY_SYS_SETTRACE=1" + ``` + +2. Run the test script: + ```bash + cd lib/micropython-lib/python-ecosys/debugpy + ../../../../ports/unix/build-coverage/micropython test_debugpy.py + ``` + +3. In VS Code, open the debugpy folder and press F5 to attach the debugger + +4. Set breakpoints in the test script and observe debugging functionality + +## API Reference + +### `debugpy.listen(port=5678, host="127.0.0.1")` + +Start listening for debugger connections. + +**Parameters:** +- `port`: Port number to listen on (default: 5678) +- `host`: Host address to bind to (default: "127.0.0.1") + +**Returns:** Tuple of (host, port) actually used + +### `debugpy.debug_this_thread()` + +Enable debugging for the current thread by installing the trace function. + +### `debugpy.breakpoint()` + +Trigger a manual breakpoint that will pause execution if a debugger is attached. + +### `debugpy.wait_for_client()` + +Wait for the debugger client to connect and initialize. + +### `debugpy.is_client_connected()` + +Check if a debugger client is currently connected. + +**Returns:** Boolean indicating connection status + +### `debugpy.disconnect()` + +Disconnect from the debugger client and clean up resources. + +## Architecture + +The implementation consists of several key components: + +1. **Public API** (`public_api.py`): Main entry points for users +2. **Debug Session** (`server/debug_session.py`): Handles DAP protocol communication +3. **PDB Adapter** (`server/pdb_adapter.py`): Bridges DAP and MicroPython's trace system +4. **Messaging** (`common/messaging.py`): JSON message handling for DAP +5. **Constants** (`common/constants.py`): DAP protocol constants + +## Limitations + +This is a minimal implementation with the following limitations: + +- Single-threaded debugging only +- No conditional breakpoints +- No function breakpoints +- Limited variable inspection (no nested object expansion) +- No step back functionality +- No hot code reloading +- Simplified stepping implementation + +## Compatibility + +Tested with: +- MicroPython Unix port +- VS Code with Python/debugpy extension +- CPython 3.x (for comparison) + +## Contributing + +This implementation provides a foundation for MicroPython debugging. Contributions are welcome to add: + +- Conditional breakpoint support +- Better variable inspection +- Multi-threading support +- Performance optimizations +- Additional DAP features + +## License + +MIT License - see the MicroPython project license for details. diff --git a/python-ecosys/debugpy/dap_monitor.py b/python-ecosys/debugpy/dap_monitor.py new file mode 100644 index 000000000..3af4eba16 --- /dev/null +++ b/python-ecosys/debugpy/dap_monitor.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""DAP protocol monitor - sits between VS Code and MicroPython debugpy.""" + +import socket +import threading +import json +import time +import sys + +class DAPMonitor: + def __init__(self, listen_port=5679, target_host='127.0.0.1', target_port=5678): + self.listen_port = listen_port + self.target_host = target_host + self.target_port = target_port + self.client_sock = None + self.server_sock = None + + def start(self): + """Start the DAP monitor proxy.""" + print(f"DAP Monitor starting on port {self.listen_port}") + print(f"Will forward to {self.target_host}:{self.target_port}") + print("Start MicroPython debugpy server first, then connect VS Code to port 5679") + + # Create listening socket + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(('127.0.0.1', self.listen_port)) + listener.listen(1) + + print(f"Listening for VS Code connection on port {self.listen_port}...") + + try: + # Wait for VS Code to connect + self.client_sock, client_addr = listener.accept() + print(f"VS Code connected from {client_addr}") + + # Connect to MicroPython debugpy server + self.server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.server_sock.connect((self.target_host, self.target_port)) + print(f"Connected to MicroPython debugpy at {self.target_host}:{self.target_port}") + + # Start forwarding threads + threading.Thread(target=self.forward_client_to_server, daemon=True).start() + threading.Thread(target=self.forward_server_to_client, daemon=True).start() + + print("DAP Monitor active - press Ctrl+C to stop") + while True: + time.sleep(1) + + except KeyboardInterrupt: + print("\nStopping DAP Monitor...") + except Exception as e: + print(f"Error: {e}") + finally: + self.cleanup() + + def forward_client_to_server(self): + """Forward messages from VS Code client to MicroPython server.""" + try: + while True: + data = self.receive_dap_message(self.client_sock, "VS Code") + if data is None: + break + self.send_raw_data(self.server_sock, data) + except Exception as e: + print(f"Client->Server forwarding error: {e}") + + def forward_server_to_client(self): + """Forward messages from MicroPython server to VS Code client.""" + try: + while True: + data = self.receive_dap_message(self.server_sock, "MicroPython") + if data is None: + break + self.send_raw_data(self.client_sock, data) + except Exception as e: + print(f"Server->Client forwarding error: {e}") + + def receive_dap_message(self, sock, source): + """Receive and log a DAP message.""" + try: + # Read headers + header = b"" + while b"\r\n\r\n" not in header: + byte = sock.recv(1) + if not byte: + return None + header += byte + + # Parse content length + header_str = header.decode('utf-8') + content_length = 0 + for line in header_str.split('\r\n'): + if line.startswith('Content-Length:'): + content_length = int(line.split(':', 1)[1].strip()) + break + + if content_length == 0: + return None + + # Read content + content = b"" + while len(content) < content_length: + chunk = sock.recv(content_length - len(content)) + if not chunk: + return None + content += chunk + + # Log the message + try: + message = json.loads(content.decode('utf-8')) + msg_type = message.get('type', 'unknown') + command = message.get('command', message.get('event', 'unknown')) + seq = message.get('seq', 0) + + print(f"\n[{source}] {msg_type.upper()}: {command} (seq={seq})") + + if msg_type == 'request': + args = message.get('arguments', {}) + if args: + print(f" Arguments: {json.dumps(args, indent=2)}") + elif msg_type == 'response': + success = message.get('success', False) + req_seq = message.get('request_seq', 0) + print(f" Success: {success}, Request Seq: {req_seq}") + body = message.get('body') + if body: + print(f" Body: {json.dumps(body, indent=2)}") + msg = message.get('message') + if msg: + print(f" Message: {msg}") + elif msg_type == 'event': + body = message.get('body', {}) + if body: + print(f" Body: {json.dumps(body, indent=2)}") + + except json.JSONDecodeError: + print(f"\n[{source}] Invalid JSON: {content}") + + return header + content + + except Exception as e: + print(f"Error receiving from {source}: {e}") + return None + + def send_raw_data(self, sock, data): + """Send raw data to socket.""" + try: + sock.send(data) + except Exception as e: + print(f"Error sending data: {e}") + + def cleanup(self): + """Clean up sockets.""" + if self.client_sock: + self.client_sock.close() + if self.server_sock: + self.server_sock.close() + +if __name__ == "__main__": + monitor = DAPMonitor() + monitor.start() \ No newline at end of file diff --git a/python-ecosys/debugpy/debugpy/__init__.py b/python-ecosys/debugpy/debugpy/__init__.py new file mode 100644 index 000000000..b7649bd5c --- /dev/null +++ b/python-ecosys/debugpy/debugpy/__init__.py @@ -0,0 +1,20 @@ +"""MicroPython debugpy implementation. + +A minimal port of debugpy for MicroPython to enable VS Code debugging support. +This implementation focuses on the core DAP (Debug Adapter Protocol) functionality +needed for basic debugging operations like breakpoints, stepping, and variable inspection. +""" + +__version__ = "0.1.0" + +from .public_api import listen, wait_for_client, breakpoint, debug_this_thread +from .common.constants import DEFAULT_HOST, DEFAULT_PORT + +__all__ = [ + "listen", + "wait_for_client", + "breakpoint", + "debug_this_thread", + "DEFAULT_HOST", + "DEFAULT_PORT", +] diff --git a/python-ecosys/debugpy/debugpy/common/__init__.py b/python-ecosys/debugpy/debugpy/common/__init__.py new file mode 100644 index 000000000..c53632010 --- /dev/null +++ b/python-ecosys/debugpy/debugpy/common/__init__.py @@ -0,0 +1 @@ +# Common utilities and constants for debugpy diff --git a/python-ecosys/debugpy/debugpy/common/constants.py b/python-ecosys/debugpy/debugpy/common/constants.py new file mode 100644 index 000000000..aeee675e3 --- /dev/null +++ b/python-ecosys/debugpy/debugpy/common/constants.py @@ -0,0 +1,60 @@ +"""Constants used throughout debugpy.""" + +# Default networking settings +DEFAULT_HOST = "127.0.0.1" +DEFAULT_PORT = 5678 + +# DAP message types +MSG_TYPE_REQUEST = "request" +MSG_TYPE_RESPONSE = "response" +MSG_TYPE_EVENT = "event" + +# DAP events +EVENT_INITIALIZED = "initialized" +EVENT_STOPPED = "stopped" +EVENT_CONTINUED = "continued" +EVENT_THREAD = "thread" +EVENT_BREAKPOINT = "breakpoint" +EVENT_OUTPUT = "output" +EVENT_TERMINATED = "terminated" +EVENT_EXITED = "exited" + +# DAP commands +CMD_INITIALIZE = "initialize" +CMD_LAUNCH = "launch" +CMD_ATTACH = "attach" +CMD_SET_BREAKPOINTS = "setBreakpoints" +CMD_CONTINUE = "continue" +CMD_NEXT = "next" +CMD_STEP_IN = "stepIn" +CMD_STEP_OUT = "stepOut" +CMD_PAUSE = "pause" +CMD_STACK_TRACE = "stackTrace" +CMD_SCOPES = "scopes" +CMD_VARIABLES = "variables" +CMD_EVALUATE = "evaluate" +CMD_DISCONNECT = "disconnect" +CMD_CONFIGURATION_DONE = "configurationDone" +CMD_THREADS = "threads" +CMD_SOURCE = "source" + +# Stop reasons +STOP_REASON_STEP = "step" +STOP_REASON_BREAKPOINT = "breakpoint" +STOP_REASON_EXCEPTION = "exception" +STOP_REASON_PAUSE = "pause" +STOP_REASON_ENTRY = "entry" + +# Thread reasons +THREAD_REASON_STARTED = "started" +THREAD_REASON_EXITED = "exited" + +# Trace events +TRACE_CALL = "call" +TRACE_LINE = "line" +TRACE_RETURN = "return" +TRACE_EXCEPTION = "exception" + +# Scope types +SCOPE_LOCALS = "locals" +SCOPE_GLOBALS = "globals" diff --git a/python-ecosys/debugpy/debugpy/common/messaging.py b/python-ecosys/debugpy/debugpy/common/messaging.py new file mode 100644 index 000000000..bc264e3ff --- /dev/null +++ b/python-ecosys/debugpy/debugpy/common/messaging.py @@ -0,0 +1,154 @@ +"""JSON message handling for DAP protocol.""" + +import json +from .constants import MSG_TYPE_REQUEST, MSG_TYPE_RESPONSE, MSG_TYPE_EVENT + + +class JsonMessageChannel: + """Handles JSON message communication over a socket using DAP format.""" + + def __init__(self, sock, debug_callback=None): + self.sock = sock + self.seq = 0 + self.closed = False + self._recv_buffer = b"" + self._debug_print = debug_callback or (lambda x: None) # Default to no-op + + def send_message(self, msg_type, command=None, **kwargs): + """Send a DAP message.""" + if self.closed: + return + + self.seq += 1 + message = { + "seq": self.seq, + "type": msg_type, + } + + if command: + if msg_type == MSG_TYPE_REQUEST: + message["command"] = command + if kwargs: + message["arguments"] = kwargs + elif msg_type == MSG_TYPE_RESPONSE: + message["command"] = command + message["request_seq"] = kwargs.get("request_seq", 0) + message["success"] = kwargs.get("success", True) + if "body" in kwargs: + message["body"] = kwargs["body"] + if "message" in kwargs: + message["message"] = kwargs["message"] + elif msg_type == MSG_TYPE_EVENT: + message["event"] = command + if kwargs: + message["body"] = kwargs + + json_str = json.dumps(message) + content = json_str.encode("utf-8") + header = f"Content-Length: {len(content)}\r\n\r\n".encode("utf-8") + + try: + self.sock.send(header + content) + except OSError: + self.closed = True + + def send_request(self, command, **kwargs): + """Send a request message.""" + self.send_message(MSG_TYPE_REQUEST, command, **kwargs) + + def send_response(self, command, request_seq, success=True, body=None, message=None): + """Send a response message.""" + kwargs = {"request_seq": request_seq, "success": success} + if body is not None: + kwargs["body"] = body + if message is not None: + kwargs["message"] = message + + self._debug_print(f"[DAP] SEND: response {command} (req_seq={request_seq}, success={success})") + if body: + self._debug_print(f"[DAP] body: {body}") + if message: + self._debug_print(f"[DAP] message: {message}") + + self.send_message(MSG_TYPE_RESPONSE, command, **kwargs) + + def send_event(self, event, **kwargs): + """Send an event message.""" + self._debug_print(f"[DAP] SEND: event {event}") + if kwargs: + self._debug_print(f"[DAP] body: {kwargs}") + self.send_message(MSG_TYPE_EVENT, event, **kwargs) + + def recv_message(self): + """Receive a DAP message.""" + if self.closed: + return None + + try: + # Read headers + while b"\r\n\r\n" not in self._recv_buffer: + try: + data = self.sock.recv(1024) + if not data: + self.closed = True + return None + self._recv_buffer += data + except OSError as e: + # Handle timeout and other socket errors + if hasattr(e, 'errno') and e.errno in (11, 35): # EAGAIN, EWOULDBLOCK + return None # No data available + self.closed = True + return None + + header_end = self._recv_buffer.find(b"\r\n\r\n") + header_str = self._recv_buffer[:header_end].decode("utf-8") + self._recv_buffer = self._recv_buffer[header_end + 4:] + + # Parse Content-Length + content_length = 0 + for line in header_str.split("\r\n"): + if line.startswith("Content-Length:"): + content_length = int(line.split(":", 1)[1].strip()) + break + + if content_length == 0: + return None + + # Read body + while len(self._recv_buffer) < content_length: + try: + data = self.sock.recv(content_length - len(self._recv_buffer)) + if not data: + self.closed = True + return None + self._recv_buffer += data + except OSError as e: + if hasattr(e, 'errno') and e.errno in (11, 35): # EAGAIN, EWOULDBLOCK + return None + self.closed = True + return None + + body = self._recv_buffer[:content_length] + self._recv_buffer = self._recv_buffer[content_length:] + + # Parse JSON + try: + message = json.loads(body.decode("utf-8")) + self._debug_print(f"[DAP] Successfully received message: {message.get('type')} {message.get('command', message.get('event', 'unknown'))}") + return message + except (ValueError, UnicodeDecodeError) as e: + print(f"[DAP] JSON parse error: {e}") + return None + + except OSError as e: + print(f"[DAP] Socket error in recv_message: {e}") + self.closed = True + return None + + def close(self): + """Close the channel.""" + self.closed = True + try: + self.sock.close() + except OSError: + pass diff --git a/python-ecosys/debugpy/debugpy/public_api.py b/python-ecosys/debugpy/debugpy/public_api.py new file mode 100644 index 000000000..137706efe --- /dev/null +++ b/python-ecosys/debugpy/debugpy/public_api.py @@ -0,0 +1,126 @@ +"""Public API for debugpy.""" + +import socket +import sys +from .common.constants import DEFAULT_HOST, DEFAULT_PORT +from .server.debug_session import DebugSession + +_debug_session = None + + +def listen(port=DEFAULT_PORT, host=DEFAULT_HOST): + """Start listening for debugger connections. + + Args: + port: Port number to listen on (default: 5678) + host: Host address to bind to (default: "127.0.0.1") + + Returns: + (host, port) tuple of the actual listening address + """ + global _debug_session + + if _debug_session is not None: + raise RuntimeError("Already listening for debugger") + + # Create listening socket + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + except: + pass # Not supported in MicroPython + + # Use getaddrinfo for MicroPython compatibility + addr_info = socket.getaddrinfo(host, port) + addr = addr_info[0][-1] # Get the sockaddr + listener.bind(addr) + listener.listen(1) + + # getsockname not available in MicroPython, use original values + print(f"Debugpy listening on {host}:{port}") + + # Wait for connection + client_sock = None + try: + client_sock, client_addr = listener.accept() + print(f"Debugger connected from {client_addr}") + + # Create debug session + _debug_session = DebugSession(client_sock) + + # Handle just the initialize request, then return immediately + print("[DAP] Waiting for initialize request...") + init_message = _debug_session.channel.recv_message() + if init_message and init_message.get('command') == 'initialize': + _debug_session._handle_message(init_message) + print("[DAP] Initialize request handled - returning control immediately") + else: + print(f"[DAP] Warning: Expected initialize, got {init_message}") + + # Set socket to non-blocking for subsequent message processing + _debug_session.channel.sock.settimeout(0.001) + + print("[DAP] Debug session ready - all other messages will be handled in trace function") + + except Exception as e: + print(f"[DAP] Connection error: {e}") + if client_sock: + client_sock.close() + _debug_session = None + finally: + # Only close the listener, not the client connection + listener.close() + + return (host, port) + + +def wait_for_client(): + """Wait for the debugger client to connect and initialize.""" + global _debug_session + if _debug_session: + _debug_session.wait_for_client() + + +def breakpoint(): + """Trigger a breakpoint in the debugger.""" + global _debug_session + if _debug_session: + _debug_session.trigger_breakpoint() + else: + # Fallback to built-in breakpoint if available + if hasattr(__builtins__, 'breakpoint'): + __builtins__.breakpoint() + + +def debug_this_thread(): + """Enable debugging for the current thread.""" + global _debug_session + if _debug_session: + _debug_session.debug_this_thread() + else: + # Install trace function even if no session yet + if hasattr(sys, 'settrace'): + sys.settrace(_default_trace_func) + else: + raise RuntimeError("MICROPY_PY_SYS_SETTRACE required") + + +def _default_trace_func(frame, event, arg): + """Default trace function when no debug session is active.""" + # Just return None to continue execution + return None + + + +def is_client_connected(): + """Check if a debugger client is connected.""" + global _debug_session + return _debug_session is not None and _debug_session.is_connected() + + +def disconnect(): + """Disconnect from the debugger client.""" + global _debug_session + if _debug_session: + _debug_session.disconnect() + _debug_session = None diff --git a/python-ecosys/debugpy/debugpy/server/__init__.py b/python-ecosys/debugpy/debugpy/server/__init__.py new file mode 100644 index 000000000..1ab7a0ff5 --- /dev/null +++ b/python-ecosys/debugpy/debugpy/server/__init__.py @@ -0,0 +1 @@ +# Debug server components diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py new file mode 100644 index 000000000..4f60ee358 --- /dev/null +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -0,0 +1,423 @@ +"""Main debug session handling DAP protocol communication.""" + +import sys +from ..common.messaging import JsonMessageChannel +from ..common.constants import ( + CMD_INITIALIZE, CMD_LAUNCH, CMD_ATTACH, CMD_SET_BREAKPOINTS, + CMD_CONTINUE, CMD_NEXT, CMD_STEP_IN, CMD_STEP_OUT, CMD_PAUSE, + CMD_STACK_TRACE, CMD_SCOPES, CMD_VARIABLES, CMD_EVALUATE, CMD_DISCONNECT, + CMD_CONFIGURATION_DONE, CMD_THREADS, CMD_SOURCE, EVENT_INITIALIZED, EVENT_STOPPED, EVENT_CONTINUED, EVENT_TERMINATED, + STOP_REASON_BREAKPOINT, STOP_REASON_STEP, STOP_REASON_PAUSE, + TRACE_CALL, TRACE_LINE, TRACE_RETURN, TRACE_EXCEPTION +) +from .pdb_adapter import PdbAdapter + + +class DebugSession: + """Manages a debugging session with a DAP client.""" + + def __init__(self, client_socket): + self.debug_logging = False # Initialize first + self.channel = JsonMessageChannel(client_socket, self._debug_print) + self.pdb = PdbAdapter() + self.pdb._debug_session = self # Allow PDB to process messages during wait + self.initialized = False + self.connected = True + self.thread_id = 1 # Simple single-thread model + self.stepping = False + self.paused = False + + def _debug_print(self, message): + """Print debug message only if debug logging is enabled.""" + if self.debug_logging: + print(message) + + def start(self): + """Start the debug session message loop.""" + try: + while self.connected and not self.channel.closed: + message = self.channel.recv_message() + if message is None: + break + + self._handle_message(message) + + except Exception as e: + print(f"Debug session error: {e}") + finally: + self.disconnect() + + def initialize_connection(self): + """Initialize the connection - handle just the essential initial messages then return.""" + # Note: debug_logging not available yet during init, so we always show these messages + print("[DAP] Processing initial DAP messages...") + + try: + # Process initial messages quickly and return control to main thread + # We'll handle ongoing messages in the trace function + attached = False + message_count = 0 + max_init_messages = 6 # Just handle the first few essential messages + + while message_count < max_init_messages and not attached: + try: + # Short timeout - don't block the main thread for long + self.channel.sock.settimeout(1.0) + message = self.channel.recv_message() + if message is None: + print(f"[DAP] No more messages in initial batch") + break + + print(f"[DAP] Initial message #{message_count + 1}: {message.get('command')}") + self._handle_message(message) + message_count += 1 + + # Just wait for attach, then we can return control + if message.get('command') == 'attach': + attached = True + print("[DAP] ✅ Attach received - returning control to main thread") + break + + except Exception as e: + print(f"[DAP] Exception in initial processing: {e}") + break + finally: + self.channel.sock.settimeout(None) + + # After attach, continue processing a few more messages quickly + if attached: + self._debug_print("[DAP] Processing remaining setup messages...") + additional_count = 0 + while additional_count < 4: # Just a few more + try: + self.channel.sock.settimeout(0.5) # Short timeout + message = self.channel.recv_message() + if message is None: + break + self._debug_print(f"[DAP] Setup message: {message.get('command')}") + self._handle_message(message) + additional_count += 1 + except: + break + finally: + self.channel.sock.settimeout(None) + + print(f"[DAP] Initial setup complete - main thread can continue") + + except Exception as e: + print(f"[DAP] Initialization error: {e}") + + def process_pending_messages(self): + """Process any pending DAP messages without blocking.""" + try: + # Set socket to non-blocking mode for message processing + self.channel.sock.settimeout(0.001) # Very short timeout + + while True: + message = self.channel.recv_message() + if message is None: + break + self._handle_message(message) + + except Exception: + # No messages available or socket error + pass + finally: + # Reset to blocking mode + self.channel.sock.settimeout(None) + + def _handle_message(self, message): + """Handle incoming DAP messages.""" + msg_type = message.get("type") + command = message.get("command", message.get("event", "unknown")) + seq = message.get("seq", 0) + + self._debug_print(f"[DAP] RECV: {msg_type} {command} (seq={seq})") + if message.get("arguments"): + self._debug_print(f"[DAP] args: {message['arguments']}") + + if msg_type == "request": + self._handle_request(message) + elif msg_type == "response": + # We don't expect responses from client + self._debug_print(f"[DAP] Unexpected response from client: {message}") + elif msg_type == "event": + # We don't expect events from client + self._debug_print(f"[DAP] Unexpected event from client: {message}") + + def _handle_request(self, message): + """Handle DAP request messages.""" + command = message.get("command") + seq = message.get("seq", 0) + args = message.get("arguments", {}) + + try: + if command == CMD_INITIALIZE: + self._handle_initialize(seq, args) + elif command == CMD_LAUNCH: + self._handle_launch(seq, args) + elif command == CMD_ATTACH: + self._handle_attach(seq, args) + elif command == CMD_SET_BREAKPOINTS: + self._handle_set_breakpoints(seq, args) + elif command == CMD_CONTINUE: + self._handle_continue(seq, args) + elif command == CMD_NEXT: + self._handle_next(seq, args) + elif command == CMD_STEP_IN: + self._handle_step_in(seq, args) + elif command == CMD_STEP_OUT: + self._handle_step_out(seq, args) + elif command == CMD_PAUSE: + self._handle_pause(seq, args) + elif command == CMD_STACK_TRACE: + self._handle_stack_trace(seq, args) + elif command == CMD_SCOPES: + self._handle_scopes(seq, args) + elif command == CMD_VARIABLES: + self._handle_variables(seq, args) + elif command == CMD_EVALUATE: + self._handle_evaluate(seq, args) + elif command == CMD_DISCONNECT: + self._handle_disconnect(seq, args) + elif command == CMD_CONFIGURATION_DONE: + self._handle_configuration_done(seq, args) + elif command == CMD_THREADS: + self._handle_threads(seq, args) + elif command == CMD_SOURCE: + self._handle_source(seq, args) + else: + self.channel.send_response(command, seq, success=False, + message=f"Unknown command: {command}") + + except Exception as e: + self.channel.send_response(command, seq, success=False, + message=str(e)) + + def _handle_initialize(self, seq, args): + """Handle initialize request.""" + capabilities = { + "supportsConfigurationDoneRequest": True, + "supportsFunctionBreakpoints": False, + "supportsConditionalBreakpoints": False, + "supportsHitConditionalBreakpoints": False, + "supportsEvaluateForHovers": True, + "supportsStepBack": False, + "supportsSetVariable": False, + "supportsRestartFrame": False, + "supportsGotoTargetsRequest": False, + "supportsStepInTargetsRequest": False, + "supportsCompletionsRequest": False, + "supportsModulesRequest": False, + "additionalModuleColumns": [], + "supportedChecksumAlgorithms": [], + "supportsRestartRequest": False, + "supportsExceptionOptions": False, + "supportsValueFormattingOptions": False, + "supportsExceptionInfoRequest": False, + "supportTerminateDebuggee": True, + "supportSuspendDebuggee": True, + "supportsDelayedStackTraceLoading": False, + "supportsLoadedSourcesRequest": False, + "supportsLogPoints": False, + "supportsTerminateThreadsRequest": False, + "supportsSetExpression": False, + "supportsTerminateRequest": True, + "supportsDataBreakpoints": False, + "supportsReadMemoryRequest": False, + "supportsWriteMemoryRequest": False, + "supportsDisassembleRequest": False, + "supportsCancelRequest": False, + "supportsBreakpointLocationsRequest": False, + "supportsClipboardContext": False, + } + + self.channel.send_response(CMD_INITIALIZE, seq, body=capabilities) + self.channel.send_event(EVENT_INITIALIZED) + self.initialized = True + + def _handle_launch(self, seq, args): + """Handle launch request.""" + # For attach-mode debugging, we don't need to launch anything + self.channel.send_response(CMD_LAUNCH, seq) + + def _handle_attach(self, seq, args): + """Handle attach request.""" + # Check if debug logging should be enabled + self.debug_logging = args.get("logToFile", False) + + self._debug_print(f"[DAP] Processing attach request with args: {args}") + print(f"[DAP] Debug logging {'enabled' if self.debug_logging else 'disabled'} (logToFile={self.debug_logging})") + + # Enable trace function + self.pdb.set_trace_function(self._trace_function) + self.channel.send_response(CMD_ATTACH, seq) + + # After successful attach, we might need to send additional events + # Some debuggers expect a 'process' event or thread events + self._debug_print("[DAP] Attach completed, debugging is now active") + + def _handle_set_breakpoints(self, seq, args): + """Handle setBreakpoints request.""" + source = args.get("source", {}) + filename = source.get("path", "") + breakpoints = args.get("breakpoints", []) + + # Set breakpoints in pdb adapter + actual_breakpoints = self.pdb.set_breakpoints(filename, breakpoints) + + self.channel.send_response(CMD_SET_BREAKPOINTS, seq, + body={"breakpoints": actual_breakpoints}) + + def _handle_continue(self, seq, args): + """Handle continue request.""" + self.stepping = False + self.paused = False + self.pdb.continue_execution() + self.channel.send_response(CMD_CONTINUE, seq) + + def _handle_next(self, seq, args): + """Handle next (step over) request.""" + self.stepping = True + self.paused = False + self.pdb.step_over() + self.channel.send_response(CMD_NEXT, seq) + + def _handle_step_in(self, seq, args): + """Handle stepIn request.""" + self.stepping = True + self.paused = False + self.pdb.step_into() + self.channel.send_response(CMD_STEP_IN, seq) + + def _handle_step_out(self, seq, args): + """Handle stepOut request.""" + self.stepping = True + self.paused = False + self.pdb.step_out() + self.channel.send_response(CMD_STEP_OUT, seq) + + def _handle_pause(self, seq, args): + """Handle pause request.""" + self.paused = True + self.pdb.pause() + self.channel.send_response(CMD_PAUSE, seq) + + def _handle_stack_trace(self, seq, args): + """Handle stackTrace request.""" + stack_frames = self.pdb.get_stack_trace() + self.channel.send_response(CMD_STACK_TRACE, seq, + body={"stackFrames": stack_frames, "totalFrames": len(stack_frames)}) + + def _handle_scopes(self, seq, args): + """Handle scopes request.""" + frame_id = args.get("frameId", 0) + self._debug_print(f"[DAP] Processing scopes request for frameId={frame_id}") + scopes = self.pdb.get_scopes(frame_id) + self._debug_print(f"[DAP] Generated scopes: {scopes}") + self.channel.send_response(CMD_SCOPES, seq, body={"scopes": scopes}) + + def _handle_variables(self, seq, args): + """Handle variables request.""" + variables_ref = args.get("variablesReference", 0) + variables = self.pdb.get_variables(variables_ref) + self.channel.send_response(CMD_VARIABLES, seq, body={"variables": variables}) + + def _handle_evaluate(self, seq, args): + """Handle evaluate request.""" + expression = args.get("expression", "") + frame_id = args.get("frameId") + context = args.get("context", "watch") + + try: + result = self.pdb.evaluate_expression(expression, frame_id) + self.channel.send_response(CMD_EVALUATE, seq, body={ + "result": str(result), + "variablesReference": 0 + }) + except Exception as e: + self.channel.send_response(CMD_EVALUATE, seq, success=False, + message=str(e)) + + def _handle_disconnect(self, seq, args): + """Handle disconnect request.""" + self.channel.send_response(CMD_DISCONNECT, seq) + self.disconnect() + + def _handle_configuration_done(self, seq, args): + """Handle configurationDone request.""" + # This indicates that the client has finished configuring breakpoints + # and is ready to start debugging + self.channel.send_response(CMD_CONFIGURATION_DONE, seq) + + def _handle_threads(self, seq, args): + """Handle threads request.""" + # MicroPython is single-threaded, so return one thread + threads = [{ + "id": self.thread_id, + "name": "main" + }] + self.channel.send_response(CMD_THREADS, seq, body={"threads": threads}) + + def _handle_source(self, seq, args): + """Handle source request.""" + source = args.get("source", {}) + source_path = source.get("path", "") + + try: + # Try to read the source file + with open(source_path, 'r') as f: + content = f.read() + self.channel.send_response(CMD_SOURCE, seq, body={"content": content}) + except Exception as e: + self.channel.send_response(CMD_SOURCE, seq, success=False, + message=f"Could not read source: {e}") + + def _trace_function(self, frame, event, arg): + """Trace function called by sys.settrace.""" + # Process any pending DAP messages frequently + self.process_pending_messages() + + # Handle breakpoints and stepping + if self.pdb.should_stop(frame, event, arg): + self._send_stopped_event(STOP_REASON_BREAKPOINT if self.pdb.hit_breakpoint else + STOP_REASON_STEP if self.stepping else STOP_REASON_PAUSE) + # Wait for continue command + self.pdb.wait_for_continue() + + return self._trace_function + + def _send_stopped_event(self, reason): + """Send stopped event to client.""" + self.channel.send_event(EVENT_STOPPED, + reason=reason, + threadId=self.thread_id, + allThreadsStopped=True) + + def wait_for_client(self): + """Wait for client to initialize.""" + # This is a simplified version - in a real implementation + # we might want to wait for specific initialization steps + pass + + def trigger_breakpoint(self): + """Trigger a manual breakpoint.""" + if self.initialized: + self._send_stopped_event(STOP_REASON_BREAKPOINT) + + def debug_this_thread(self): + """Enable debugging for current thread.""" + if hasattr(sys, 'settrace'): + sys.settrace(self._trace_function) + + def is_connected(self): + """Check if client is connected.""" + return self.connected and not self.channel.closed + + def disconnect(self): + """Disconnect from client.""" + self.connected = False + if hasattr(sys, 'settrace'): + sys.settrace(None) + self.pdb.cleanup() + self.channel.close() diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py new file mode 100644 index 000000000..83693c65c --- /dev/null +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -0,0 +1,285 @@ +"""PDB adapter for integrating with MicroPython's trace system.""" + +import sys +import time +from ..common.constants import ( + TRACE_CALL, TRACE_LINE, TRACE_RETURN, TRACE_EXCEPTION, + SCOPE_LOCALS, SCOPE_GLOBALS +) + + +class PdbAdapter: + """Adapter between DAP protocol and MicroPython's sys.settrace functionality.""" + + def __init__(self): + self.breakpoints = {} # filename -> {line_no: breakpoint_info} + self.current_frame = None + self.step_mode = None # None, 'over', 'into', 'out' + self.step_frame = None + self.step_depth = 0 + self.hit_breakpoint = False + self.continue_event = False + self.variables_cache = {} # frameId -> variables + self.frame_id_counter = 1 + + def _debug_print(self, message): + """Print debug message only if debug logging is enabled.""" + if hasattr(self, '_debug_session') and self._debug_session.debug_logging: + print(message) + + def set_trace_function(self, trace_func): + """Install the trace function.""" + if hasattr(sys, 'settrace'): + sys.settrace(trace_func) + else: + raise RuntimeError("sys.settrace not available") + + def set_breakpoints(self, filename, breakpoints): + """Set breakpoints for a file.""" + self.breakpoints[filename] = {} + actual_breakpoints = [] + + for bp in breakpoints: + line = bp.get("line") + if line: + self.breakpoints[filename][line] = { + "line": line, + "verified": True, + "source": {"path": filename} + } + actual_breakpoints.append({ + "line": line, + "verified": True, + "source": {"path": filename} + }) + + return actual_breakpoints + + def should_stop(self, frame, event, arg): + """Determine if execution should stop at this point.""" + self.current_frame = frame + self.hit_breakpoint = False + + # Get frame information + filename = frame.f_code.co_filename + lineno = frame.f_lineno + + # Debug: print filename and line for debugging + if event == TRACE_LINE and lineno in [20, 21, 22, 23, 24]: # Only log lines near our breakpoints + self._debug_print(f"[PDB] Checking {filename}:{lineno} (event={event})") + self._debug_print(f"[PDB] Available breakpoint files: {list(self.breakpoints.keys())}") + + # Check for exact filename match first + if filename in self.breakpoints: + if lineno in self.breakpoints[filename]: + self._debug_print(f"[PDB] HIT BREAKPOINT (exact match) at {filename}:{lineno}") + self.hit_breakpoint = True + return True + + # Also try checking by basename for path mismatches + def basename(path): + return path.split('/')[-1] if '/' in path else path + + file_basename = basename(filename) + self._debug_print(f"[PDB] Fallback basename match: '{file_basename}' vs available files") + for bp_file in self.breakpoints: + bp_basename = basename(bp_file) + self._debug_print(f"[PDB] Comparing '{file_basename}' == '{bp_basename}' ?") + if bp_basename == file_basename: + self._debug_print(f"[PDB] Basename match found! Checking line {lineno} in {list(self.breakpoints[bp_file].keys())}") + if lineno in self.breakpoints[bp_file]: + self._debug_print(f"[PDB] HIT BREAKPOINT (fallback basename match) at {filename}:{lineno} -> {bp_file}") + self.hit_breakpoint = True + return True + + # Check stepping + if self.step_mode == 'into': + if event in (TRACE_CALL, TRACE_LINE): + self.step_mode = None + return True + + elif self.step_mode == 'over': + if event == TRACE_LINE and frame == self.step_frame: + self.step_mode = None + return True + elif event == TRACE_RETURN and frame == self.step_frame: + # Continue stepping in caller + if hasattr(frame, 'f_back') and frame.f_back: + self.step_frame = frame.f_back + else: + self.step_mode = None + + elif self.step_mode == 'out': + if event == TRACE_RETURN and frame == self.step_frame: + self.step_mode = None + return True + + return False + + def continue_execution(self): + """Continue execution.""" + self.step_mode = None + self.continue_event = True + + def step_over(self): + """Step over (next line).""" + self.step_mode = 'over' + self.step_frame = self.current_frame + self.continue_event = True + + def step_into(self): + """Step into function calls.""" + self.step_mode = 'into' + self.continue_event = True + + def step_out(self): + """Step out of current function.""" + self.step_mode = 'out' + self.step_frame = self.current_frame + self.continue_event = True + + def pause(self): + """Pause execution at next opportunity.""" + # This is handled by the debug session + pass + + def wait_for_continue(self): + """Wait for continue command (simplified implementation).""" + # In a real implementation, this would block until continue + # For MicroPython, we'll use a simple polling approach + self.continue_event = False + + # Process DAP messages while waiting for continue + self._debug_print("[PDB] Waiting for continue command...") + while not self.continue_event: + # Process any pending DAP messages (scopes, variables, etc.) + if hasattr(self, '_debug_session'): + self._debug_session.process_pending_messages() + time.sleep(0.01) + + def get_stack_trace(self): + """Get the current stack trace.""" + if not self.current_frame: + return [] + + frames = [] + frame = self.current_frame + frame_id = 0 + + while frame: + filename = frame.f_code.co_filename + name = frame.f_code.co_name + line = frame.f_lineno + + # Create frame info + frames.append({ + "id": frame_id, + "name": name, + "source": {"path": filename}, + "line": line, + "column": 1, + "endLine": line, + "endColumn": 1 + }) + + # Cache frame for variable access + self.variables_cache[frame_id] = frame + + # MicroPython doesn't have f_back attribute + if hasattr(frame, 'f_back'): + frame = frame.f_back + else: + # Only return the current frame for MicroPython + break + frame_id += 1 + + return frames + + def get_scopes(self, frame_id): + """Get variable scopes for a frame.""" + scopes = [ + { + "name": "Locals", + "variablesReference": frame_id * 1000 + 1, + "expensive": False + }, + { + "name": "Globals", + "variablesReference": frame_id * 1000 + 2, + "expensive": False + } + ] + return scopes + + def get_variables(self, variables_ref): + """Get variables for a scope.""" + frame_id = variables_ref // 1000 + scope_type = variables_ref % 1000 + + if frame_id not in self.variables_cache: + return [] + + frame = self.variables_cache[frame_id] + variables = [] + + if scope_type == 1: # Locals + var_dict = frame.f_locals if hasattr(frame, 'f_locals') else {} + elif scope_type == 2: # Globals + var_dict = frame.f_globals if hasattr(frame, 'f_globals') else {} + else: + return [] + + for name, value in var_dict.items(): + # Skip private/internal variables + if name.startswith('__') and name.endswith('__'): + continue + + try: + value_str = str(value) + type_str = type(value).__name__ + + variables.append({ + "name": name, + "value": value_str, + "type": type_str, + "variablesReference": 0 # Simple implementation - no nested objects + }) + except Exception: + variables.append({ + "name": name, + "value": "", + "type": "unknown", + "variablesReference": 0 + }) + + return variables + + def evaluate_expression(self, expression, frame_id=None): + """Evaluate an expression in the context of a frame.""" + if frame_id is not None and frame_id in self.variables_cache: + frame = self.variables_cache[frame_id] + globals_dict = frame.f_globals if hasattr(frame, 'f_globals') else {} + locals_dict = frame.f_locals if hasattr(frame, 'f_locals') else {} + else: + # Use current frame + frame = self.current_frame + if frame: + globals_dict = frame.f_globals if hasattr(frame, 'f_globals') else {} + locals_dict = frame.f_locals if hasattr(frame, 'f_locals') else {} + else: + globals_dict = globals() + locals_dict = {} + + try: + # Evaluate the expression + result = eval(expression, globals_dict, locals_dict) + return result + except Exception as e: + raise Exception(f"Evaluation error: {e}") + + def cleanup(self): + """Clean up resources.""" + self.variables_cache.clear() + self.breakpoints.clear() + if hasattr(sys, 'settrace'): + sys.settrace(None) diff --git a/python-ecosys/debugpy/demo.py b/python-ecosys/debugpy/demo.py new file mode 100644 index 000000000..d5b3d0923 --- /dev/null +++ b/python-ecosys/debugpy/demo.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Simple demo of MicroPython debugpy functionality.""" + +import sys +sys.path.insert(0, '.') + +import debugpy + +def simple_function(a, b): + """A simple function to demonstrate debugging.""" + result = a + b + print(f"Computing {a} + {b} = {result}") + return result + +def main(): + print("MicroPython debugpy Demo") + print("========================") + print() + + # Demonstrate trace functionality + print("1. Testing trace functionality:") + + def trace_function(frame, event, arg): + if event == 'call': + print(f" -> Entering function: {frame.f_code.co_name}") + elif event == 'line': + print(f" -> Executing line {frame.f_lineno} in {frame.f_code.co_name}") + elif event == 'return': + print(f" -> Returning from {frame.f_code.co_name} with value: {arg}") + return trace_function + + # Enable tracing + sys.settrace(trace_function) + + # Execute traced function + result = simple_function(5, 3) + + # Disable tracing + sys.settrace(None) + + print(f"Result: {result}") + print() + + # Demonstrate debugpy components + print("2. Testing debugpy components:") + + # Test PDB adapter + from debugpy.server.pdb_adapter import PdbAdapter + pdb = PdbAdapter() + + # Set some mock breakpoints + breakpoints = pdb.set_breakpoints("demo.py", [{"line": 10}, {"line": 15}]) + print(f" Set breakpoints: {len(breakpoints)} breakpoints") + + # Test messaging + from debugpy.common.messaging import JsonMessageChannel + print(" JsonMessageChannel available") + + print() + print("3. debugpy is ready for VS Code integration!") + print(" To use with VS Code:") + print(" - Import debugpy in your script") + print(" - Call debugpy.listen() to start the debug server") + print(" - Connect VS Code using the 'Attach to MicroPython' configuration") + print(" - Set breakpoints and debug normally") + +if __name__ == "__main__": + main() diff --git a/python-ecosys/debugpy/development_guide.md b/python-ecosys/debugpy/development_guide.md new file mode 100644 index 000000000..94f06b420 --- /dev/null +++ b/python-ecosys/debugpy/development_guide.md @@ -0,0 +1,84 @@ +# Debugging MicroPython debugpy with VS Code + +## Method 1: Direct Connection with Enhanced Logging + +1. **Start MicroPython with enhanced logging:** + ```bash + ~/micropython2/ports/unix/build-standard/micropython test_vscode.py + ``` + + This will now show detailed DAP protocol messages like: + ``` + [DAP] RECV: request initialize (seq=1) + [DAP] args: {...} + [DAP] SEND: response initialize (req_seq=1, success=True) + ``` + +2. **Connect VS Code debugger:** + - Use the launch configuration in `.vscode/launch.json` + - Or manually attach to `127.0.0.1:5678` + +3. **Look for issues in the terminal output** - you'll see all DAP message exchanges + +## Method 2: Using DAP Monitor (Recommended for detailed analysis) + +1. **Start MicroPython debugpy server:** + ```bash + ~/micropython2/ports/unix/build-standard/micropython test_vscode.py + ``` + +2. **In another terminal, start the DAP monitor:** + ```bash + python3 dap_monitor.py + ``` + + The monitor listens on port 5679 and forwards to port 5678 + +3. **Connect VS Code to the monitor:** + - Modify your VS Code launch config to connect to port `5679` instead of `5678` + - Or create a new launch config: + ```json + { + "name": "Debug via Monitor", + "type": "python", + "request": "attach", + "connect": { + "host": "127.0.0.1", + "port": 5679 + } + } + ``` + +4. **Analyze the complete DAP conversation** in the monitor terminal + +## VS Code Debug Logging + +Enable VS Code's built-in DAP logging: + +1. **Open VS Code settings** (Ctrl+,) +2. **Search for:** `debug.console.verbosity` +3. **Set to:** `verbose` +4. **Also set:** `debug.allowBreakpointsEverywhere` to `true` + +## Common Issues to Look For + +1. **Missing required DAP capabilities** - check the `initialize` response +2. **Breakpoint verification failures** - look for `setBreakpoints` exchanges +3. **Thread/stack frame issues** - check `stackTrace` and `scopes` responses +4. **Evaluation problems** - monitor `evaluate` request/response pairs + +## Expected DAP Sequence + +A successful debug session should show this sequence: + +1. `initialize` request → response with capabilities +2. `initialized` event +3. `setBreakpoints` request → response with verified breakpoints +4. `configurationDone` request → response +5. `attach` request → response +6. When execution hits breakpoint: `stopped` event +7. `stackTrace` request → response with frames +8. `scopes` request → response with local/global scopes +9. `continue` request → response to resume + +If any step fails or is missing, that's where the issue lies. \ No newline at end of file diff --git a/python-ecosys/debugpy/manifest.py b/python-ecosys/debugpy/manifest.py new file mode 100644 index 000000000..6c4228298 --- /dev/null +++ b/python-ecosys/debugpy/manifest.py @@ -0,0 +1,6 @@ +metadata( + description="MicroPython implementation of debugpy for remote debugging", + version="0.1.0", +) + +package("debugpy") diff --git a/python-ecosys/debugpy/test_vscode.py b/python-ecosys/debugpy/test_vscode.py new file mode 100644 index 000000000..aca063baf --- /dev/null +++ b/python-ecosys/debugpy/test_vscode.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Test script for VS Code debugging with MicroPython debugpy.""" + +import sys +sys.path.insert(0, '.') + +import debugpy + +def fibonacci(n): + """Calculate fibonacci number (iterative for efficiency).""" + if n <= 1: + return n + a, b = 0, 1 + for _ in range(2, n + 1): + a, b = b, a + b + return b + +def debuggable_code(): + """The actual code we want to debug - wrapped in a function so sys.settrace will trace it.""" + print("Starting debuggable code...") + + # Test data - set breakpoint here (using smaller numbers to avoid slow fibonacci) + numbers = [3, 4, 5] + for i, num in enumerate(numbers): + print(f"Calculating fibonacci({num})...") + result = fibonacci(num) # <-- SET BREAKPOINT HERE (line 26) + print(f"fibonacci({num}) = {result}") + print(sys.implementation) + import machine + print(dir(machine)) + + # Test manual breakpoint + print("\nTriggering manual breakpoint...") + debugpy.breakpoint() + print("Manual breakpoint triggered!") + + print("Test completed successfully!") + +def main(): + print("MicroPython VS Code Debugging Test") + print("==================================") + + # Start debug server + try: + debugpy.listen() + print("Debug server attached on 127.0.0.1:5678") + print("Connecting back to VS Code debugger now...") + # print("Set a breakpoint on line 26: 'result = fibonacci(num)'") + # print("Press Enter to continue after connecting debugger...") + # try: + # input() + # except: + # pass + + # Enable debugging for this thread + debugpy.debug_this_thread() + + # Give VS Code a moment to set breakpoints after attach + print("\nGiving VS Code time to set breakpoints...") + import time + time.sleep(2) + + # Call the debuggable code function so it gets traced + debuggable_code() + + except KeyboardInterrupt: + print("\nTest interrupted by user") + except Exception as e: + print(f"Error: {e}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/python-ecosys/debugpy/vscode_launch_example.json b/python-ecosys/debugpy/vscode_launch_example.json new file mode 100644 index 000000000..358f4d543 --- /dev/null +++ b/python-ecosys/debugpy/vscode_launch_example.json @@ -0,0 +1,22 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Micropython Attach", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}/lib/micropython-lib/python-ecosys/debugpy", + "remoteRoot": "." + } + ], + // "logToFile": true, + "justMyCode": false + } + ] +} \ No newline at end of file From c80bfd3198d12f2cb6fa72d192e780eb9a9779dd Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Thu, 12 Jun 2025 17:37:23 +0200 Subject: [PATCH 23/74] test_vscode: Add global variables to show vaiable tracking and hover. Signed-off-by: Jos Verlinde --- python-ecosys/debugpy/test_vscode.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python-ecosys/debugpy/test_vscode.py b/python-ecosys/debugpy/test_vscode.py index aca063baf..2dca82d34 100644 --- a/python-ecosys/debugpy/test_vscode.py +++ b/python-ecosys/debugpy/test_vscode.py @@ -2,10 +2,14 @@ """Test script for VS Code debugging with MicroPython debugpy.""" import sys + sys.path.insert(0, '.') import debugpy +foo = 42 +bar = "Hello, MicroPython!" + def fibonacci(n): """Calculate fibonacci number (iterative for efficiency).""" if n <= 1: @@ -17,6 +21,7 @@ def fibonacci(n): def debuggable_code(): """The actual code we want to debug - wrapped in a function so sys.settrace will trace it.""" + global foo print("Starting debuggable code...") # Test data - set breakpoint here (using smaller numbers to avoid slow fibonacci) @@ -24,6 +29,7 @@ def debuggable_code(): for i, num in enumerate(numbers): print(f"Calculating fibonacci({num})...") result = fibonacci(num) # <-- SET BREAKPOINT HERE (line 26) + foo += result # Modify foo to see if it gets traced print(f"fibonacci({num}) = {result}") print(sys.implementation) import machine From 291e7059f6170b1333dbe6a6774a3099f0f3197d Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Thu, 12 Jun 2025 17:38:44 +0200 Subject: [PATCH 24/74] debugpy: Improve variable retrievals. Signed-off-by: Jos Verlinde --- python-ecosys/debugpy/debugpy/server/debug_session.py | 5 ++++- python-ecosys/debugpy/debugpy/server/pdb_adapter.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index 4f60ee358..624467522 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -328,7 +328,10 @@ def _handle_evaluate(self, seq, args): expression = args.get("expression", "") frame_id = args.get("frameId") context = args.get("context", "watch") - + if not expression: + self.channel.send_response(CMD_EVALUATE, seq, success=False, + message="No expression provided") + return try: result = self.pdb.evaluate_expression(expression, frame_id) self.channel.send_response(CMD_EVALUATE, seq, body={ diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index 83693c65c..a33cf6655 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -235,7 +235,7 @@ def get_variables(self, variables_ref): continue try: - value_str = str(value) + value_str = repr(value) type_str = type(value).__name__ variables.append({ From 03bdabb6966a9448e7dcb227cbba34681bf81ee0 Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Thu, 12 Jun 2025 22:42:35 +0200 Subject: [PATCH 25/74] dap_monitor: Exit session on debugger disconnect. Signed-off-by: Jos Verlinde --- python-ecosys/debugpy/dap_monitor.py | 83 ++++++++++++++++------------ 1 file changed, 48 insertions(+), 35 deletions(-) diff --git a/python-ecosys/debugpy/dap_monitor.py b/python-ecosys/debugpy/dap_monitor.py index 3af4eba16..b323a61cb 100644 --- a/python-ecosys/debugpy/dap_monitor.py +++ b/python-ecosys/debugpy/dap_monitor.py @@ -9,6 +9,7 @@ class DAPMonitor: def __init__(self, listen_port=5679, target_host='127.0.0.1', target_port=5678): + self.disconnect = False self.listen_port = listen_port self.target_host = target_host self.target_port = target_port @@ -44,9 +45,9 @@ def start(self): threading.Thread(target=self.forward_server_to_client, daemon=True).start() print("DAP Monitor active - press Ctrl+C to stop") - while True: + while not self.disconnect: time.sleep(1) - + except KeyboardInterrupt: print("\nStopping DAP Monitor...") except Exception as e: @@ -106,43 +107,55 @@ def receive_dap_message(self, sock, source): return None content += chunk - # Log the message - try: - message = json.loads(content.decode('utf-8')) - msg_type = message.get('type', 'unknown') - command = message.get('command', message.get('event', 'unknown')) - seq = message.get('seq', 0) - - print(f"\n[{source}] {msg_type.upper()}: {command} (seq={seq})") - - if msg_type == 'request': - args = message.get('arguments', {}) - if args: - print(f" Arguments: {json.dumps(args, indent=2)}") - elif msg_type == 'response': - success = message.get('success', False) - req_seq = message.get('request_seq', 0) - print(f" Success: {success}, Request Seq: {req_seq}") - body = message.get('body') - if body: - print(f" Body: {json.dumps(body, indent=2)}") - msg = message.get('message') - if msg: - print(f" Message: {msg}") - elif msg_type == 'event': - body = message.get('body', {}) - if body: - print(f" Body: {json.dumps(body, indent=2)}") - - except json.JSONDecodeError: - print(f"\n[{source}] Invalid JSON: {content}") - + # Parse and Log the message + message = self.parse_dap(source, content) + self.log_dap_message(source, message) + # Check for disconnect command + if message: + if "disconnect" == message.get('command', message.get('event', 'unknown')): + print(f"\n[{source}] Disconnect command received, stopping monitor.") + self.disconnect = True return header + content - except Exception as e: print(f"Error receiving from {source}: {e}") return None - + + def parse_dap(self, source, content): + """Parse DAP message and log it.""" + try: + message = json.loads(content.decode('utf-8')) + return message + except json.JSONDecodeError: + print(f"\n[{source}] Invalid JSON: {content}") + return None + + def log_dap_message(self, source, message): + """Log DAP message details.""" + msg_type = message.get('type', 'unknown') + command = message.get('command', message.get('event', 'unknown')) + seq = message.get('seq', 0) + + print(f"\n[{source}] {msg_type.upper()}: {command} (seq={seq})") + + if msg_type == 'request': + args = message.get('arguments', {}) + if args: + print(f" Arguments: {json.dumps(args, indent=2)}") + elif msg_type == 'response': + success = message.get('success', False) + req_seq = message.get('request_seq', 0) + print(f" Success: {success}, Request Seq: {req_seq}") + body = message.get('body') + if body: + print(f" Body: {json.dumps(body, indent=2)}") + msg = message.get('message') + if msg: + print(f" Message: {msg}") + elif msg_type == 'event': + body = message.get('body', {}) + if body: + print(f" Body: {json.dumps(body, indent=2)}") + def send_raw_data(self, sock, data): """Send raw data to socket.""" try: From eebafee36f2d3c70f07a63c411ae0d060ccbe9ac Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Mon, 16 Jun 2025 12:12:30 +1000 Subject: [PATCH 26/74] debugpy: Fix VS Code path mapping to prevent read-only file copies. When breakpoints are hit, VS Code was opening read-only copies of source files instead of the original workspace files due to path mismatches between VS Code's absolute paths and MicroPython's runtime paths. Changes: - Add a path mapping dictionary tracking VS Code path <-> runtime path - Enhance breakpoint matching to handle relative paths and basename matches - Update stack trace reporting to use mapped VS Code paths - Add debug logging for path mapping diagnostics - Fix VS Code launch configuration (debugpy -> python, enable logging) This ensures VS Code correctly opens the original editable source files when debugging, rather than creating read-only temporary copies. Signed-off-by: Andrew Leech --- .../debugpy/debugpy/server/debug_session.py | 3 ++ .../debugpy/debugpy/server/pdb_adapter.py | 50 ++++++++++++++++++- .../debugpy/vscode_launch_example.json | 8 +-- 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index 624467522..3a1a5135d 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -263,6 +263,9 @@ def _handle_set_breakpoints(self, seq, args): filename = source.get("path", "") breakpoints = args.get("breakpoints", []) + # Debug log the source information + self._debug_print(f"[DAP] setBreakpoints source info: {source}") + # Set breakpoints in pdb adapter actual_breakpoints = self.pdb.set_breakpoints(filename, breakpoints) diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index a33cf6655..204862073 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -2,6 +2,7 @@ import sys import time +import os from ..common.constants import ( TRACE_CALL, TRACE_LINE, TRACE_RETURN, TRACE_EXCEPTION, SCOPE_LOCALS, SCOPE_GLOBALS @@ -21,11 +22,27 @@ def __init__(self): self.continue_event = False self.variables_cache = {} # frameId -> variables self.frame_id_counter = 1 + self.path_mapping = {} # runtime_path -> vscode_path mapping def _debug_print(self, message): """Print debug message only if debug logging is enabled.""" if hasattr(self, '_debug_session') and self._debug_session.debug_logging: print(message) + + def _normalize_path(self, path): + """Normalize a file path for consistent comparisons.""" + # Convert to absolute path if possible + try: + if hasattr(os.path, 'abspath'): + path = os.path.abspath(path) + elif hasattr(os.path, 'realpath'): + path = os.path.realpath(path) + except: + pass + + # Ensure consistent separators + path = path.replace('\\', '/') + return path def set_trace_function(self, trace_func): """Install the trace function.""" @@ -39,6 +56,9 @@ def set_breakpoints(self, filename, breakpoints): self.breakpoints[filename] = {} actual_breakpoints = [] + # Debug log the breakpoint path + self._debug_print(f"[PDB] Setting breakpoints for file: {filename}") + for bp in breakpoints: line = bp.get("line") if line: @@ -73,12 +93,23 @@ def should_stop(self, frame, event, arg): if filename in self.breakpoints: if lineno in self.breakpoints[filename]: self._debug_print(f"[PDB] HIT BREAKPOINT (exact match) at {filename}:{lineno}") + # Record the path mapping (in this case, they're already the same) + self.path_mapping[filename] = filename self.hit_breakpoint = True return True # Also try checking by basename for path mismatches def basename(path): return path.split('/')[-1] if '/' in path else path + + # Check if this might be a relative path match + def ends_with_path(full_path, relative_path): + """Check if full_path ends with relative_path components.""" + full_parts = full_path.replace('\\', '/').split('/') + rel_parts = relative_path.replace('\\', '/').split('/') + if len(rel_parts) > len(full_parts): + return False + return full_parts[-len(rel_parts):] == rel_parts file_basename = basename(filename) self._debug_print(f"[PDB] Fallback basename match: '{file_basename}' vs available files") @@ -89,6 +120,18 @@ def basename(path): self._debug_print(f"[PDB] Basename match found! Checking line {lineno} in {list(self.breakpoints[bp_file].keys())}") if lineno in self.breakpoints[bp_file]: self._debug_print(f"[PDB] HIT BREAKPOINT (fallback basename match) at {filename}:{lineno} -> {bp_file}") + # Record the path mapping so we can report the correct path in stack traces + self.path_mapping[filename] = bp_file + self.hit_breakpoint = True + return True + + # Also check if the runtime path might be relative and the breakpoint path absolute + if ends_with_path(bp_file, filename): + self._debug_print(f"[PDB] Relative path match: {bp_file} ends with {filename}") + if lineno in self.breakpoints[bp_file]: + self._debug_print(f"[PDB] HIT BREAKPOINT (relative path match) at {filename}:{lineno} -> {bp_file}") + # Record the path mapping so we can report the correct path in stack traces + self.path_mapping[filename] = bp_file self.hit_breakpoint = True return True @@ -171,11 +214,16 @@ def get_stack_trace(self): name = frame.f_code.co_name line = frame.f_lineno + # Use the VS Code path if we have a mapping, otherwise use the original path + display_path = self.path_mapping.get(filename, filename) + if filename != display_path: + self._debug_print(f"[PDB] Stack trace path mapping: {filename} -> {display_path}") + # Create frame info frames.append({ "id": frame_id, "name": name, - "source": {"path": filename}, + "source": {"path": display_path}, "line": line, "column": 1, "endLine": line, diff --git a/python-ecosys/debugpy/vscode_launch_example.json b/python-ecosys/debugpy/vscode_launch_example.json index 358f4d543..388e696bd 100644 --- a/python-ecosys/debugpy/vscode_launch_example.json +++ b/python-ecosys/debugpy/vscode_launch_example.json @@ -2,8 +2,8 @@ "version": "0.2.0", "configurations": [ { - "name": "Micropython Attach", - "type": "debugpy", + "name": "Attach to MicroPython", + "type": "python", "request": "attach", "connect": { "host": "localhost", @@ -11,11 +11,11 @@ }, "pathMappings": [ { - "localRoot": "${workspaceFolder}/lib/micropython-lib/python-ecosys/debugpy", + "localRoot": "${workspaceFolder}", "remoteRoot": "." } ], - // "logToFile": true, + "logToFile": true, "justMyCode": false } ] From 69c3b26176225853d1f5261442dfcf9e79284f50 Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Thu, 19 Jun 2025 01:02:47 +0200 Subject: [PATCH 27/74] debugpy: Enhance PDB adapter with special variable processing. Signed-off-by: Jos Verlinde --- .../debugpy/debugpy/server/pdb_adapter.py | 123 ++++++++++++------ 1 file changed, 84 insertions(+), 39 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index 204862073..c05aec615 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -3,10 +3,15 @@ import sys import time import os +import json from ..common.constants import ( TRACE_CALL, TRACE_LINE, TRACE_RETURN, TRACE_EXCEPTION, SCOPE_LOCALS, SCOPE_GLOBALS ) +VARREF_LOCALS = 1 +VARREF_GLOBALS = 2 +VARREF_LOCALS_SPECIAL = 3 +VARREF_GLOBALS_SPECIAL = 4 class PdbAdapter: @@ -26,7 +31,7 @@ def __init__(self): def _debug_print(self, message): """Print debug message only if debug logging is enabled.""" - if hasattr(self, '_debug_session') and self._debug_session.debug_logging: + if hasattr(self, '_debug_session') and self._debug_session.debug_logging: # type: ignore print(message) def _normalize_path(self, path): @@ -197,7 +202,7 @@ def wait_for_continue(self): while not self.continue_event: # Process any pending DAP messages (scopes, variables, etc.) if hasattr(self, '_debug_session'): - self._debug_session.process_pending_messages() + self._debug_session.process_pending_messages() # type: ignore time.sleep(0.01) def get_stack_trace(self): @@ -213,21 +218,25 @@ def get_stack_trace(self): filename = frame.f_code.co_filename name = frame.f_code.co_name line = frame.f_lineno - + if "" in filename or filename.endswith("debugpy.py") : + hint = 'subtle' + else : + hint = 'normal' + # Use the VS Code path if we have a mapping, otherwise use the original path display_path = self.path_mapping.get(filename, filename) if filename != display_path: self._debug_print(f"[PDB] Stack trace path mapping: {filename} -> {display_path}") - - # Create frame info + # Create StackFrame info frames.append({ "id": frame_id, - "name": name, + "name": name + f" {type(frame.f_code.co_filename).__name__}", "source": {"path": display_path}, "line": line, "column": 1, "endLine": line, - "endColumn": 1 + "endColumn": 1, + "presentationHint": hint }) # Cache frame for variable access @@ -248,60 +257,97 @@ def get_scopes(self, frame_id): scopes = [ { "name": "Locals", - "variablesReference": frame_id * 1000 + 1, + "variablesReference": frame_id * 1000 + VARREF_LOCALS, "expensive": False }, { "name": "Globals", - "variablesReference": frame_id * 1000 + 2, + "variablesReference": frame_id * 1000 + VARREF_GLOBALS , "expensive": False } ] return scopes - def get_variables(self, variables_ref): - """Get variables for a scope.""" - frame_id = variables_ref // 1000 - scope_type = variables_ref % 1000 - - if frame_id not in self.variables_cache: - return [] - - frame = self.variables_cache[frame_id] + def _process_special_variables(self, var_dict): + """Process special variables (those starting and ending with __).""" + variables = [] + for name, value in var_dict.items(): + if name.startswith('__') and name.endswith('__'): + try: + value_str = json.dumps(value) + type_str = type(value).__name__ + variables.append({ + "name": name, + "value": value_str, + "type": type_str, + "variablesReference": 0 + }) + except Exception: + variables.append(self._var_error(name)) + return variables + + def _process_regular_variables(self, var_dict): + """Process regular variables (excluding special ones).""" variables = [] - - if scope_type == 1: # Locals - var_dict = frame.f_locals if hasattr(frame, 'f_locals') else {} - elif scope_type == 2: # Globals - var_dict = frame.f_globals if hasattr(frame, 'f_globals') else {} - else: - return [] - for name, value in var_dict.items(): # Skip private/internal variables if name.startswith('__') and name.endswith('__'): continue - try: - value_str = repr(value) + value_str = json.dumps(value) type_str = type(value).__name__ - variables.append({ "name": name, "value": value_str, "type": type_str, - "variablesReference": 0 # Simple implementation - no nested objects - }) - except Exception: - variables.append({ - "name": name, - "value": "", - "type": "unknown", "variablesReference": 0 }) - + except Exception: + variables.append(self._var_error(name)) return variables + + @staticmethod + def _var_error(name:str): + return {"name": name, "value": "", "type": "unknown", "variablesReference": 0 } + + @staticmethod + def _special_vars(varref:int): + return {"name": "Special", "value": "", "variablesReference": varref} + + def get_variables(self, variables_ref): + """Get variables for a scope.""" + frame_id = variables_ref // 1000 + scope_type = variables_ref % 1000 + + if frame_id not in self.variables_cache: + return [] + + frame = self.variables_cache[frame_id] + # Handle special scope types first + if scope_type == VARREF_LOCALS_SPECIAL: + var_dict = frame.f_locals if hasattr(frame, 'f_locals') else {} + return self._process_special_variables(var_dict) + elif scope_type == VARREF_GLOBALS_SPECIAL: + var_dict = frame.f_globals if hasattr(frame, 'f_globals') else {} + return self._process_special_variables(var_dict) + + # Handle regular scope types with special folder + variables = [] + if scope_type == VARREF_LOCALS: + var_dict = frame.f_locals if hasattr(frame, 'f_locals') else {} + variables.append(self._special_vars( VARREF_LOCALS_SPECIAL)) + elif scope_type == VARREF_GLOBALS: + var_dict = frame.f_globals if hasattr(frame, 'f_globals') else {} + variables.append(self._special_vars( VARREF_GLOBALS_SPECIAL)) + else: + # Invalid reference, return empty + return [] + + # Add regular variables + variables.extend(self._process_regular_variables(var_dict)) + return variables + def evaluate_expression(self, expression, frame_id=None): """Evaluate an expression in the context of a frame.""" if frame_id is not None and frame_id in self.variables_cache: @@ -317,14 +363,13 @@ def evaluate_expression(self, expression, frame_id=None): else: globals_dict = globals() locals_dict = {} - try: # Evaluate the expression result = eval(expression, globals_dict, locals_dict) return result except Exception as e: raise Exception(f"Evaluation error: {e}") - + def cleanup(self): """Clean up resources.""" self.variables_cache.clear() From 02bba918b5d5b4e9ee9f1a52a27f561ccbabf574 Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Thu, 19 Jun 2025 02:13:14 +0200 Subject: [PATCH 28/74] debugpy/dap_monitor: Add cli for target and ports. Signed-off-by: Jos Verlinde --- python-ecosys/debugpy/dap_monitor.py | 14 +++++++++++++- .../debugpy/debugpy/server/pdb_adapter.py | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/python-ecosys/debugpy/dap_monitor.py b/python-ecosys/debugpy/dap_monitor.py index b323a61cb..20ab90800 100644 --- a/python-ecosys/debugpy/dap_monitor.py +++ b/python-ecosys/debugpy/dap_monitor.py @@ -6,6 +6,7 @@ import json import time import sys +import argparse class DAPMonitor: def __init__(self, listen_port=5679, target_host='127.0.0.1', target_port=5678): @@ -171,5 +172,16 @@ def cleanup(self): self.server_sock.close() if __name__ == "__main__": - monitor = DAPMonitor() + + parser = argparse.ArgumentParser(description="DAP protocol monitor proxy") + parser.add_argument("--target-host", "--th", default="127.0.0.1", help="Target debugpy host (default: 127.0.0.1)") + parser.add_argument("--target-port", "--tp", type=int, default=5678, help="Target debugpy port (default: 5678)") + parser.add_argument("--listen-port", "--lp", type=int, default=5679, help="Port to listen for VS Code (default: 5679)") + args = parser.parse_args() + + monitor = DAPMonitor( + listen_port=args.listen_port, + target_host=args.target_host, + target_port=args.target_port + ) monitor.start() \ No newline at end of file diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index c05aec615..524574b2b 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -230,7 +230,7 @@ def get_stack_trace(self): # Create StackFrame info frames.append({ "id": frame_id, - "name": name + f" {type(frame.f_code.co_filename).__name__}", + "name": name, "source": {"path": display_path}, "line": line, "column": 1, From c5082c1230bf35fd8ceb45ffb029be038e3d791b Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Thu, 19 Jun 2025 13:07:46 +0200 Subject: [PATCH 29/74] debugpy/debug_session: Detect bare metal ports. Signed-off-by: Jos Verlinde --- .../debugpy/debugpy/server/debug_session.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index 3a1a5135d..9b0f8edbe 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -31,6 +31,10 @@ def _debug_print(self, message): """Print debug message only if debug logging is enabled.""" if self.debug_logging: print(message) + + @property + def _baremetal(self) -> bool: + return sys.platform not in ("linux") # to be expanded def start(self): """Start the debug session message loop.""" @@ -369,15 +373,23 @@ def _handle_source(self, seq, args): """Handle source request.""" source = args.get("source", {}) source_path = source.get("path", "") - + if self._baremetal or not source_path: + # BUGBUG: unable to read the source on ESP32 + # Possible an effect of the import / inialization sequence ? + # Nothe that other source files ( other.py) do not seem to get requested in the same way + self.channel.send_response(CMD_SOURCE, seq, success=False) + return + self._debug_print(f"[DAP] Processing source request for path: {source}") try: # Try to read the source file - with open(source_path, 'r') as f: + with open(source_path) as f: content = f.read() self.channel.send_response(CMD_SOURCE, seq, body={"content": content}) except Exception as e: self.channel.send_response(CMD_SOURCE, seq, success=False, - message=f"Could not read source: {e}") + message="cancelled" + # message=f"Could not read source: {e}" + ) def _trace_function(self, frame, event, arg): """Trace function called by sys.settrace.""" From e2c5bb3601c801ab76540c517337f396b62f025c Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Thu, 19 Jun 2025 13:15:16 +0200 Subject: [PATCH 30/74] debugpy : Format code. Signed-off-by: Jos Verlinde --- python-ecosys/debugpy/dap_monitor.py | 32 ++-- python-ecosys/debugpy/debugpy/__init__.py | 8 +- .../debugpy/debugpy/common/messaging.py | 42 ++--- python-ecosys/debugpy/debugpy/public_api.py | 22 +-- .../debugpy/debugpy/server/debug_session.py | 143 +++++++++--------- .../debugpy/debugpy/server/pdb_adapter.py | 85 +++++------ python-ecosys/debugpy/demo.py | 24 +-- python-ecosys/debugpy/test_vscode.py | 18 +-- 8 files changed, 186 insertions(+), 188 deletions(-) diff --git a/python-ecosys/debugpy/dap_monitor.py b/python-ecosys/debugpy/dap_monitor.py index 20ab90800..93d02ddf7 100644 --- a/python-ecosys/debugpy/dap_monitor.py +++ b/python-ecosys/debugpy/dap_monitor.py @@ -16,35 +16,35 @@ def __init__(self, listen_port=5679, target_host='127.0.0.1', target_port=5678): self.target_port = target_port self.client_sock = None self.server_sock = None - + def start(self): """Start the DAP monitor proxy.""" print(f"DAP Monitor starting on port {self.listen_port}") print(f"Will forward to {self.target_host}:{self.target_port}") print("Start MicroPython debugpy server first, then connect VS Code to port 5679") - + # Create listening socket listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) listener.bind(('127.0.0.1', self.listen_port)) listener.listen(1) - + print(f"Listening for VS Code connection on port {self.listen_port}...") - + try: # Wait for VS Code to connect self.client_sock, client_addr = listener.accept() print(f"VS Code connected from {client_addr}") - + # Connect to MicroPython debugpy server self.server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.server_sock.connect((self.target_host, self.target_port)) print(f"Connected to MicroPython debugpy at {self.target_host}:{self.target_port}") - + # Start forwarding threads threading.Thread(target=self.forward_client_to_server, daemon=True).start() threading.Thread(target=self.forward_server_to_client, daemon=True).start() - + print("DAP Monitor active - press Ctrl+C to stop") while not self.disconnect: time.sleep(1) @@ -55,7 +55,7 @@ def start(self): print(f"Error: {e}") finally: self.cleanup() - + def forward_client_to_server(self): """Forward messages from VS Code client to MicroPython server.""" try: @@ -66,7 +66,7 @@ def forward_client_to_server(self): self.send_raw_data(self.server_sock, data) except Exception as e: print(f"Client->Server forwarding error: {e}") - + def forward_server_to_client(self): """Forward messages from MicroPython server to VS Code client.""" try: @@ -77,7 +77,7 @@ def forward_server_to_client(self): self.send_raw_data(self.client_sock, data) except Exception as e: print(f"Server->Client forwarding error: {e}") - + def receive_dap_message(self, sock, source): """Receive and log a DAP message.""" try: @@ -88,7 +88,7 @@ def receive_dap_message(self, sock, source): if not byte: return None header += byte - + # Parse content length header_str = header.decode('utf-8') content_length = 0 @@ -96,10 +96,10 @@ def receive_dap_message(self, sock, source): if line.startswith('Content-Length:'): content_length = int(line.split(':', 1)[1].strip()) break - + if content_length == 0: return None - + # Read content content = b"" while len(content) < content_length: @@ -107,7 +107,7 @@ def receive_dap_message(self, sock, source): if not chunk: return None content += chunk - + # Parse and Log the message message = self.parse_dap(source, content) self.log_dap_message(source, message) @@ -163,7 +163,7 @@ def send_raw_data(self, sock, data): sock.send(data) except Exception as e: print(f"Error sending data: {e}") - + def cleanup(self): """Clean up sockets.""" if self.client_sock: @@ -184,4 +184,4 @@ def cleanup(self): target_host=args.target_host, target_port=args.target_port ) - monitor.start() \ No newline at end of file + monitor.start() diff --git a/python-ecosys/debugpy/debugpy/__init__.py b/python-ecosys/debugpy/debugpy/__init__.py index b7649bd5c..3912a49a5 100644 --- a/python-ecosys/debugpy/debugpy/__init__.py +++ b/python-ecosys/debugpy/debugpy/__init__.py @@ -11,10 +11,10 @@ from .common.constants import DEFAULT_HOST, DEFAULT_PORT __all__ = [ - "listen", - "wait_for_client", - "breakpoint", - "debug_this_thread", "DEFAULT_HOST", "DEFAULT_PORT", + "breakpoint", + "debug_this_thread", + "listen", + "wait_for_client", ] diff --git a/python-ecosys/debugpy/debugpy/common/messaging.py b/python-ecosys/debugpy/debugpy/common/messaging.py index bc264e3ff..7a588bab3 100644 --- a/python-ecosys/debugpy/debugpy/common/messaging.py +++ b/python-ecosys/debugpy/debugpy/common/messaging.py @@ -6,25 +6,25 @@ class JsonMessageChannel: """Handles JSON message communication over a socket using DAP format.""" - + def __init__(self, sock, debug_callback=None): self.sock = sock self.seq = 0 self.closed = False self._recv_buffer = b"" self._debug_print = debug_callback or (lambda x: None) # Default to no-op - + def send_message(self, msg_type, command=None, **kwargs): """Send a DAP message.""" if self.closed: return - + self.seq += 1 message = { "seq": self.seq, "type": msg_type, } - + if command: if msg_type == MSG_TYPE_REQUEST: message["command"] = command @@ -42,20 +42,20 @@ def send_message(self, msg_type, command=None, **kwargs): message["event"] = command if kwargs: message["body"] = kwargs - + json_str = json.dumps(message) content = json_str.encode("utf-8") header = f"Content-Length: {len(content)}\r\n\r\n".encode("utf-8") - + try: self.sock.send(header + content) except OSError: self.closed = True - + def send_request(self, command, **kwargs): """Send a request message.""" self.send_message(MSG_TYPE_REQUEST, command, **kwargs) - + def send_response(self, command, request_seq, success=True, body=None, message=None): """Send a response message.""" kwargs = {"request_seq": request_seq, "success": success} @@ -63,27 +63,27 @@ def send_response(self, command, request_seq, success=True, body=None, message=N kwargs["body"] = body if message is not None: kwargs["message"] = message - + self._debug_print(f"[DAP] SEND: response {command} (req_seq={request_seq}, success={success})") if body: self._debug_print(f"[DAP] body: {body}") if message: self._debug_print(f"[DAP] message: {message}") - + self.send_message(MSG_TYPE_RESPONSE, command, **kwargs) - + def send_event(self, event, **kwargs): """Send an event message.""" self._debug_print(f"[DAP] SEND: event {event}") if kwargs: self._debug_print(f"[DAP] body: {kwargs}") self.send_message(MSG_TYPE_EVENT, event, **kwargs) - + def recv_message(self): """Receive a DAP message.""" if self.closed: return None - + try: # Read headers while b"\r\n\r\n" not in self._recv_buffer: @@ -99,21 +99,21 @@ def recv_message(self): return None # No data available self.closed = True return None - + header_end = self._recv_buffer.find(b"\r\n\r\n") header_str = self._recv_buffer[:header_end].decode("utf-8") self._recv_buffer = self._recv_buffer[header_end + 4:] - + # Parse Content-Length content_length = 0 for line in header_str.split("\r\n"): if line.startswith("Content-Length:"): content_length = int(line.split(":", 1)[1].strip()) break - + if content_length == 0: return None - + # Read body while len(self._recv_buffer) < content_length: try: @@ -127,10 +127,10 @@ def recv_message(self): return None self.closed = True return None - + body = self._recv_buffer[:content_length] self._recv_buffer = self._recv_buffer[content_length:] - + # Parse JSON try: message = json.loads(body.decode("utf-8")) @@ -139,12 +139,12 @@ def recv_message(self): except (ValueError, UnicodeDecodeError) as e: print(f"[DAP] JSON parse error: {e}") return None - + except OSError as e: print(f"[DAP] Socket error in recv_message: {e}") self.closed = True return None - + def close(self): """Close the channel.""" self.closed = True diff --git a/python-ecosys/debugpy/debugpy/public_api.py b/python-ecosys/debugpy/debugpy/public_api.py index 137706efe..8642be989 100644 --- a/python-ecosys/debugpy/debugpy/public_api.py +++ b/python-ecosys/debugpy/debugpy/public_api.py @@ -19,35 +19,35 @@ def listen(port=DEFAULT_PORT, host=DEFAULT_HOST): (host, port) tuple of the actual listening address """ global _debug_session - + if _debug_session is not None: raise RuntimeError("Already listening for debugger") - + # Create listening socket listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) except: pass # Not supported in MicroPython - + # Use getaddrinfo for MicroPython compatibility addr_info = socket.getaddrinfo(host, port) addr = addr_info[0][-1] # Get the sockaddr listener.bind(addr) listener.listen(1) - + # getsockname not available in MicroPython, use original values print(f"Debugpy listening on {host}:{port}") - + # Wait for connection client_sock = None try: client_sock, client_addr = listener.accept() print(f"Debugger connected from {client_addr}") - + # Create debug session _debug_session = DebugSession(client_sock) - + # Handle just the initialize request, then return immediately print("[DAP] Waiting for initialize request...") init_message = _debug_session.channel.recv_message() @@ -56,12 +56,12 @@ def listen(port=DEFAULT_PORT, host=DEFAULT_HOST): print("[DAP] Initialize request handled - returning control immediately") else: print(f"[DAP] Warning: Expected initialize, got {init_message}") - + # Set socket to non-blocking for subsequent message processing _debug_session.channel.sock.settimeout(0.001) - + print("[DAP] Debug session ready - all other messages will be handled in trace function") - + except Exception as e: print(f"[DAP] Connection error: {e}") if client_sock: @@ -70,7 +70,7 @@ def listen(port=DEFAULT_PORT, host=DEFAULT_HOST): finally: # Only close the listener, not the client connection listener.close() - + return (host, port) diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index 9b0f8edbe..43e2d442c 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -15,7 +15,7 @@ class DebugSession: """Manages a debugging session with a DAP client.""" - + def __init__(self, client_socket): self.debug_logging = False # Initialize first self.channel = JsonMessageChannel(client_socket, self._debug_print) @@ -26,7 +26,7 @@ def __init__(self, client_socket): self.thread_id = 1 # Simple single-thread model self.stepping = False self.paused = False - + def _debug_print(self, message): """Print debug message only if debug logging is enabled.""" if self.debug_logging: @@ -34,8 +34,8 @@ def _debug_print(self, message): @property def _baremetal(self) -> bool: - return sys.platform not in ("linux") # to be expanded - + return sys.platform not in ("linux") # to be expanded + def start(self): """Start the debug session message loop.""" try: @@ -43,51 +43,51 @@ def start(self): message = self.channel.recv_message() if message is None: break - + self._handle_message(message) - + except Exception as e: print(f"Debug session error: {e}") finally: self.disconnect() - + def initialize_connection(self): """Initialize the connection - handle just the essential initial messages then return.""" # Note: debug_logging not available yet during init, so we always show these messages print("[DAP] Processing initial DAP messages...") - + try: # Process initial messages quickly and return control to main thread # We'll handle ongoing messages in the trace function attached = False message_count = 0 max_init_messages = 6 # Just handle the first few essential messages - + while message_count < max_init_messages and not attached: try: # Short timeout - don't block the main thread for long self.channel.sock.settimeout(1.0) message = self.channel.recv_message() if message is None: - print(f"[DAP] No more messages in initial batch") + print("[DAP] No more messages in initial batch") break - + print(f"[DAP] Initial message #{message_count + 1}: {message.get('command')}") self._handle_message(message) message_count += 1 - + # Just wait for attach, then we can return control if message.get('command') == 'attach': attached = True print("[DAP] ✅ Attach received - returning control to main thread") break - + except Exception as e: print(f"[DAP] Exception in initial processing: {e}") break finally: self.channel.sock.settimeout(None) - + # After attach, continue processing a few more messages quickly if attached: self._debug_print("[DAP] Processing remaining setup messages...") @@ -105,41 +105,41 @@ def initialize_connection(self): break finally: self.channel.sock.settimeout(None) - - print(f"[DAP] Initial setup complete - main thread can continue") - + + print("[DAP] Initial setup complete - main thread can continue") + except Exception as e: print(f"[DAP] Initialization error: {e}") - + def process_pending_messages(self): """Process any pending DAP messages without blocking.""" try: # Set socket to non-blocking mode for message processing self.channel.sock.settimeout(0.001) # Very short timeout - + while True: message = self.channel.recv_message() if message is None: break self._handle_message(message) - + except Exception: # No messages available or socket error pass finally: # Reset to blocking mode self.channel.sock.settimeout(None) - + def _handle_message(self, message): """Handle incoming DAP messages.""" msg_type = message.get("type") command = message.get("command", message.get("event", "unknown")) seq = message.get("seq", 0) - + self._debug_print(f"[DAP] RECV: {msg_type} {command} (seq={seq})") if message.get("arguments"): self._debug_print(f"[DAP] args: {message['arguments']}") - + if msg_type == "request": self._handle_request(message) elif msg_type == "response": @@ -148,13 +148,13 @@ def _handle_message(self, message): elif msg_type == "event": # We don't expect events from client self._debug_print(f"[DAP] Unexpected event from client: {message}") - + def _handle_request(self, message): """Handle DAP request messages.""" command = message.get("command") seq = message.get("seq", 0) args = message.get("arguments", {}) - + try: if command == CMD_INITIALIZE: self._handle_initialize(seq, args) @@ -191,13 +191,13 @@ def _handle_request(self, message): elif command == CMD_SOURCE: self._handle_source(seq, args) else: - self.channel.send_response(command, seq, success=False, + self.channel.send_response(command, seq, success=False, message=f"Unknown command: {command}") - + except Exception as e: - self.channel.send_response(command, seq, success=False, + self.channel.send_response(command, seq, success=False, message=str(e)) - + def _handle_initialize(self, seq, args): """Handle initialize request.""" capabilities = { @@ -235,87 +235,87 @@ def _handle_initialize(self, seq, args): "supportsBreakpointLocationsRequest": False, "supportsClipboardContext": False, } - + self.channel.send_response(CMD_INITIALIZE, seq, body=capabilities) self.channel.send_event(EVENT_INITIALIZED) self.initialized = True - + def _handle_launch(self, seq, args): """Handle launch request.""" # For attach-mode debugging, we don't need to launch anything self.channel.send_response(CMD_LAUNCH, seq) - + def _handle_attach(self, seq, args): """Handle attach request.""" # Check if debug logging should be enabled self.debug_logging = args.get("logToFile", False) - + self._debug_print(f"[DAP] Processing attach request with args: {args}") print(f"[DAP] Debug logging {'enabled' if self.debug_logging else 'disabled'} (logToFile={self.debug_logging})") - + # Enable trace function self.pdb.set_trace_function(self._trace_function) self.channel.send_response(CMD_ATTACH, seq) - + # After successful attach, we might need to send additional events # Some debuggers expect a 'process' event or thread events self._debug_print("[DAP] Attach completed, debugging is now active") - + def _handle_set_breakpoints(self, seq, args): """Handle setBreakpoints request.""" source = args.get("source", {}) filename = source.get("path", "") breakpoints = args.get("breakpoints", []) - + # Debug log the source information self._debug_print(f"[DAP] setBreakpoints source info: {source}") - + # Set breakpoints in pdb adapter actual_breakpoints = self.pdb.set_breakpoints(filename, breakpoints) - - self.channel.send_response(CMD_SET_BREAKPOINTS, seq, + + self.channel.send_response(CMD_SET_BREAKPOINTS, seq, body={"breakpoints": actual_breakpoints}) - + def _handle_continue(self, seq, args): """Handle continue request.""" self.stepping = False self.paused = False self.pdb.continue_execution() self.channel.send_response(CMD_CONTINUE, seq) - + def _handle_next(self, seq, args): """Handle next (step over) request.""" self.stepping = True self.paused = False self.pdb.step_over() self.channel.send_response(CMD_NEXT, seq) - + def _handle_step_in(self, seq, args): """Handle stepIn request.""" self.stepping = True self.paused = False self.pdb.step_into() self.channel.send_response(CMD_STEP_IN, seq) - + def _handle_step_out(self, seq, args): """Handle stepOut request.""" self.stepping = True self.paused = False self.pdb.step_out() self.channel.send_response(CMD_STEP_OUT, seq) - + def _handle_pause(self, seq, args): """Handle pause request.""" self.paused = True self.pdb.pause() self.channel.send_response(CMD_PAUSE, seq) - + def _handle_stack_trace(self, seq, args): """Handle stackTrace request.""" stack_frames = self.pdb.get_stack_trace() - self.channel.send_response(CMD_STACK_TRACE, seq, + self.channel.send_response(CMD_STACK_TRACE, seq, body={"stackFrames": stack_frames, "totalFrames": len(stack_frames)}) - + def _handle_scopes(self, seq, args): """Handle scopes request.""" frame_id = args.get("frameId", 0) @@ -323,20 +323,20 @@ def _handle_scopes(self, seq, args): scopes = self.pdb.get_scopes(frame_id) self._debug_print(f"[DAP] Generated scopes: {scopes}") self.channel.send_response(CMD_SCOPES, seq, body={"scopes": scopes}) - + def _handle_variables(self, seq, args): """Handle variables request.""" variables_ref = args.get("variablesReference", 0) variables = self.pdb.get_variables(variables_ref) self.channel.send_response(CMD_VARIABLES, seq, body={"variables": variables}) - + def _handle_evaluate(self, seq, args): """Handle evaluate request.""" expression = args.get("expression", "") frame_id = args.get("frameId") context = args.get("context", "watch") if not expression: - self.channel.send_response(CMD_EVALUATE, seq, success=False, + self.channel.send_response(CMD_EVALUATE, seq, success=False, message="No expression provided") return try: @@ -346,20 +346,20 @@ def _handle_evaluate(self, seq, args): "variablesReference": 0 }) except Exception as e: - self.channel.send_response(CMD_EVALUATE, seq, success=False, + self.channel.send_response(CMD_EVALUATE, seq, success=False, message=str(e)) - + def _handle_disconnect(self, seq, args): """Handle disconnect request.""" self.channel.send_response(CMD_DISCONNECT, seq) self.disconnect() - + def _handle_configuration_done(self, seq, args): """Handle configurationDone request.""" # This indicates that the client has finished configuring breakpoints # and is ready to start debugging self.channel.send_response(CMD_CONFIGURATION_DONE, seq) - + def _handle_threads(self, seq, args): """Handle threads request.""" # MicroPython is single-threaded, so return one thread @@ -368,13 +368,13 @@ def _handle_threads(self, seq, args): "name": "main" }] self.channel.send_response(CMD_THREADS, seq, body={"threads": threads}) - + def _handle_source(self, seq, args): """Handle source request.""" source = args.get("source", {}) source_path = source.get("path", "") if self._baremetal or not source_path: - # BUGBUG: unable to read the source on ESP32 + # BUGBUG: unable to read the source on ESP32 # Possible an effect of the import / inialization sequence ? # Nothe that other source files ( other.py) do not seem to get requested in the same way self.channel.send_response(CMD_SOURCE, seq, success=False) @@ -385,53 +385,52 @@ def _handle_source(self, seq, args): with open(source_path) as f: content = f.read() self.channel.send_response(CMD_SOURCE, seq, body={"content": content}) - except Exception as e: - self.channel.send_response(CMD_SOURCE, seq, success=False, + except Exception: + self.channel.send_response(CMD_SOURCE, seq, success=False, message="cancelled" # message=f"Could not read source: {e}" ) - + def _trace_function(self, frame, event, arg): """Trace function called by sys.settrace.""" # Process any pending DAP messages frequently self.process_pending_messages() - + # Handle breakpoints and stepping if self.pdb.should_stop(frame, event, arg): - self._send_stopped_event(STOP_REASON_BREAKPOINT if self.pdb.hit_breakpoint else + self._send_stopped_event(STOP_REASON_BREAKPOINT if self.pdb.hit_breakpoint else STOP_REASON_STEP if self.stepping else STOP_REASON_PAUSE) # Wait for continue command self.pdb.wait_for_continue() - + return self._trace_function - + def _send_stopped_event(self, reason): """Send stopped event to client.""" - self.channel.send_event(EVENT_STOPPED, - reason=reason, + self.channel.send_event(EVENT_STOPPED, + reason=reason, threadId=self.thread_id, allThreadsStopped=True) - + def wait_for_client(self): """Wait for client to initialize.""" # This is a simplified version - in a real implementation # we might want to wait for specific initialization steps - pass - + def trigger_breakpoint(self): """Trigger a manual breakpoint.""" if self.initialized: self._send_stopped_event(STOP_REASON_BREAKPOINT) - + def debug_this_thread(self): """Enable debugging for current thread.""" if hasattr(sys, 'settrace'): sys.settrace(self._trace_function) - + def is_connected(self): """Check if client is connected.""" return self.connected and not self.channel.closed - + def disconnect(self): """Disconnect from client.""" self.connected = False diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index 524574b2b..b40bcb35e 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -16,7 +16,7 @@ class PdbAdapter: """Adapter between DAP protocol and MicroPython's sys.settrace functionality.""" - + def __init__(self): self.breakpoints = {} # filename -> {line_no: breakpoint_info} self.current_frame = None @@ -28,12 +28,12 @@ def __init__(self): self.variables_cache = {} # frameId -> variables self.frame_id_counter = 1 self.path_mapping = {} # runtime_path -> vscode_path mapping - + def _debug_print(self, message): """Print debug message only if debug logging is enabled.""" if hasattr(self, '_debug_session') and self._debug_session.debug_logging: # type: ignore print(message) - + def _normalize_path(self, path): """Normalize a file path for consistent comparisons.""" # Convert to absolute path if possible @@ -44,26 +44,26 @@ def _normalize_path(self, path): path = os.path.realpath(path) except: pass - + # Ensure consistent separators path = path.replace('\\', '/') return path - + def set_trace_function(self, trace_func): """Install the trace function.""" if hasattr(sys, 'settrace'): sys.settrace(trace_func) else: raise RuntimeError("sys.settrace not available") - + def set_breakpoints(self, filename, breakpoints): """Set breakpoints for a file.""" self.breakpoints[filename] = {} actual_breakpoints = [] - + # Debug log the breakpoint path self._debug_print(f"[PDB] Setting breakpoints for file: {filename}") - + for bp in breakpoints: line = bp.get("line") if line: @@ -77,23 +77,23 @@ def set_breakpoints(self, filename, breakpoints): "verified": True, "source": {"path": filename} }) - + return actual_breakpoints - + def should_stop(self, frame, event, arg): """Determine if execution should stop at this point.""" self.current_frame = frame self.hit_breakpoint = False - + # Get frame information filename = frame.f_code.co_filename lineno = frame.f_lineno - + # Debug: print filename and line for debugging if event == TRACE_LINE and lineno in [20, 21, 22, 23, 24]: # Only log lines near our breakpoints self._debug_print(f"[PDB] Checking {filename}:{lineno} (event={event})") self._debug_print(f"[PDB] Available breakpoint files: {list(self.breakpoints.keys())}") - + # Check for exact filename match first if filename in self.breakpoints: if lineno in self.breakpoints[filename]: @@ -102,11 +102,11 @@ def should_stop(self, frame, event, arg): self.path_mapping[filename] = filename self.hit_breakpoint = True return True - + # Also try checking by basename for path mismatches def basename(path): return path.split('/')[-1] if '/' in path else path - + # Check if this might be a relative path match def ends_with_path(full_path, relative_path): """Check if full_path ends with relative_path components.""" @@ -129,7 +129,7 @@ def ends_with_path(full_path, relative_path): self.path_mapping[filename] = bp_file self.hit_breakpoint = True return True - + # Also check if the runtime path might be relative and the breakpoint path absolute if ends_with_path(bp_file, filename): self._debug_print(f"[PDB] Relative path match: {bp_file} ends with {filename}") @@ -139,13 +139,13 @@ def ends_with_path(full_path, relative_path): self.path_mapping[filename] = bp_file self.hit_breakpoint = True return True - + # Check stepping if self.step_mode == 'into': if event in (TRACE_CALL, TRACE_LINE): self.step_mode = None return True - + elif self.step_mode == 'over': if event == TRACE_LINE and frame == self.step_frame: self.step_mode = None @@ -156,47 +156,46 @@ def ends_with_path(full_path, relative_path): self.step_frame = frame.f_back else: self.step_mode = None - + elif self.step_mode == 'out': if event == TRACE_RETURN and frame == self.step_frame: self.step_mode = None return True - + return False - + def continue_execution(self): """Continue execution.""" self.step_mode = None self.continue_event = True - + def step_over(self): """Step over (next line).""" self.step_mode = 'over' self.step_frame = self.current_frame self.continue_event = True - + def step_into(self): """Step into function calls.""" self.step_mode = 'into' self.continue_event = True - + def step_out(self): """Step out of current function.""" self.step_mode = 'out' self.step_frame = self.current_frame self.continue_event = True - + def pause(self): """Pause execution at next opportunity.""" # This is handled by the debug session - pass - + def wait_for_continue(self): """Wait for continue command (simplified implementation).""" # In a real implementation, this would block until continue # For MicroPython, we'll use a simple polling approach self.continue_event = False - + # Process DAP messages while waiting for continue self._debug_print("[PDB] Waiting for continue command...") while not self.continue_event: @@ -204,16 +203,16 @@ def wait_for_continue(self): if hasattr(self, '_debug_session'): self._debug_session.process_pending_messages() # type: ignore time.sleep(0.01) - + def get_stack_trace(self): """Get the current stack trace.""" if not self.current_frame: return [] - + frames = [] frame = self.current_frame frame_id = 0 - + while frame: filename = frame.f_code.co_filename name = frame.f_code.co_name @@ -238,10 +237,10 @@ def get_stack_trace(self): "endColumn": 1, "presentationHint": hint }) - + # Cache frame for variable access self.variables_cache[frame_id] = frame - + # MicroPython doesn't have f_back attribute if hasattr(frame, 'f_back'): frame = frame.f_back @@ -249,9 +248,9 @@ def get_stack_trace(self): # Only return the current frame for MicroPython break frame_id += 1 - + return frames - + def get_scopes(self, frame_id): """Get variable scopes for a frame.""" scopes = [ @@ -261,13 +260,13 @@ def get_scopes(self, frame_id): "expensive": False }, { - "name": "Globals", + "name": "Globals", "variablesReference": frame_id * 1000 + VARREF_GLOBALS , "expensive": False } ] return scopes - + def _process_special_variables(self, var_dict): """Process special variables (those starting and ending with __).""" variables = [] @@ -309,7 +308,7 @@ def _process_regular_variables(self, var_dict): @staticmethod def _var_error(name:str): return {"name": name, "value": "", "type": "unknown", "variablesReference": 0 } - + @staticmethod def _special_vars(varref:int): return {"name": "Special", "value": "", "variablesReference": varref} @@ -318,12 +317,12 @@ def get_variables(self, variables_ref): """Get variables for a scope.""" frame_id = variables_ref // 1000 scope_type = variables_ref % 1000 - + if frame_id not in self.variables_cache: return [] - + frame = self.variables_cache[frame_id] - + # Handle special scope types first if scope_type == VARREF_LOCALS_SPECIAL: var_dict = frame.f_locals if hasattr(frame, 'f_locals') else {} @@ -331,7 +330,7 @@ def get_variables(self, variables_ref): elif scope_type == VARREF_GLOBALS_SPECIAL: var_dict = frame.f_globals if hasattr(frame, 'f_globals') else {} return self._process_special_variables(var_dict) - + # Handle regular scope types with special folder variables = [] if scope_type == VARREF_LOCALS: @@ -343,7 +342,7 @@ def get_variables(self, variables_ref): else: # Invalid reference, return empty return [] - + # Add regular variables variables.extend(self._process_regular_variables(var_dict)) return variables diff --git a/python-ecosys/debugpy/demo.py b/python-ecosys/debugpy/demo.py index d5b3d0923..02a927257 100644 --- a/python-ecosys/debugpy/demo.py +++ b/python-ecosys/debugpy/demo.py @@ -16,10 +16,10 @@ def main(): print("MicroPython debugpy Demo") print("========================") print() - + # Demonstrate trace functionality print("1. Testing trace functionality:") - + def trace_function(frame, event, arg): if event == 'call': print(f" -> Entering function: {frame.f_code.co_name}") @@ -28,34 +28,34 @@ def trace_function(frame, event, arg): elif event == 'return': print(f" -> Returning from {frame.f_code.co_name} with value: {arg}") return trace_function - + # Enable tracing sys.settrace(trace_function) - + # Execute traced function result = simple_function(5, 3) - + # Disable tracing sys.settrace(None) - + print(f"Result: {result}") print() - + # Demonstrate debugpy components print("2. Testing debugpy components:") - + # Test PDB adapter from debugpy.server.pdb_adapter import PdbAdapter pdb = PdbAdapter() - + # Set some mock breakpoints breakpoints = pdb.set_breakpoints("demo.py", [{"line": 10}, {"line": 15}]) print(f" Set breakpoints: {len(breakpoints)} breakpoints") - + # Test messaging from debugpy.common.messaging import JsonMessageChannel print(" JsonMessageChannel available") - + print() print("3. debugpy is ready for VS Code integration!") print(" To use with VS Code:") @@ -63,6 +63,6 @@ def trace_function(frame, event, arg): print(" - Call debugpy.listen() to start the debug server") print(" - Connect VS Code using the 'Attach to MicroPython' configuration") print(" - Set breakpoints and debug normally") - + if __name__ == "__main__": main() diff --git a/python-ecosys/debugpy/test_vscode.py b/python-ecosys/debugpy/test_vscode.py index 2dca82d34..9a5672822 100644 --- a/python-ecosys/debugpy/test_vscode.py +++ b/python-ecosys/debugpy/test_vscode.py @@ -23,7 +23,7 @@ def debuggable_code(): """The actual code we want to debug - wrapped in a function so sys.settrace will trace it.""" global foo print("Starting debuggable code...") - + # Test data - set breakpoint here (using smaller numbers to avoid slow fibonacci) numbers = [3, 4, 5] for i, num in enumerate(numbers): @@ -34,18 +34,18 @@ def debuggable_code(): print(sys.implementation) import machine print(dir(machine)) - + # Test manual breakpoint print("\nTriggering manual breakpoint...") debugpy.breakpoint() print("Manual breakpoint triggered!") - + print("Test completed successfully!") def main(): print("MicroPython VS Code Debugging Test") print("==================================") - + # Start debug server try: debugpy.listen() @@ -57,22 +57,22 @@ def main(): # input() # except: # pass - + # Enable debugging for this thread debugpy.debug_this_thread() - + # Give VS Code a moment to set breakpoints after attach print("\nGiving VS Code time to set breakpoints...") import time time.sleep(2) - + # Call the debuggable code function so it gets traced debuggable_code() - + except KeyboardInterrupt: print("\nTest interrupted by user") except Exception as e: print(f"Error: {e}") if __name__ == "__main__": - main() \ No newline at end of file + main() From 6e3f207a9b6808ceb86b8f29c8d6b05a85635dcc Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Tue, 24 Jun 2025 16:03:52 +0200 Subject: [PATCH 31/74] debugpy : Decode debugger IP address on connect. Signed-off-by: Jos Verlinde --- python-ecosys/debugpy/debugpy/public_api.py | 23 ++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/python-ecosys/debugpy/debugpy/public_api.py b/python-ecosys/debugpy/debugpy/public_api.py index 8642be989..c8f1363e9 100644 --- a/python-ecosys/debugpy/debugpy/public_api.py +++ b/python-ecosys/debugpy/debugpy/public_api.py @@ -1,6 +1,7 @@ """Public API for debugpy.""" import socket +import struct import sys from .common.constants import DEFAULT_HOST, DEFAULT_PORT from .server.debug_session import DebugSession @@ -43,7 +44,7 @@ def listen(port=DEFAULT_PORT, host=DEFAULT_HOST): client_sock = None try: client_sock, client_addr = listener.accept() - print(f"Debugger connected from {client_addr}") + print(f"Debugger connected from {format_client_addr(client_addr)}") # Create debug session _debug_session = DebugSession(client_sock) @@ -73,6 +74,26 @@ def listen(port=DEFAULT_PORT, host=DEFAULT_HOST): return (host, port) +def format_client_addr(client_addr): + """Format client address using socket module methods""" + if isinstance(client_addr, (tuple, list)): + # Already in (ip, port) format + return f"{client_addr[0]}:{client_addr[1]}" + elif isinstance(client_addr, bytes) and len(client_addr) >= 8: + # Extract port (bytes 2-4, network byte order) + port = struct.unpack('!H', client_addr[2:4])[0] + # Extract IP address (bytes 4-8) using inet_ntoa + ip_packed = client_addr[4:8] + try: + # inet_ntoa expects 4-byte string in network byte order + ip_addr = socket.inet_ntoa(ip_packed) + return f"{ip_addr}:{port}" + except: + # Fallback if inet_ntoa not available (MicroPython) + ip_addr = '.'.join(str(b) for b in ip_packed) + return f"{ip_addr}:{port}" + else: + return str(client_addr) def wait_for_client(): """Wait for the debugger client to connect and initialize.""" From 7240308ba950ad97a1858d3ced1daa349300f2fb Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Sun, 29 Jun 2025 23:48:06 +0200 Subject: [PATCH 32/74] debugpy: Add type hints and improve path mapping logic in PDB adapter. Signed-off-by: Jos Verlinde --- .../debugpy/debugpy/server/debug_session.py | 2 +- .../debugpy/debugpy/server/pdb_adapter.py | 47 ++++++++----------- 2 files changed, 21 insertions(+), 28 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index 43e2d442c..fe0784f77 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -20,7 +20,7 @@ def __init__(self, client_socket): self.debug_logging = False # Initialize first self.channel = JsonMessageChannel(client_socket, self._debug_print) self.pdb = PdbAdapter() - self.pdb._debug_session = self # Allow PDB to process messages during wait + self.pdb._debug_session = self # Allow PDB to process messages during wait # type: ignore self.initialized = False self.connected = True self.thread_id = 1 # Simple single-thread model diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index b40bcb35e..ab485ff06 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -14,6 +14,19 @@ VARREF_GLOBALS_SPECIAL = 4 +# Also try checking by basename for path mismatches +def basename(path:str): + return path.split('/')[-1] if '/' in path else path + +# Check if this might be a relative path match +def ends_with_path(full_path:str, relative_path:str): + """Check if full_path ends with relative_path components.""" + full_parts = full_path.replace('\\', '/').split('/') + rel_parts = relative_path.replace('\\', '/').split('/') + if len(rel_parts) > len(full_parts): + return False + return full_parts[-len(rel_parts):] == rel_parts + class PdbAdapter: """Adapter between DAP protocol and MicroPython's sys.settrace functionality.""" @@ -27,14 +40,14 @@ def __init__(self): self.continue_event = False self.variables_cache = {} # frameId -> variables self.frame_id_counter = 1 - self.path_mapping = {} # runtime_path -> vscode_path mapping + self.path_mappings : dict[str,str] = {} # runtime_path -> vscode_path mapping def _debug_print(self, message): """Print debug message only if debug logging is enabled.""" if hasattr(self, '_debug_session') and self._debug_session.debug_logging: # type: ignore print(message) - def _normalize_path(self, path): + def _normalize_path(self, path:str): """Normalize a file path for consistent comparisons.""" # Convert to absolute path if possible try: @@ -44,7 +57,6 @@ def _normalize_path(self, path): path = os.path.realpath(path) except: pass - # Ensure consistent separators path = path.replace('\\', '/') return path @@ -80,7 +92,7 @@ def set_breakpoints(self, filename, breakpoints): return actual_breakpoints - def should_stop(self, frame, event, arg): + def should_stop(self, frame, event:str, arg): """Determine if execution should stop at this point.""" self.current_frame = frame self.hit_breakpoint = False @@ -88,34 +100,15 @@ def should_stop(self, frame, event, arg): # Get frame information filename = frame.f_code.co_filename lineno = frame.f_lineno - - # Debug: print filename and line for debugging - if event == TRACE_LINE and lineno in [20, 21, 22, 23, 24]: # Only log lines near our breakpoints - self._debug_print(f"[PDB] Checking {filename}:{lineno} (event={event})") - self._debug_print(f"[PDB] Available breakpoint files: {list(self.breakpoints.keys())}") - # Check for exact filename match first if filename in self.breakpoints: if lineno in self.breakpoints[filename]: self._debug_print(f"[PDB] HIT BREAKPOINT (exact match) at {filename}:{lineno}") # Record the path mapping (in this case, they're already the same) - self.path_mapping[filename] = filename + self.path_mappings[filename] = filename self.hit_breakpoint = True return True - # Also try checking by basename for path mismatches - def basename(path): - return path.split('/')[-1] if '/' in path else path - - # Check if this might be a relative path match - def ends_with_path(full_path, relative_path): - """Check if full_path ends with relative_path components.""" - full_parts = full_path.replace('\\', '/').split('/') - rel_parts = relative_path.replace('\\', '/').split('/') - if len(rel_parts) > len(full_parts): - return False - return full_parts[-len(rel_parts):] == rel_parts - file_basename = basename(filename) self._debug_print(f"[PDB] Fallback basename match: '{file_basename}' vs available files") for bp_file in self.breakpoints: @@ -126,7 +119,7 @@ def ends_with_path(full_path, relative_path): if lineno in self.breakpoints[bp_file]: self._debug_print(f"[PDB] HIT BREAKPOINT (fallback basename match) at {filename}:{lineno} -> {bp_file}") # Record the path mapping so we can report the correct path in stack traces - self.path_mapping[filename] = bp_file + self.path_mappings[filename] = bp_file self.hit_breakpoint = True return True @@ -136,7 +129,7 @@ def ends_with_path(full_path, relative_path): if lineno in self.breakpoints[bp_file]: self._debug_print(f"[PDB] HIT BREAKPOINT (relative path match) at {filename}:{lineno} -> {bp_file}") # Record the path mapping so we can report the correct path in stack traces - self.path_mapping[filename] = bp_file + self.path_mappings[filename] = bp_file self.hit_breakpoint = True return True @@ -223,7 +216,7 @@ def get_stack_trace(self): hint = 'normal' # Use the VS Code path if we have a mapping, otherwise use the original path - display_path = self.path_mapping.get(filename, filename) + display_path = self.path_mappings.get(filename, filename) if filename != display_path: self._debug_print(f"[PDB] Stack trace path mapping: {filename} -> {display_path}") # Create StackFrame info From 69cd729e7448fc42ff738e72a38a4ffd8ff628b6 Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Mon, 30 Jun 2025 23:42:38 +0200 Subject: [PATCH 33/74] debugpy: Add complex variable handling and caching. Signed-off-by: Jos Verlinde --- .../debugpy/debugpy/server/pdb_adapter.py | 396 +++++++++++++----- 1 file changed, 298 insertions(+), 98 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index cd3354d8c..11b4dfb81 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -4,34 +4,92 @@ import time import os import json + +Any = object from ..common.constants import ( - TRACE_CALL, TRACE_LINE, TRACE_RETURN, TRACE_EXCEPTION, - SCOPE_LOCALS, SCOPE_GLOBALS + TRACE_CALL, + TRACE_LINE, + TRACE_RETURN, + TRACE_EXCEPTION, + SCOPE_LOCALS, + SCOPE_GLOBALS, ) + VARREF_LOCALS = 1 VARREF_GLOBALS = 2 VARREF_LOCALS_SPECIAL = 3 VARREF_GLOBALS_SPECIAL = 4 +# New constants for complex variable references +VARREF_COMPLEX_BASE = 10000 # Base for complex variable references +MAX_CACHE_SIZE = 50 # Limit cache size for memory constraints + + +class VariableReferenceCache: + """Lightweight cache for complex variable references optimized for MicroPython.""" + + def __init__(self, max_size: int = MAX_CACHE_SIZE): + self.cache: dict[int, Any] = {} + self.insertion_order: list[int] = [] # Track insertion order for proper FIFO + self.next_ref: int = VARREF_COMPLEX_BASE + self.max_size: int = max_size + + def add_variable(self, value: Any) -> int: + """Add a complex variable and return its reference ID.""" + # Clean cache if approaching limit + if len(self.cache) >= self.max_size: + self._cleanup_oldest() + + ref_id = self.next_ref + self.cache[ref_id] = value + self.insertion_order.append(ref_id) + self.next_ref += 1 + return ref_id + + def get_variable(self, ref_id: int): # -> Optional[Any] + """Get variable by reference ID.""" + return self.cache.get(ref_id) + + def _cleanup_oldest(self) -> None: + """Remove oldest entries to free memory.""" + if self.cache and self.insertion_order: + # Remove first quarter of entries (true FIFO based on insertion order) + to_remove = max(1, len(self.cache) // 4) # Remove at least 1 entry + keys_to_remove = self.insertion_order[:to_remove] + for key in keys_to_remove: + if key in self.cache: + del self.cache[key] + # Update insertion order + self.insertion_order = self.insertion_order[to_remove:] + + def clear(self) -> None: + """Clear all cached variables.""" + self.cache.clear() + self.insertion_order.clear() + # Also try checking by basename for path mismatches -def basename(path:str): - return path.split('/')[-1] if '/' in path else path +def basename(path: str): + return path.split("/")[-1] if "/" in path else path + # Check if this might be a relative path match -def ends_with_path(full_path:str, relative_path:str): +def ends_with_path(full_path: str, relative_path: str): """Check if full_path ends with relative_path components.""" - full_parts = full_path.replace('\\', '/').split('/') - rel_parts = relative_path.replace('\\', '/').split('/') + full_parts = full_path.replace("\\", "/").split("/") + rel_parts = relative_path.replace("\\", "/").split("/") if len(rel_parts) > len(full_parts): return False - return full_parts[-len(rel_parts):] == rel_parts + return full_parts[-len(rel_parts) :] == rel_parts + class PdbAdapter: """Adapter between DAP protocol and MicroPython's sys.settrace functionality.""" def __init__(self): - self.breakpoints : dict[str,dict[int,dict]] = {} # filename -> {line_no: breakpoint_info} # todo - simplify - reduce info stored + self.breakpoints: dict[ + str, dict[int, dict] + ] = {} # filename -> {line_no: breakpoint_info} # todo - simplify - reduce info stored self.current_frame = None self.step_mode = None # None, 'over', 'into', 'out' self.step_frame = None @@ -39,37 +97,42 @@ def __init__(self): self.hit_breakpoint = False self.continue_event = False self.variables_cache = {} # frameId -> variables + self.var_cache = VariableReferenceCache() # Enhanced variable reference cache self.frame_id_counter = 1 - self.path_mappings : list[tuple[str,str]] = [] # runtime_path -> vscode_path mapping # todo: move to session level - self.file_mappings : dict[str,str] = {} # runtime_path -> vscode_path mapping # todo : merge with .breakpoints + self.path_mappings: list[ + tuple[str, str] + ] = [] # runtime_path -> vscode_path mapping # todo: move to session level + self.file_mappings: dict[ + str, str + ] = {} # runtime_path -> vscode_path mapping # todo : merge with .breakpoints def _debug_print(self, message): """Print debug message only if debug logging is enabled.""" - if hasattr(self, '_debug_session') and self._debug_session.debug_logging: # type: ignore + if hasattr(self, "_debug_session") and self._debug_session.debug_logging: # type: ignore print(message) - def _normalize_path(self, path:str): + def _normalize_path(self, path: str): """Normalize a file path for consistent comparisons.""" # Convert to absolute path if possible try: - if hasattr(os.path, 'abspath'): + if hasattr(os.path, "abspath"): path = os.path.abspath(path) - elif hasattr(os.path, 'realpath'): + elif hasattr(os.path, "realpath"): path = os.path.realpath(path) except: pass # Ensure consistent separators - path = path.replace('\\', '/') + path = path.replace("\\", "/") return path def set_trace_function(self, trace_func): """Install the trace function.""" - if hasattr(sys, 'settrace'): + if hasattr(sys, "settrace"): sys.settrace(trace_func) else: raise RuntimeError("sys.settrace not available") - def _filename_as_debugee(self, path:str): + def _filename_as_debugee(self, path: str): # check if we have a 1:1 file mapping for this path if self.file_mappings.get(path): return self.file_mappings[path] @@ -77,17 +140,17 @@ def _filename_as_debugee(self, path:str): for runtime_path, vscode_path in self.path_mappings: if path.startswith(vscode_path): path = path.replace(vscode_path, runtime_path, 1) - if path.startswith('//'): + if path.startswith("//"): path = path[1:] # If no mapping found, return the original path return path - - def _filename_as_debugger(self, path:str): + + def _filename_as_debugger(self, path: str): """Convert a file path to the debugger's expected format.""" path = path or "" if not path: return path - if path.startswith('<'): + if path.startswith("<"): # Special case for or similar return path # Check if we have a 1:1 file mapping for this path @@ -100,12 +163,12 @@ def _filename_as_debugger(self, path:str): for runtime_path, vscode_path in self.path_mappings: if path.startswith(runtime_path): path = path.replace(runtime_path, vscode_path, 1) - if path.startswith('//'): + if path.startswith("//"): path = path[1:] # If no mapping found, return the original path return path - def set_breakpoints(self, filename:str, breakpoints:list[dict]): + def set_breakpoints(self, filename: str, breakpoints: list[dict]): """Set breakpoints for a file.""" self.breakpoints[filename] = {} local_name = self._filename_as_debugee(filename) @@ -122,24 +185,22 @@ def set_breakpoints(self, filename:str, breakpoints:list[dict]): self.breakpoints[local_name][line] = { "line": line, "verified": True, - "source": {"path": filename} + "source": {"path": filename}, } self.breakpoints[filename][line] = { "line": line, "verified": True, - "source": {"path": filename} + "source": {"path": filename}, } - actual_breakpoints.append({ - "line": line, - "verified": True, - "source": {"path": filename} - }) + actual_breakpoints.append( + {"line": line, "verified": True, "source": {"path": filename}} + ) self._debug_print(f"[PDB] Breakpoints set : {self.breakpoints}") return actual_breakpoints - def should_stop(self, frame, event:str, arg): + def should_stop(self, frame, event: str, arg): """Determine if execution should stop at this point.""" self.current_frame = frame self.hit_breakpoint = False @@ -163,26 +224,28 @@ def should_stop(self, frame, event:str, arg): self.breakpoints[filename] = {} # Ensure the filename is in the breakpoints dict if not filename in self.file_mappings: self.file_mappings[filename] = self._filename_as_debugger(filename) - self._debug_print(f"[PDB] add mapping for :'{filename}' -> '{self.file_mappings[filename]}'") + self._debug_print( + f"[PDB] add mapping for :'{filename}' -> '{self.file_mappings[filename]}'" + ) # Check stepping - if self.step_mode == 'into': + if self.step_mode == "into": if event in (TRACE_CALL, TRACE_LINE): self.step_mode = None return True - elif self.step_mode == 'over': + elif self.step_mode == "over": if event == TRACE_LINE and frame == self.step_frame: self.step_mode = None return True elif event == TRACE_RETURN and frame == self.step_frame: # Continue stepping in caller - if hasattr(frame, 'f_back') and frame.f_back: + if hasattr(frame, "f_back") and frame.f_back: self.step_frame = frame.f_back else: self.step_mode = None - elif self.step_mode == 'out': + elif self.step_mode == "out": if event == TRACE_RETURN and frame == self.step_frame: self.step_mode = None return True @@ -196,18 +259,18 @@ def continue_execution(self): def step_over(self): """Step over (next line).""" - self.step_mode = 'over' + self.step_mode = "over" self.step_frame = self.current_frame self.continue_event = True def step_into(self): """Step into function calls.""" - self.step_mode = 'into' + self.step_mode = "into" self.continue_event = True def step_out(self): """Step out of current function.""" - self.step_mode = 'out' + self.step_mode = "out" self.step_frame = self.current_frame self.continue_event = True @@ -225,8 +288,8 @@ def wait_for_continue(self): self._debug_print("[PDB] Waiting for continue command...") while not self.continue_event: # Process any pending DAP messages (scopes, variables, etc.) - if hasattr(self, '_debug_session'): - self._debug_session.process_pending_messages() # type: ignore + if hasattr(self, "_debug_session"): + self._debug_session.process_pending_messages() # type: ignore time.sleep(0.01) def get_stack_trace(self): @@ -242,10 +305,10 @@ def get_stack_trace(self): filename = frame.f_code.co_filename name = frame.f_code.co_name line = frame.f_lineno - if "" in filename or filename.endswith("debugpy.py") : - hint = 'subtle' - else : - hint = 'normal' + if "" in filename or filename.endswith("debugpy.py"): + hint = "subtle" + else: + hint = "normal" # self._debug_print("=" * 40 ) # self._debug_print(f"[PDB] file mappings: {repr(self.file_mappings)} " ) @@ -257,22 +320,24 @@ def get_stack_trace(self): if filename != debugger_path: self._debug_print(f"[PDB] Stack trace path mapping: {filename} -> {debugger_path}") # Create StackFrame info - frames.append({ - "id": frame_id, - "name": name, - "source": {"path": debugger_path}, - "line": line, - "column": 1, - "endLine": line, - "endColumn": 1, - "presentationHint": hint - }) + frames.append( + { + "id": frame_id, + "name": name, + "source": {"path": debugger_path}, + "line": line, + "column": 1, + "endLine": line, + "endColumn": 1, + "presentationHint": hint, + } + ) # Cache frame for variable access self.variables_cache[frame_id] = frame # MicroPython doesn't have f_back attribute - if hasattr(frame, 'f_back'): + if hasattr(frame, "f_back"): frame = frame.f_back else: # Only return the current frame for MicroPython @@ -285,15 +350,15 @@ def get_scopes(self, frame_id): """Get variable scopes for a frame.""" scopes = [ { - "name": "Locals", + "name": SCOPE_LOCALS, "variablesReference": frame_id * 1000 + VARREF_LOCALS, - "expensive": False + "expensive": False, }, { - "name": "Globals", - "variablesReference": frame_id * 1000 + VARREF_GLOBALS , - "expensive": False - } + "name": SCOPE_GLOBALS, + "variablesReference": frame_id * 1000 + VARREF_GLOBALS, + "expensive": False, + }, ] return scopes @@ -301,16 +366,18 @@ def _process_special_variables(self, var_dict): """Process special variables (those starting and ending with __).""" variables = [] for name, value in var_dict.items(): - if name.startswith('__') and name.endswith('__'): + if name.startswith("__") and name.endswith("__"): try: value_str = json.dumps(value) type_str = type(value).__name__ - variables.append({ - "name": name, - "value": value_str, - "type": type_str, - "variablesReference": 0 - }) + variables.append( + { + "name": name, + "value": value_str, + "type": type_str, + "variablesReference": 0, + } + ) except Exception: variables.append(self._var_error(name)) return variables @@ -320,31 +387,163 @@ def _process_regular_variables(self, var_dict): variables = [] for name, value in var_dict.items(): # Skip private/internal variables - if name.startswith('__') and name.endswith('__'): + if name.startswith("__") and name.endswith("__"): continue + variables.append(self._get_variable_info(name, value)) + return variables + + def _is_expandable(self, value: Any) -> bool: + """Check if a variable can be expanded (has child elements).""" + return isinstance(value, (dict, list, tuple, set)) + + def _get_preview(self, value: Any, fallback_text: str = "") -> str: + """Get a truncated preview of a variable value.""" + try: + if value is None: + return "None" + + # Try to get a meaningful representation + preview_repr = repr(value) + if len(preview_repr) > 30: + return preview_repr[:30] + "..." + else: + return preview_repr + except (TypeError, ValueError): + # If repr fails, try str try: - value_str = json.dumps(value) - type_str = type(value).__name__ - variables.append({ - "name": name, - "value": value_str, - "type": type_str, - "variablesReference": 0 - }) - except Exception: - variables.append(self._var_error(name)) + preview_str = str(value) + if len(preview_str) > 30: + return preview_str[:30] + "..." + else: + return preview_str + except: + # Final fallback + return fallback_text or f"<{type(value).__name__} object>" + + def _get_variable_info(self, name: str, value: Any) -> dict[str, str | int]: + """Get DAP-compliant variable information with proper type handling.""" + try: + # Handle expandable types + if self._is_expandable(value): + var_ref = self.var_cache.add_variable(value) + + if isinstance(value, dict): + preview = ( + self._get_preview(value, f"dict({len(value)} items)") + if value + else "dict(empty)" + ) + return { + "name": name, + "value": preview, + "type": "dict", + "variablesReference": var_ref, + "namedVariables": len(value), + "indexedVariables": 0, + } + elif isinstance(value, list): + preview = ( + self._get_preview(value, f"list({len(value)} items)") + if value + else "list(empty)" + ) + return { + "name": name, + "value": preview, + "type": "list", + "variablesReference": var_ref, + "indexedVariables": len(value), + "namedVariables": 0, + } + elif isinstance(value, tuple): + preview = ( + self._get_preview(value, f"tuple({len(value)} items)") + if value + else "tuple(empty)" + ) + return { + "name": name, + "value": preview, + "type": "tuple", + "variablesReference": var_ref, + "indexedVariables": len(value), + "namedVariables": 0, + } + elif isinstance(value, set): + preview = ( + self._get_preview(value, f"set({len(value)} items)") + if value + else "set(empty)" + ) + return { + "name": name, + "value": preview, + "type": "set", + "variablesReference": var_ref, + "indexedVariables": len(value), + "namedVariables": 0, + } + + # Simple types - use the preview helper + preview = self._get_preview(value) + + return { + "name": name, + "value": preview, + "type": type(value).__name__, + "variablesReference": 0, + } + except Exception: + return self._var_error(name) + + def _expand_complex_variable(self, ref_id: int) -> list[dict[str, str | int]]: + """Expand a complex variable into its child elements.""" + value = self.var_cache.get_variable(ref_id) + if value is None: + return [] + + variables = [] + try: + if isinstance(value, dict): + # Handle dictionary keys and values + for key, val in value.items(): + key_str = str(key) + variables.append(self._get_variable_info(key_str, val)) + elif isinstance(value, (list, tuple)): + # Handle list/tuple elements + for i, val in enumerate(value): + variables.append(self._get_variable_info(f"[{i}]", val)) + elif isinstance(value, set): + # Handle set elements (sorted for consistent display) + for i, val in enumerate(sorted(value, key=lambda x: str(x))): + variables.append(self._get_variable_info(f"<{i}>", val)) + except Exception as e: + # Return error info for debugging + variables.append( + { + "name": "error", + "value": f"Failed to expand: {e}", + "type": "error", + "variablesReference": 0, + } + ) + return variables @staticmethod - def _var_error(name:str): - return {"name": name, "value": "", "type": "unknown", "variablesReference": 0 } + def _var_error(name: str): + return {"name": name, "value": "", "type": "unknown", "variablesReference": 0} @staticmethod - def _special_vars(varref:int): + def _special_vars(varref: int): return {"name": "Special", "value": "", "variablesReference": varref} def get_variables(self, variables_ref): - """Get variables for a scope.""" + """Get variables for a scope with enhanced complex variable support.""" + # Handle complex variable expansion + if variables_ref >= VARREF_COMPLEX_BASE: + return self._expand_complex_variable(variables_ref) + frame_id = variables_ref // 1000 scope_type = variables_ref % 1000 @@ -355,25 +554,25 @@ def get_variables(self, variables_ref): # Handle special scope types first if scope_type == VARREF_LOCALS_SPECIAL: - var_dict = frame.f_locals if hasattr(frame, 'f_locals') else {} + var_dict = frame.f_locals if hasattr(frame, "f_locals") else {} return self._process_special_variables(var_dict) elif scope_type == VARREF_GLOBALS_SPECIAL: - var_dict = frame.f_globals if hasattr(frame, 'f_globals') else {} + var_dict = frame.f_globals if hasattr(frame, "f_globals") else {} return self._process_special_variables(var_dict) # Handle regular scope types with special folder variables = [] if scope_type == VARREF_LOCALS: - var_dict = frame.f_locals if hasattr(frame, 'f_locals') else {} - variables.append(self._special_vars( VARREF_LOCALS_SPECIAL)) + var_dict = frame.f_locals if hasattr(frame, "f_locals") else {} + variables.append(self._special_vars(frame_id * 1000 + VARREF_LOCALS_SPECIAL)) elif scope_type == VARREF_GLOBALS: - var_dict = frame.f_globals if hasattr(frame, 'f_globals') else {} - variables.append(self._special_vars( VARREF_GLOBALS_SPECIAL)) + var_dict = frame.f_globals if hasattr(frame, "f_globals") else {} + variables.append(self._special_vars(frame_id * 1000 + VARREF_GLOBALS_SPECIAL)) else: # Invalid reference, return empty return [] - # Add regular variables + # Add regular variables with enhanced processing variables.extend(self._process_regular_variables(var_dict)) return variables @@ -381,14 +580,14 @@ def evaluate_expression(self, expression, frame_id=None): """Evaluate an expression in the context of a frame.""" if frame_id is not None and frame_id in self.variables_cache: frame = self.variables_cache[frame_id] - globals_dict = frame.f_globals if hasattr(frame, 'f_globals') else {} - locals_dict = frame.f_locals if hasattr(frame, 'f_locals') else {} + globals_dict = frame.f_globals if hasattr(frame, "f_globals") else {} + locals_dict = frame.f_locals if hasattr(frame, "f_locals") else {} else: # Use current frame frame = self.current_frame if frame: - globals_dict = frame.f_globals if hasattr(frame, 'f_globals') else {} - locals_dict = frame.f_locals if hasattr(frame, 'f_locals') else {} + globals_dict = frame.f_globals if hasattr(frame, "f_globals") else {} + locals_dict = frame.f_locals if hasattr(frame, "f_locals") else {} else: globals_dict = globals() locals_dict = {} @@ -400,8 +599,9 @@ def evaluate_expression(self, expression, frame_id=None): raise Exception(f"Evaluation error: {e}") def cleanup(self): - """Clean up resources.""" + """Clean up resources with enhanced cache management.""" self.variables_cache.clear() + self.var_cache.clear() # Clear variable reference cache self.breakpoints.clear() - if hasattr(sys, 'settrace'): + if hasattr(sys, "settrace"): sys.settrace(None) From ec4dc982479a466611ab75fe3281fd89f2a237b8 Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Sun, 29 Jun 2025 23:58:00 +0200 Subject: [PATCH 34/74] debugpy: Enhance path mapping handling in PDB adapter and debug session. Store both folder mappings from the debugger, and 1:1 file mappings . Signed-off-by: Jos Verlinde --- .../debugpy/debugpy/server/debug_session.py | 9 +++++++++ python-ecosys/debugpy/debugpy/server/pdb_adapter.py | 13 +++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index fe0784f77..a2a00c170 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -252,6 +252,15 @@ def _handle_attach(self, seq, args): self._debug_print(f"[DAP] Processing attach request with args: {args}") print(f"[DAP] Debug logging {'enabled' if self.debug_logging else 'disabled'} (logToFile={self.debug_logging})") + + # get debugger root and debugee root from pathMappings + for pm in args.get("pathMappings",[]): + # debugee - debugger + self.pdb.path_mappings.append( + (pm.get("remoteRoot", "./"), + pm.get("localRoot", "./")) + ) + # # TODO: justMyCode, debugOptions , # Enable trace function self.pdb.set_trace_function(self._trace_function) diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index ab485ff06..a5ab3d2ce 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -40,7 +40,8 @@ def __init__(self): self.continue_event = False self.variables_cache = {} # frameId -> variables self.frame_id_counter = 1 - self.path_mappings : dict[str,str] = {} # runtime_path -> vscode_path mapping + self.path_mappings : list[tuple[str,str]] = [] # runtime_path -> vscode_path mapping + self.file_mappings : dict[str,str] = {} # runtime_path -> vscode_path mapping def _debug_print(self, message): """Print debug message only if debug logging is enabled.""" @@ -68,7 +69,7 @@ def set_trace_function(self, trace_func): else: raise RuntimeError("sys.settrace not available") - def set_breakpoints(self, filename, breakpoints): + def set_breakpoints(self, filename, breakpoints:list[dict]): """Set breakpoints for a file.""" self.breakpoints[filename] = {} actual_breakpoints = [] @@ -105,7 +106,7 @@ def should_stop(self, frame, event:str, arg): if lineno in self.breakpoints[filename]: self._debug_print(f"[PDB] HIT BREAKPOINT (exact match) at {filename}:{lineno}") # Record the path mapping (in this case, they're already the same) - self.path_mappings[filename] = filename + self.file_mappings[filename] = filename self.hit_breakpoint = True return True @@ -119,7 +120,7 @@ def should_stop(self, frame, event:str, arg): if lineno in self.breakpoints[bp_file]: self._debug_print(f"[PDB] HIT BREAKPOINT (fallback basename match) at {filename}:{lineno} -> {bp_file}") # Record the path mapping so we can report the correct path in stack traces - self.path_mappings[filename] = bp_file + self.file_mappings[filename] = bp_file self.hit_breakpoint = True return True @@ -129,7 +130,7 @@ def should_stop(self, frame, event:str, arg): if lineno in self.breakpoints[bp_file]: self._debug_print(f"[PDB] HIT BREAKPOINT (relative path match) at {filename}:{lineno} -> {bp_file}") # Record the path mapping so we can report the correct path in stack traces - self.path_mappings[filename] = bp_file + self.file_mappings[filename] = bp_file self.hit_breakpoint = True return True @@ -216,7 +217,7 @@ def get_stack_trace(self): hint = 'normal' # Use the VS Code path if we have a mapping, otherwise use the original path - display_path = self.path_mappings.get(filename, filename) + display_path = self.file_mappings.get(filename, filename) if filename != display_path: self._debug_print(f"[PDB] Stack trace path mapping: {filename} -> {display_path}") # Create StackFrame info From c19d6bf82aba7668e2591c80d2efc63f06fa692e Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Mon, 30 Jun 2025 23:43:26 +0200 Subject: [PATCH 35/74] debugpy: Format code. Signed-off-by: Jos Verlinde --- .../debugpy/debugpy/common/messaging.py | 14 +- python-ecosys/debugpy/debugpy/public_api.py | 17 +-- .../debugpy/debugpy/server/debug_session.py | 124 +++++++++++------- 3 files changed, 94 insertions(+), 61 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/common/messaging.py b/python-ecosys/debugpy/debugpy/common/messaging.py index 7a588bab3..a491578ad 100644 --- a/python-ecosys/debugpy/debugpy/common/messaging.py +++ b/python-ecosys/debugpy/debugpy/common/messaging.py @@ -64,7 +64,9 @@ def send_response(self, command, request_seq, success=True, body=None, message=N if message is not None: kwargs["message"] = message - self._debug_print(f"[DAP] SEND: response {command} (req_seq={request_seq}, success={success})") + self._debug_print( + f"[DAP] SEND: response {command} (req_seq={request_seq}, success={success})" + ) if body: self._debug_print(f"[DAP] body: {body}") if message: @@ -95,14 +97,14 @@ def recv_message(self): self._recv_buffer += data except OSError as e: # Handle timeout and other socket errors - if hasattr(e, 'errno') and e.errno in (11, 35): # EAGAIN, EWOULDBLOCK + if hasattr(e, "errno") and e.errno in (11, 35): # EAGAIN, EWOULDBLOCK return None # No data available self.closed = True return None header_end = self._recv_buffer.find(b"\r\n\r\n") header_str = self._recv_buffer[:header_end].decode("utf-8") - self._recv_buffer = self._recv_buffer[header_end + 4:] + self._recv_buffer = self._recv_buffer[header_end + 4 :] # Parse Content-Length content_length = 0 @@ -123,7 +125,7 @@ def recv_message(self): return None self._recv_buffer += data except OSError as e: - if hasattr(e, 'errno') and e.errno in (11, 35): # EAGAIN, EWOULDBLOCK + if hasattr(e, "errno") and e.errno in (11, 35): # EAGAIN, EWOULDBLOCK return None self.closed = True return None @@ -134,7 +136,9 @@ def recv_message(self): # Parse JSON try: message = json.loads(body.decode("utf-8")) - self._debug_print(f"[DAP] Successfully received message: {message.get('type')} {message.get('command', message.get('event', 'unknown'))}") + self._debug_print( + f"[DAP] Successfully received message: {message.get('type')} {message.get('command', message.get('event', 'unknown'))}" + ) return message except (ValueError, UnicodeDecodeError) as e: print(f"[DAP] JSON parse error: {e}") diff --git a/python-ecosys/debugpy/debugpy/public_api.py b/python-ecosys/debugpy/debugpy/public_api.py index c8f1363e9..06b928965 100644 --- a/python-ecosys/debugpy/debugpy/public_api.py +++ b/python-ecosys/debugpy/debugpy/public_api.py @@ -11,11 +11,11 @@ def listen(port=DEFAULT_PORT, host=DEFAULT_HOST): """Start listening for debugger connections. - + Args: port: Port number to listen on (default: 5678) host: Host address to bind to (default: "127.0.0.1") - + Returns: (host, port) tuple of the actual listening address """ @@ -52,7 +52,7 @@ def listen(port=DEFAULT_PORT, host=DEFAULT_HOST): # Handle just the initialize request, then return immediately print("[DAP] Waiting for initialize request...") init_message = _debug_session.channel.recv_message() - if init_message and init_message.get('command') == 'initialize': + if init_message and init_message.get("command") == "initialize": _debug_session._handle_message(init_message) print("[DAP] Initialize request handled - returning control immediately") else: @@ -74,6 +74,7 @@ def listen(port=DEFAULT_PORT, host=DEFAULT_HOST): return (host, port) + def format_client_addr(client_addr): """Format client address using socket module methods""" if isinstance(client_addr, (tuple, list)): @@ -81,7 +82,7 @@ def format_client_addr(client_addr): return f"{client_addr[0]}:{client_addr[1]}" elif isinstance(client_addr, bytes) and len(client_addr) >= 8: # Extract port (bytes 2-4, network byte order) - port = struct.unpack('!H', client_addr[2:4])[0] + port = struct.unpack("!H", client_addr[2:4])[0] # Extract IP address (bytes 4-8) using inet_ntoa ip_packed = client_addr[4:8] try: @@ -90,11 +91,12 @@ def format_client_addr(client_addr): return f"{ip_addr}:{port}" except: # Fallback if inet_ntoa not available (MicroPython) - ip_addr = '.'.join(str(b) for b in ip_packed) + ip_addr = ".".join(str(b) for b in ip_packed) return f"{ip_addr}:{port}" else: return str(client_addr) + def wait_for_client(): """Wait for the debugger client to connect and initialize.""" global _debug_session @@ -109,7 +111,7 @@ def breakpoint(): _debug_session.trigger_breakpoint() else: # Fallback to built-in breakpoint if available - if hasattr(__builtins__, 'breakpoint'): + if hasattr(__builtins__, "breakpoint"): __builtins__.breakpoint() @@ -120,7 +122,7 @@ def debug_this_thread(): _debug_session.debug_this_thread() else: # Install trace function even if no session yet - if hasattr(sys, 'settrace'): + if hasattr(sys, "settrace"): sys.settrace(_default_trace_func) else: raise RuntimeError("MICROPY_PY_SYS_SETTRACE required") @@ -132,7 +134,6 @@ def _default_trace_func(frame, event, arg): return None - def is_client_connected(): """Check if a debugger client is connected.""" global _debug_session diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index a2a00c170..c7553a604 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -3,12 +3,34 @@ import sys from ..common.messaging import JsonMessageChannel from ..common.constants import ( - CMD_INITIALIZE, CMD_LAUNCH, CMD_ATTACH, CMD_SET_BREAKPOINTS, - CMD_CONTINUE, CMD_NEXT, CMD_STEP_IN, CMD_STEP_OUT, CMD_PAUSE, - CMD_STACK_TRACE, CMD_SCOPES, CMD_VARIABLES, CMD_EVALUATE, CMD_DISCONNECT, - CMD_CONFIGURATION_DONE, CMD_THREADS, CMD_SOURCE, EVENT_INITIALIZED, EVENT_STOPPED, EVENT_CONTINUED, EVENT_TERMINATED, - STOP_REASON_BREAKPOINT, STOP_REASON_STEP, STOP_REASON_PAUSE, - TRACE_CALL, TRACE_LINE, TRACE_RETURN, TRACE_EXCEPTION + CMD_INITIALIZE, + CMD_LAUNCH, + CMD_ATTACH, + CMD_SET_BREAKPOINTS, + CMD_CONTINUE, + CMD_NEXT, + CMD_STEP_IN, + CMD_STEP_OUT, + CMD_PAUSE, + CMD_STACK_TRACE, + CMD_SCOPES, + CMD_VARIABLES, + CMD_EVALUATE, + CMD_DISCONNECT, + CMD_CONFIGURATION_DONE, + CMD_THREADS, + CMD_SOURCE, + EVENT_INITIALIZED, + EVENT_STOPPED, + EVENT_CONTINUED, + EVENT_TERMINATED, + STOP_REASON_BREAKPOINT, + STOP_REASON_STEP, + STOP_REASON_PAUSE, + TRACE_CALL, + TRACE_LINE, + TRACE_RETURN, + TRACE_EXCEPTION, ) from .pdb_adapter import PdbAdapter @@ -34,7 +56,7 @@ def _debug_print(self, message): @property def _baremetal(self) -> bool: - return sys.platform not in ("linux") # to be expanded + return sys.platform not in ("linux") # to be expanded def start(self): """Start the debug session message loop.""" @@ -77,7 +99,7 @@ def initialize_connection(self): message_count += 1 # Just wait for attach, then we can return control - if message.get('command') == 'attach': + if message.get("command") == "attach": attached = True print("[DAP] ✅ Attach received - returning control to main thread") break @@ -191,12 +213,12 @@ def _handle_request(self, message): elif command == CMD_SOURCE: self._handle_source(seq, args) else: - self.channel.send_response(command, seq, success=False, - message=f"Unknown command: {command}") + self.channel.send_response( + command, seq, success=False, message=f"Unknown command: {command}" + ) except Exception as e: - self.channel.send_response(command, seq, success=False, - message=str(e)) + self.channel.send_response(command, seq, success=False, message=str(e)) def _handle_initialize(self, seq, args): """Handle initialize request.""" @@ -251,16 +273,15 @@ def _handle_attach(self, seq, args): self.debug_logging = args.get("logToFile", False) self._debug_print(f"[DAP] Processing attach request with args: {args}") - print(f"[DAP] Debug logging {'enabled' if self.debug_logging else 'disabled'} (logToFile={self.debug_logging})") - + print( + f"[DAP] Debug logging {'enabled' if self.debug_logging else 'disabled'} (logToFile={self.debug_logging})" + ) + # get debugger root and debugee root from pathMappings - for pm in args.get("pathMappings",[]): + for pm in args.get("pathMappings", []): # debugee - debugger - self.pdb.path_mappings.append( - (pm.get("remoteRoot", "./"), - pm.get("localRoot", "./")) - ) - # # TODO: justMyCode, debugOptions , + self.pdb.path_mappings.append((pm.get("remoteRoot", "./"), pm.get("localRoot", "./"))) + # # TODO: justMyCode, debugOptions , # Enable trace function self.pdb.set_trace_function(self._trace_function) @@ -282,8 +303,9 @@ def _handle_set_breakpoints(self, seq, args): # Set breakpoints in pdb adapter actual_breakpoints = self.pdb.set_breakpoints(filename, breakpoints) - self.channel.send_response(CMD_SET_BREAKPOINTS, seq, - body={"breakpoints": actual_breakpoints}) + self.channel.send_response( + CMD_SET_BREAKPOINTS, seq, body={"breakpoints": actual_breakpoints} + ) def _handle_continue(self, seq, args): """Handle continue request.""" @@ -322,8 +344,11 @@ def _handle_pause(self, seq, args): def _handle_stack_trace(self, seq, args): """Handle stackTrace request.""" stack_frames = self.pdb.get_stack_trace() - self.channel.send_response(CMD_STACK_TRACE, seq, - body={"stackFrames": stack_frames, "totalFrames": len(stack_frames)}) + self.channel.send_response( + CMD_STACK_TRACE, + seq, + body={"stackFrames": stack_frames, "totalFrames": len(stack_frames)}, + ) def _handle_scopes(self, seq, args): """Handle scopes request.""" @@ -345,18 +370,17 @@ def _handle_evaluate(self, seq, args): frame_id = args.get("frameId") context = args.get("context", "watch") if not expression: - self.channel.send_response(CMD_EVALUATE, seq, success=False, - message="No expression provided") + self.channel.send_response( + CMD_EVALUATE, seq, success=False, message="No expression provided" + ) return try: result = self.pdb.evaluate_expression(expression, frame_id) - self.channel.send_response(CMD_EVALUATE, seq, body={ - "result": str(result), - "variablesReference": 0 - }) + self.channel.send_response( + CMD_EVALUATE, seq, body={"result": str(result), "variablesReference": 0} + ) except Exception as e: - self.channel.send_response(CMD_EVALUATE, seq, success=False, - message=str(e)) + self.channel.send_response(CMD_EVALUATE, seq, success=False, message=str(e)) def _handle_disconnect(self, seq, args): """Handle disconnect request.""" @@ -372,10 +396,7 @@ def _handle_configuration_done(self, seq, args): def _handle_threads(self, seq, args): """Handle threads request.""" # MicroPython is single-threaded, so return one thread - threads = [{ - "id": self.thread_id, - "name": "main" - }] + threads = [{"id": self.thread_id, "name": "main"}] self.channel.send_response(CMD_THREADS, seq, body={"threads": threads}) def _handle_source(self, seq, args): @@ -395,10 +416,13 @@ def _handle_source(self, seq, args): content = f.read() self.channel.send_response(CMD_SOURCE, seq, body={"content": content}) except Exception: - self.channel.send_response(CMD_SOURCE, seq, success=False, - message="cancelled" - # message=f"Could not read source: {e}" - ) + self.channel.send_response( + CMD_SOURCE, + seq, + success=False, + message="cancelled", + # message=f"Could not read source: {e}" + ) def _trace_function(self, frame, event, arg): """Trace function called by sys.settrace.""" @@ -407,8 +431,13 @@ def _trace_function(self, frame, event, arg): # Handle breakpoints and stepping if self.pdb.should_stop(frame, event, arg): - self._send_stopped_event(STOP_REASON_BREAKPOINT if self.pdb.hit_breakpoint else - STOP_REASON_STEP if self.stepping else STOP_REASON_PAUSE) + self._send_stopped_event( + STOP_REASON_BREAKPOINT + if self.pdb.hit_breakpoint + else STOP_REASON_STEP + if self.stepping + else STOP_REASON_PAUSE + ) # Wait for continue command self.pdb.wait_for_continue() @@ -416,10 +445,9 @@ def _trace_function(self, frame, event, arg): def _send_stopped_event(self, reason): """Send stopped event to client.""" - self.channel.send_event(EVENT_STOPPED, - reason=reason, - threadId=self.thread_id, - allThreadsStopped=True) + self.channel.send_event( + EVENT_STOPPED, reason=reason, threadId=self.thread_id, allThreadsStopped=True + ) def wait_for_client(self): """Wait for client to initialize.""" @@ -433,7 +461,7 @@ def trigger_breakpoint(self): def debug_this_thread(self): """Enable debugging for current thread.""" - if hasattr(sys, 'settrace'): + if hasattr(sys, "settrace"): sys.settrace(self._trace_function) def is_connected(self): @@ -443,7 +471,7 @@ def is_connected(self): def disconnect(self): """Disconnect from client.""" self.connected = False - if hasattr(sys, 'settrace'): + if hasattr(sys, "settrace"): sys.settrace(None) self.pdb.cleanup() self.channel.close() From 80dc0670866682eaf71f062302b3ca0a7d0cba00 Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Mon, 30 Jun 2025 13:10:05 +0200 Subject: [PATCH 36/74] debugpy: Enhance breakpoint handling and path mapping in PdbAdapter. Signed-off-by: Jos Verlinde --- .../debugpy/debugpy/server/pdb_adapter.py | 106 ++++++++++++------ 1 file changed, 71 insertions(+), 35 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index a5ab3d2ce..cd3354d8c 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -31,7 +31,7 @@ class PdbAdapter: """Adapter between DAP protocol and MicroPython's sys.settrace functionality.""" def __init__(self): - self.breakpoints = {} # filename -> {line_no: breakpoint_info} + self.breakpoints : dict[str,dict[int,dict]] = {} # filename -> {line_no: breakpoint_info} # todo - simplify - reduce info stored self.current_frame = None self.step_mode = None # None, 'over', 'into', 'out' self.step_frame = None @@ -40,8 +40,8 @@ def __init__(self): self.continue_event = False self.variables_cache = {} # frameId -> variables self.frame_id_counter = 1 - self.path_mappings : list[tuple[str,str]] = [] # runtime_path -> vscode_path mapping - self.file_mappings : dict[str,str] = {} # runtime_path -> vscode_path mapping + self.path_mappings : list[tuple[str,str]] = [] # runtime_path -> vscode_path mapping # todo: move to session level + self.file_mappings : dict[str,str] = {} # runtime_path -> vscode_path mapping # todo : merge with .breakpoints def _debug_print(self, message): """Print debug message only if debug logging is enabled.""" @@ -69,17 +69,61 @@ def set_trace_function(self, trace_func): else: raise RuntimeError("sys.settrace not available") - def set_breakpoints(self, filename, breakpoints:list[dict]): + def _filename_as_debugee(self, path:str): + # check if we have a 1:1 file mapping for this path + if self.file_mappings.get(path): + return self.file_mappings[path] + # Check if we have a folder mapping for this path + for runtime_path, vscode_path in self.path_mappings: + if path.startswith(vscode_path): + path = path.replace(vscode_path, runtime_path, 1) + if path.startswith('//'): + path = path[1:] + # If no mapping found, return the original path + return path + + def _filename_as_debugger(self, path:str): + """Convert a file path to the debugger's expected format.""" + path = path or "" + if not path: + return path + if path.startswith('<'): + # Special case for or similar + return path + # Check if we have a 1:1 file mapping for this path + for runtime_path, vscode_path in self.path_mappings: + if path.startswith(runtime_path): + path = path.replace(runtime_path, vscode_path, 1) + return path + + # Check if we have a folder mapping for this path + for runtime_path, vscode_path in self.path_mappings: + if path.startswith(runtime_path): + path = path.replace(runtime_path, vscode_path, 1) + if path.startswith('//'): + path = path[1:] + # If no mapping found, return the original path + return path + + def set_breakpoints(self, filename:str, breakpoints:list[dict]): """Set breakpoints for a file.""" self.breakpoints[filename] = {} + local_name = self._filename_as_debugee(filename) + self.file_mappings[local_name] = filename actual_breakpoints = [] - - # Debug log the breakpoint path self._debug_print(f"[PDB] Setting breakpoints for file: {filename}") for bp in breakpoints: line = bp.get("line") if line: + if local_name != filename: + self.breakpoints[local_name] = {} + self._debug_print(f"[>>>] Setting breakpoints for local: {local_name}:{line}") + self.breakpoints[local_name][line] = { + "line": line, + "verified": True, + "source": {"path": filename} + } self.breakpoints[filename][line] = { "line": line, "verified": True, @@ -91,6 +135,8 @@ def set_breakpoints(self, filename, breakpoints:list[dict]): "source": {"path": filename} }) + self._debug_print(f"[PDB] Breakpoints set : {self.breakpoints}") + return actual_breakpoints def should_stop(self, frame, event:str, arg): @@ -106,33 +152,18 @@ def should_stop(self, frame, event:str, arg): if lineno in self.breakpoints[filename]: self._debug_print(f"[PDB] HIT BREAKPOINT (exact match) at {filename}:{lineno}") # Record the path mapping (in this case, they're already the same) - self.file_mappings[filename] = filename + self.file_mappings[filename] = self._filename_as_debugger(filename) self.hit_breakpoint = True return True - - file_basename = basename(filename) - self._debug_print(f"[PDB] Fallback basename match: '{file_basename}' vs available files") - for bp_file in self.breakpoints: - bp_basename = basename(bp_file) - self._debug_print(f"[PDB] Comparing '{file_basename}' == '{bp_basename}' ?") - if bp_basename == file_basename: - self._debug_print(f"[PDB] Basename match found! Checking line {lineno} in {list(self.breakpoints[bp_file].keys())}") - if lineno in self.breakpoints[bp_file]: - self._debug_print(f"[PDB] HIT BREAKPOINT (fallback basename match) at {filename}:{lineno} -> {bp_file}") - # Record the path mapping so we can report the correct path in stack traces - self.file_mappings[filename] = bp_file - self.hit_breakpoint = True - return True - - # Also check if the runtime path might be relative and the breakpoint path absolute - if ends_with_path(bp_file, filename): - self._debug_print(f"[PDB] Relative path match: {bp_file} ends with {filename}") - if lineno in self.breakpoints[bp_file]: - self._debug_print(f"[PDB] HIT BREAKPOINT (relative path match) at {filename}:{lineno} -> {bp_file}") - # Record the path mapping so we can report the correct path in stack traces - self.file_mappings[filename] = bp_file - self.hit_breakpoint = True - return True + # path/file.py matched - but not the line number - keep running + else: + # file not (yet) matched - this is slow so we do not want to do this often. + # TODO: use builins - sys.path method to find the file + # if we have a path match , but no breakpoints - add it to the file_mappings dict avoid this check + self.breakpoints[filename] = {} # Ensure the filename is in the breakpoints dict + if not filename in self.file_mappings: + self.file_mappings[filename] = self._filename_as_debugger(filename) + self._debug_print(f"[PDB] add mapping for :'{filename}' -> '{self.file_mappings[filename]}'") # Check stepping if self.step_mode == 'into': @@ -216,15 +247,20 @@ def get_stack_trace(self): else : hint = 'normal' + # self._debug_print("=" * 40 ) + # self._debug_print(f"[PDB] file mappings: {repr(self.file_mappings)} " ) + # self._debug_print(f"[PDB] path mappings: {repr(self.path_mappings)}" ) + # self._debug_print("=" * 40 ) + # Use the VS Code path if we have a mapping, otherwise use the original path - display_path = self.file_mappings.get(filename, filename) - if filename != display_path: - self._debug_print(f"[PDB] Stack trace path mapping: {filename} -> {display_path}") + debugger_path = self._filename_as_debugger(filename) + if filename != debugger_path: + self._debug_print(f"[PDB] Stack trace path mapping: {filename} -> {debugger_path}") # Create StackFrame info frames.append({ "id": frame_id, "name": name, - "source": {"path": display_path}, + "source": {"path": debugger_path}, "line": line, "column": 1, "endLine": line, From fdb0a186464f0d4cddcfea04abfee03538c66230 Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Tue, 1 Jul 2025 00:15:55 +0200 Subject: [PATCH 37/74] debugpy : Optimize memory management and performance in PDB adapter. Signed-off-by: Jos Verlinde --- .../debugpy/debugpy/server/pdb_adapter.py | 370 ++++++++++++++---- 1 file changed, 303 insertions(+), 67 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index 11b4dfb81..46ff5a362 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -51,16 +51,22 @@ def get_variable(self, ref_id: int): # -> Optional[Any] return self.cache.get(ref_id) def _cleanup_oldest(self) -> None: - """Remove oldest entries to free memory.""" - if self.cache and self.insertion_order: - # Remove first quarter of entries (true FIFO based on insertion order) - to_remove = max(1, len(self.cache) // 4) # Remove at least 1 entry - keys_to_remove = self.insertion_order[:to_remove] - for key in keys_to_remove: - if key in self.cache: - del self.cache[key] - # Update insertion order - self.insertion_order = self.insertion_order[to_remove:] + """Remove oldest entries to free memory - optimized for MicroPython.""" + if not self.cache or not self.insertion_order: + return + + # More aggressive cleanup for memory-constrained environments + to_remove = max(1, len(self.cache) // 3) # Remove 1/3 instead of 1/4 + + # Direct list slicing is more memory efficient than iteration + keys_to_remove = self.insertion_order[:to_remove] + + # Batch delete for efficiency + for key in keys_to_remove: + self.cache.pop(key, None) # Use pop with default to avoid KeyError + + # Update insertion order in one operation + self.insertion_order = self.insertion_order[to_remove:] def clear(self) -> None: """Clear all cached variables.""" @@ -368,7 +374,8 @@ def _process_special_variables(self, var_dict): for name, value in var_dict.items(): if name.startswith("__") and name.endswith("__"): try: - value_str = json.dumps(value) + # Use lightweight serialization instead of json.dumps + value_str = self._lightweight_serialize(value) type_str = type(value).__name__ variables.append( { @@ -383,13 +390,14 @@ def _process_special_variables(self, var_dict): return variables def _process_regular_variables(self, var_dict): - """Process regular variables (excluding special ones).""" + """Process regular variables (excluding special ones) - optimized.""" variables = [] for name, value in var_dict.items(): # Skip private/internal variables if name.startswith("__") and name.endswith("__"): continue - variables.append(self._get_variable_info(name, value)) + # Use fast path for variable info generation + variables.append(self._get_variable_info_fast(name, value)) return variables def _is_expandable(self, value: Any) -> bool: @@ -397,28 +405,100 @@ def _is_expandable(self, value: Any) -> bool: return isinstance(value, (dict, list, tuple, set)) def _get_preview(self, value: Any, fallback_text: str = "") -> str: - """Get a truncated preview of a variable value.""" + """Get a truncated preview of a variable value - optimized for MicroPython.""" try: if value is None: return "None" - # Try to get a meaningful representation - preview_repr = repr(value) - if len(preview_repr) > 30: - return preview_repr[:30] + "..." - else: - return preview_repr - except (TypeError, ValueError): - # If repr fails, try str - try: - preview_str = str(value) - if len(preview_str) > 30: - return preview_str[:30] + "..." + # Fast path for common types to avoid repr() overhead + if isinstance(value, bool): + return "True" if value else "False" + elif isinstance(value, int): + return str(value) + elif isinstance(value, float): + # Limit float precision to reduce string length + return f"{value:.6g}" + elif isinstance(value, str): + if len(value) > 30: + return value[:30] + "..." + else: + return repr(value) # Only use repr for short strings + + # For collections, show actual content if small, otherwise use lightweight approach + elif isinstance(value, dict): + if len(value) == 0: + return "{}" + elif len(value) <= 3: + # Show actual content for small dictionaries + try: + repr_val = repr(value) + if len(repr_val) <= 60: + return repr_val + else: + # Fallback to key list if repr is too long + keys = list(value.keys())[:3] + key_str = ", ".join(repr(k) for k in keys) + return f"{{{key_str}}}" + except: + return f"dict({len(value)} items)" + else: + return f"dict({len(value)} items)" + elif isinstance(value, (list, tuple)): + if len(value) == 0: + return "[]" if isinstance(value, list) else "()" + elif len(value) <= 4: + # Show actual content for small lists/tuples + try: + repr_val = repr(value) + if len(repr_val) <= 60: + return repr_val + else: + # Fallback to item preview if repr is too long + items = [str(item)[:10] for item in value[:3]] + bracket = "[]" if isinstance(value, list) else "()" + return f"{bracket[0]}{', '.join(items)}...{bracket[1]}" + except: + type_name = type(value).__name__ + return f"{type_name}({len(value)} items)" else: - return preview_str + type_name = type(value).__name__ + return f"{type_name}({len(value)} items)" + elif isinstance(value, set): + if len(value) == 0: + return "set()" + elif len(value) <= 4: + # Show actual content for small sets + try: + repr_val = repr(value) + if len(repr_val) <= 60: + return repr_val + else: + # Fallback to item preview + items = [str(item)[:10] for item in list(value)[:3]] + return f"{{{', '.join(items)}...}}" + except: + return f"set({len(value)} items)" + else: + return f"set({len(value)} items)" + + # For other complex types, use lightweight approach + type_name = type(value).__name__ + try: + if hasattr(value, '__len__'): + length = len(value) # type: ignore + if length == 0: + return f"{type_name}(empty)" + else: + return f"{type_name}({length} items)" except: - # Final fallback - return fallback_text or f"<{type(value).__name__} object>" + pass + + # Final fallback - avoid expensive repr() for complex objects + return f"<{type_name} object>" + + except (TypeError, ValueError, MemoryError): + # Memory-safe fallback + return fallback_text or f"<{type(value).__name__} object>" def _get_variable_info(self, name: str, value: Any) -> dict[str, str | int]: """Get DAP-compliant variable information with proper type handling.""" @@ -428,11 +508,15 @@ def _get_variable_info(self, name: str, value: Any) -> dict[str, str | int]: var_ref = self.var_cache.add_variable(value) if isinstance(value, dict): - preview = ( - self._get_preview(value, f"dict({len(value)} items)") - if value - else "dict(empty)" - ) + # Show actual content for small dicts, generic preview for large ones + if len(value) == 0: + preview = "dict(empty)" + elif len(value) <= 3: + # Show actual keys for small dictionaries + preview = self._get_preview(value) + else: + preview = f"dict({len(value)} items)" + return { "name": name, "value": preview, @@ -442,11 +526,15 @@ def _get_variable_info(self, name: str, value: Any) -> dict[str, str | int]: "indexedVariables": 0, } elif isinstance(value, list): - preview = ( - self._get_preview(value, f"list({len(value)} items)") - if value - else "list(empty)" - ) + # Show actual content for small lists, generic preview for large ones + if len(value) == 0: + preview = "list(empty)" + elif len(value) <= 4: + # Show actual items for small lists + preview = self._get_preview(value) + else: + preview = f"list({len(value)} items)" + return { "name": name, "value": preview, @@ -456,11 +544,14 @@ def _get_variable_info(self, name: str, value: Any) -> dict[str, str | int]: "namedVariables": 0, } elif isinstance(value, tuple): - preview = ( - self._get_preview(value, f"tuple({len(value)} items)") - if value - else "tuple(empty)" - ) + # Show actual content for small tuples + if len(value) == 0: + preview = "tuple(empty)" + elif len(value) <= 4: + preview = self._get_preview(value) + else: + preview = f"tuple({len(value)} items)" + return { "name": name, "value": preview, @@ -470,11 +561,14 @@ def _get_variable_info(self, name: str, value: Any) -> dict[str, str | int]: "namedVariables": 0, } elif isinstance(value, set): - preview = ( - self._get_preview(value, f"set({len(value)} items)") - if value - else "set(empty)" - ) + # Show actual content for small sets + if len(value) == 0: + preview = "set(empty)" + elif len(value) <= 4: + preview = self._get_preview(value) + else: + preview = f"set({len(value)} items)" + return { "name": name, "value": preview, @@ -496,8 +590,72 @@ def _get_variable_info(self, name: str, value: Any) -> dict[str, str | int]: except Exception: return self._var_error(name) + def _get_variable_info_fast(self, name: str, value: Any) -> dict[str, str | int]: + """Fast path for variable info generation with reduced allocations.""" + try: + # Handle expandable types + if self._is_expandable(value): + var_ref = self.var_cache.add_variable(value) + type_name = type(value).__name__ + + # Use pre-calculated length for better performance + length = 0 + try: + length = len(value) # type: ignore + if length == 0: + preview = f"{type_name}(empty)" + else: + preview = f"{type_name}({length} items)" + except: + preview = f"<{type_name} object>" + + # Return optimized structure based on type + if isinstance(value, dict): + return { + "name": name, + "value": preview, + "type": "dict", + "variablesReference": var_ref, + "namedVariables": length if length < 1000 else 1000, # Cap for performance + "indexedVariables": 0, + } + elif isinstance(value, list): + return { + "name": name, + "value": preview, + "type": "list", + "variablesReference": var_ref, + "indexedVariables": min(length, 1000), # Cap for performance + "namedVariables": 0, + } + else: # tuple, set, other + return { + "name": name, + "value": preview, + "type": type_name, + "variablesReference": var_ref, + "indexedVariables": min(length, 1000), + "namedVariables": 0, + } + + # Simple types - optimized path + preview = self._get_preview(value) + return { + "name": name, + "value": preview, + "type": type(value).__name__, + "variablesReference": 0, + } + except Exception: + return { + "name": name, + "value": "", + "type": "unknown", + "variablesReference": 0 + } + def _expand_complex_variable(self, ref_id: int) -> list[dict[str, str | int]]: - """Expand a complex variable into its child elements.""" + """Expand a complex variable into its child elements - optimized for memory.""" value = self.var_cache.get_variable(ref_id) if value is None: return [] @@ -505,28 +663,53 @@ def _expand_complex_variable(self, ref_id: int) -> list[dict[str, str | int]]: variables = [] try: if isinstance(value, dict): - # Handle dictionary keys and values - for key, val in value.items(): - key_str = str(key) + # Limit dictionary expansion to prevent memory exhaustion + items = list(value.items()) + max_items = min(len(items), 50) # Limit to 50 items max + for i in range(max_items): + key, val = items[i] + key_str = str(key)[:50] # Limit key string length variables.append(self._get_variable_info(key_str, val)) + if len(items) > max_items: + variables.append({ + "name": f"<{len(items) - max_items} more items>", + "value": "...", + "type": "info", + "variablesReference": 0, + }) elif isinstance(value, (list, tuple)): - # Handle list/tuple elements - for i, val in enumerate(value): - variables.append(self._get_variable_info(f"[{i}]", val)) + # Limit list/tuple expansion + max_items = min(len(value), 100) # Limit to 100 items max + for i in range(max_items): + variables.append(self._get_variable_info(f"[{i}]", value[i])) + if len(value) > max_items: + variables.append({ + "name": f"<{len(value) - max_items} more items>", + "value": "...", + "type": "info", + "variablesReference": 0, + }) elif isinstance(value, set): - # Handle set elements (sorted for consistent display) - for i, val in enumerate(sorted(value, key=lambda x: str(x))): - variables.append(self._get_variable_info(f"<{i}>", val)) + # Handle set elements with size limit + items = list(value) # Convert once + max_items = min(len(items), 50) + for i in range(max_items): + variables.append(self._get_variable_info(f"<{i}>", items[i])) + if len(items) > max_items: + variables.append({ + "name": f"<{len(items) - max_items} more items>", + "value": "...", + "type": "info", + "variablesReference": 0, + }) except Exception as e: # Return error info for debugging - variables.append( - { - "name": "error", - "value": f"Failed to expand: {e}", - "type": "error", - "variablesReference": 0, - } - ) + variables.append({ + "name": "error", + "value": f"Failed to expand: {str(e)[:50]}", # Limit error message length + "type": "error", + "variablesReference": 0, + }) return variables @@ -605,3 +788,56 @@ def cleanup(self): self.breakpoints.clear() if hasattr(sys, "settrace"): sys.settrace(None) + + def _lightweight_serialize(self, value): + """Lightweight serialization optimized for MicroPython memory constraints.""" + if value is None: + return "None" + elif isinstance(value, bool): + return "true" if value else "false" + elif isinstance(value, (int, float)): + return str(value) + elif isinstance(value, str): + # Simple escaping for strings - avoid full JSON complexity + if len(value) > 30: + escaped = value[:27].replace('"', '\\"').replace('\n', '\\n') + return f'"{escaped}..."' + else: + escaped = value.replace('"', '\\"').replace('\n', '\\n') + return f'"{escaped}"' + elif isinstance(value, (list, tuple)): + if len(value) == 0: + return "[]" if isinstance(value, list) else "()" + elif len(value) <= 3: + # Show small collections in full + items = [self._lightweight_serialize(item) for item in value] + brackets = "[]" if isinstance(value, list) else "()" + return f"{brackets[0]}{', '.join(items)}{brackets[1]}" + else: + # Show preview for large collections + preview = f"{type(value).__name__}({len(value)} items)" + return preview + elif isinstance(value, dict): + if len(value) == 0: + return "{}" + elif len(value) <= 2: + # Show small dicts in preview form + items = [] + for k, v in value.items(): + key_str = self._lightweight_serialize(k) + val_str = self._lightweight_serialize(v) + items.append(f"{key_str}: {val_str}") + return "{" + ", ".join(items) + "}" + else: + return f"dict({len(value)} items)" + else: + # Fallback for other types + type_name = type(value).__name__ + try: + repr_val = repr(value) + if len(repr_val) > 30: + return f"<{type_name} object>" + else: + return repr_val + except: + return f"<{type_name} object>" From 5ab5a3f1df64d53457f7155717ac354dfbeb9083 Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Mon, 30 Jun 2025 21:12:36 +0200 Subject: [PATCH 38/74] debugpy: Store breakpoints in sets rather than lists. Signed-off-by: Jos Verlinde --- .../debugpy/debugpy/server/pdb_adapter.py | 46 ++++++++----------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index cd3354d8c..19fc89385 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -13,6 +13,7 @@ VARREF_LOCALS_SPECIAL = 3 VARREF_GLOBALS_SPECIAL = 4 +DEBUG = False # Also try checking by basename for path mismatches def basename(path:str): @@ -31,7 +32,7 @@ class PdbAdapter: """Adapter between DAP protocol and MicroPython's sys.settrace functionality.""" def __init__(self): - self.breakpoints : dict[str,dict[int,dict]] = {} # filename -> {line_no: breakpoint_info} # todo - simplify - reduce info stored + self.breakpoints : dict[str,set[int]] = {} # .breakpoints[filename] -> set of line numbers self.current_frame = None self.step_mode = None # None, 'over', 'into', 'out' self.step_frame = None @@ -40,8 +41,8 @@ def __init__(self): self.continue_event = False self.variables_cache = {} # frameId -> variables self.frame_id_counter = 1 - self.path_mappings : list[tuple[str,str]] = [] # runtime_path -> vscode_path mapping # todo: move to session level - self.file_mappings : dict[str,str] = {} # runtime_path -> vscode_path mapping # todo : merge with .breakpoints + self.path_mappings : list[tuple[str,str]] = [] # runtime_path -> vscode_path mapping + self.file_mappings : dict[str,str] = {} # runtime_path -> vscode_path mapping # todo : ? merge with .breakpoints def _debug_print(self, message): """Print debug message only if debug logging is enabled.""" @@ -107,7 +108,7 @@ def _filename_as_debugger(self, path:str): def set_breakpoints(self, filename:str, breakpoints:list[dict]): """Set breakpoints for a file.""" - self.breakpoints[filename] = {} + self.breakpoints[filename] = set() local_name = self._filename_as_debugee(filename) self.file_mappings[local_name] = filename actual_breakpoints = [] @@ -117,18 +118,11 @@ def set_breakpoints(self, filename:str, breakpoints:list[dict]): line = bp.get("line") if line: if local_name != filename: - self.breakpoints[local_name] = {} + self.breakpoints[local_name] = set() self._debug_print(f"[>>>] Setting breakpoints for local: {local_name}:{line}") - self.breakpoints[local_name][line] = { - "line": line, - "verified": True, - "source": {"path": filename} - } - self.breakpoints[filename][line] = { - "line": line, - "verified": True, - "source": {"path": filename} - } + self.breakpoints[local_name].add(line) + + self.breakpoints[filename].add(line) actual_breakpoints.append({ "line": line, "verified": True, @@ -148,19 +142,19 @@ def should_stop(self, frame, event:str, arg): filename = frame.f_code.co_filename lineno = frame.f_lineno # Check for exact filename match first - if filename in self.breakpoints: - if lineno in self.breakpoints[filename]: + if filename in self.breakpoints and lineno in self.breakpoints[filename]: self._debug_print(f"[PDB] HIT BREAKPOINT (exact match) at {filename}:{lineno}") # Record the path mapping (in this case, they're already the same) - self.file_mappings[filename] = self._filename_as_debugger(filename) + # self.file_mappings[filename] = self._filename_as_debugger(filename) self.hit_breakpoint = True return True # path/file.py matched - but not the line number - keep running else: # file not (yet) matched - this is slow so we do not want to do this often. # TODO: use builins - sys.path method to find the file - # if we have a path match , but no breakpoints - add it to the file_mappings dict avoid this check - self.breakpoints[filename] = {} # Ensure the filename is in the breakpoints dict + # if we have a path match , but no breakpoints - add it to the file_mappings dict simplify this check + if not filename in self.breakpoints: + self.breakpoints[filename] = set() # Ensure the filename is in the breakpoints dict if not filename in self.file_mappings: self.file_mappings[filename] = self._filename_as_debugger(filename) self._debug_print(f"[PDB] add mapping for :'{filename}' -> '{self.file_mappings[filename]}'") @@ -238,6 +232,12 @@ def get_stack_trace(self): frame = self.current_frame frame_id = 0 + self._debug_print("=" * 40 ) + self._debug_print(f"[PDB] file mappings: {repr(self.file_mappings)} " ) + self._debug_print(f"[PDB] path mappings: {repr(self.path_mappings)}" ) + self._debug_print(f"[PDB] breakpoints: {repr(self.breakpoints)}" ) + self._debug_print("=" * 40 ) + while frame: filename = frame.f_code.co_filename name = frame.f_code.co_name @@ -247,15 +247,9 @@ def get_stack_trace(self): else : hint = 'normal' - # self._debug_print("=" * 40 ) - # self._debug_print(f"[PDB] file mappings: {repr(self.file_mappings)} " ) - # self._debug_print(f"[PDB] path mappings: {repr(self.path_mappings)}" ) - # self._debug_print("=" * 40 ) # Use the VS Code path if we have a mapping, otherwise use the original path debugger_path = self._filename_as_debugger(filename) - if filename != debugger_path: - self._debug_print(f"[PDB] Stack trace path mapping: {filename} -> {debugger_path}") # Create StackFrame info frames.append({ "id": frame_id, From 564401b426714146a11092ae7e7a5785c1f0bad7 Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Tue, 1 Jul 2025 00:28:26 +0200 Subject: [PATCH 39/74] revert: Overly complex variable preview. Signed-off-by: Jos Verlinde --- .../debugpy/debugpy/server/pdb_adapter.py | 146 ++---------------- 1 file changed, 13 insertions(+), 133 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index fc87ef13a..9d242cd6b 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -405,100 +405,17 @@ def _is_expandable(self, value: Any) -> bool: return isinstance(value, (dict, list, tuple, set)) def _get_preview(self, value: Any, fallback_text: str = "") -> str: - """Get a truncated preview of a variable value - optimized for MicroPython.""" + """Get a 30-char preview of a variable value with '...' if truncated - optimized for MicroPython.""" try: - if value is None: - return "None" - - # Fast path for common types to avoid repr() overhead - if isinstance(value, bool): - return "True" if value else "False" - elif isinstance(value, int): - return str(value) - elif isinstance(value, float): - # Limit float precision to reduce string length - return f"{value:.6g}" - elif isinstance(value, str): - if len(value) > 30: - return value[:30] + "..." - else: - return repr(value) # Only use repr for short strings - - # For collections, show actual content if small, otherwise use lightweight approach - elif isinstance(value, dict): - if len(value) == 0: - return "{}" - elif len(value) <= 3: - # Show actual content for small dictionaries - try: - repr_val = repr(value) - if len(repr_val) <= 60: - return repr_val - else: - # Fallback to key list if repr is too long - keys = list(value.keys())[:3] - key_str = ", ".join(repr(k) for k in keys) - return f"{{{key_str}}}" - except: - return f"dict({len(value)} items)" - else: - return f"dict({len(value)} items)" - elif isinstance(value, (list, tuple)): - if len(value) == 0: - return "[]" if isinstance(value, list) else "()" - elif len(value) <= 4: - # Show actual content for small lists/tuples - try: - repr_val = repr(value) - if len(repr_val) <= 60: - return repr_val - else: - # Fallback to item preview if repr is too long - items = [str(item)[:10] for item in value[:3]] - bracket = "[]" if isinstance(value, list) else "()" - return f"{bracket[0]}{', '.join(items)}...{bracket[1]}" - except: - type_name = type(value).__name__ - return f"{type_name}({len(value)} items)" - else: - type_name = type(value).__name__ - return f"{type_name}({len(value)} items)" - elif isinstance(value, set): - if len(value) == 0: - return "set()" - elif len(value) <= 4: - # Show actual content for small sets - try: - repr_val = repr(value) - if len(repr_val) <= 60: - return repr_val - else: - # Fallback to item preview - items = [str(item)[:10] for item in list(value)[:3]] - return f"{{{', '.join(items)}...}}" - except: - return f"set({len(value)} items)" - else: - return f"set({len(value)} items)" - - # For other complex types, use lightweight approach - type_name = type(value).__name__ - try: - if hasattr(value, '__len__'): - length = len(value) # type: ignore - if length == 0: - return f"{type_name}(empty)" - else: - return f"{type_name}({length} items)" - except: - pass - - # Final fallback - avoid expensive repr() for complex objects - return f"<{type_name} object>" - + # Get repr and truncate to exactly 30 chars with "..." if needed + repr_val = repr(value) + if len(repr_val) <= 30: + return repr_val + else: + return repr_val[:30] + "..." except (TypeError, ValueError, MemoryError): # Memory-safe fallback - return fallback_text or f"<{type(value).__name__} object>" + return fallback_text or f"<{type(value).__name__} object>"[:30] def _get_variable_info(self, name: str, value: Any) -> dict[str, str | int]: """Get DAP-compliant variable information with proper type handling.""" @@ -506,17 +423,9 @@ def _get_variable_info(self, name: str, value: Any) -> dict[str, str | int]: # Handle expandable types if self._is_expandable(value): var_ref = self.var_cache.add_variable(value) + preview = self._get_preview(value) # Always use consistent preview if isinstance(value, dict): - # Show actual content for small dicts, generic preview for large ones - if len(value) == 0: - preview = "dict(empty)" - elif len(value) <= 3: - # Show actual keys for small dictionaries - preview = self._get_preview(value) - else: - preview = f"dict({len(value)} items)" - return { "name": name, "value": preview, @@ -526,15 +435,6 @@ def _get_variable_info(self, name: str, value: Any) -> dict[str, str | int]: "indexedVariables": 0, } elif isinstance(value, list): - # Show actual content for small lists, generic preview for large ones - if len(value) == 0: - preview = "list(empty)" - elif len(value) <= 4: - # Show actual items for small lists - preview = self._get_preview(value) - else: - preview = f"list({len(value)} items)" - return { "name": name, "value": preview, @@ -544,14 +444,6 @@ def _get_variable_info(self, name: str, value: Any) -> dict[str, str | int]: "namedVariables": 0, } elif isinstance(value, tuple): - # Show actual content for small tuples - if len(value) == 0: - preview = "tuple(empty)" - elif len(value) <= 4: - preview = self._get_preview(value) - else: - preview = f"tuple({len(value)} items)" - return { "name": name, "value": preview, @@ -561,14 +453,6 @@ def _get_variable_info(self, name: str, value: Any) -> dict[str, str | int]: "namedVariables": 0, } elif isinstance(value, set): - # Show actual content for small sets - if len(value) == 0: - preview = "set(empty)" - elif len(value) <= 4: - preview = self._get_preview(value) - else: - preview = f"set({len(value)} items)" - return { "name": name, "value": preview, @@ -596,18 +480,14 @@ def _get_variable_info_fast(self, name: str, value: Any) -> dict[str, str | int] # Handle expandable types if self._is_expandable(value): var_ref = self.var_cache.add_variable(value) - type_name = type(value).__name__ + preview = self._get_preview(value) # Always use consistent preview # Use pre-calculated length for better performance length = 0 try: length = len(value) # type: ignore - if length == 0: - preview = f"{type_name}(empty)" - else: - preview = f"{type_name}({length} items)" except: - preview = f"<{type_name} object>" + pass # Return optimized structure based on type if isinstance(value, dict): @@ -632,7 +512,7 @@ def _get_variable_info_fast(self, name: str, value: Any) -> dict[str, str | int] return { "name": name, "value": preview, - "type": type_name, + "type": type(value).__name__, "variablesReference": var_ref, "indexedVariables": min(length, 1000), "namedVariables": 0, @@ -719,7 +599,7 @@ def _var_error(name: str): @staticmethod def _special_vars(varref: int): - return {"name": "Special", "value": "", "variablesReference": varref} + return {"name": "special", "value": "", "variablesReference": varref} def get_variables(self, variables_ref): """Get variables for a scope with enhanced complex variable support.""" From 8f01ba4a4564e6d7cf789bf6c59afc9c46d2007b Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Tue, 1 Jul 2025 12:56:41 +0200 Subject: [PATCH 40/74] debugpy: Performance improvements. Signed-off-by: Jos Verlinde --- .../debugpy/debugpy/common/constants.py | 89 ++++++++++--------- .../debugpy/debugpy/common/messaging.py | 45 +++++++--- .../debugpy/debugpy/server/debug_session.py | 10 ++- .../debugpy/debugpy/server/pdb_adapter.py | 63 ++++++------- 4 files changed, 116 insertions(+), 91 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/common/constants.py b/python-ecosys/debugpy/debugpy/common/constants.py index aeee675e3..7f832eea8 100644 --- a/python-ecosys/debugpy/debugpy/common/constants.py +++ b/python-ecosys/debugpy/debugpy/common/constants.py @@ -1,60 +1,67 @@ """Constants used throughout debugpy.""" +from micropython import const # Default networking settings DEFAULT_HOST = "127.0.0.1" DEFAULT_PORT = 5678 # DAP message types -MSG_TYPE_REQUEST = "request" -MSG_TYPE_RESPONSE = "response" -MSG_TYPE_EVENT = "event" +MSG_TYPE_REQUEST = const("request") +MSG_TYPE_RESPONSE = const("response") +MSG_TYPE_EVENT = const("event") # DAP events -EVENT_INITIALIZED = "initialized" -EVENT_STOPPED = "stopped" -EVENT_CONTINUED = "continued" -EVENT_THREAD = "thread" -EVENT_BREAKPOINT = "breakpoint" -EVENT_OUTPUT = "output" -EVENT_TERMINATED = "terminated" -EVENT_EXITED = "exited" +EVENT_INITIALIZED = const("initialized") +EVENT_STOPPED = const("stopped") +EVENT_CONTINUED = const("continued") +EVENT_THREAD = const("thread") +EVENT_BREAKPOINT = const("breakpoint") +EVENT_OUTPUT = const("output") +EVENT_TERMINATED = const("terminated") +EVENT_EXITED = const("exited") # DAP commands -CMD_INITIALIZE = "initialize" -CMD_LAUNCH = "launch" -CMD_ATTACH = "attach" -CMD_SET_BREAKPOINTS = "setBreakpoints" -CMD_CONTINUE = "continue" -CMD_NEXT = "next" -CMD_STEP_IN = "stepIn" -CMD_STEP_OUT = "stepOut" -CMD_PAUSE = "pause" -CMD_STACK_TRACE = "stackTrace" -CMD_SCOPES = "scopes" -CMD_VARIABLES = "variables" -CMD_EVALUATE = "evaluate" -CMD_DISCONNECT = "disconnect" -CMD_CONFIGURATION_DONE = "configurationDone" -CMD_THREADS = "threads" -CMD_SOURCE = "source" +CMD_INITIALIZE = const("initialize") +CMD_LAUNCH = const("launch") +CMD_ATTACH = const("attach") +CMD_SET_BREAKPOINTS = const("setBreakpoints") +CMD_CONTINUE = const("continue") +CMD_NEXT = const("next") +CMD_STEP_IN = const("stepIn") +CMD_STEP_OUT = const("stepOut") +CMD_PAUSE = const("pause") +CMD_STACK_TRACE = const("stackTrace") +CMD_SCOPES = const("scopes") +CMD_VARIABLES = const("variables") +CMD_EVALUATE = const("evaluate") +CMD_DISCONNECT = const("disconnect") +CMD_CONFIGURATION_DONE = const("configurationDone") +CMD_THREADS = const("threads") +CMD_SOURCE = const("source") # Stop reasons -STOP_REASON_STEP = "step" -STOP_REASON_BREAKPOINT = "breakpoint" -STOP_REASON_EXCEPTION = "exception" -STOP_REASON_PAUSE = "pause" -STOP_REASON_ENTRY = "entry" +STOP_REASON_STEP = const("step") +STOP_REASON_BREAKPOINT = const("breakpoint") +STOP_REASON_EXCEPTION = const("exception") +STOP_REASON_PAUSE = const("pause") +STOP_REASON_ENTRY = const("entry") # Thread reasons -THREAD_REASON_STARTED = "started" -THREAD_REASON_EXITED = "exited" +THREAD_REASON_STARTED = const("started") +THREAD_REASON_EXITED = const("exited") # Trace events -TRACE_CALL = "call" -TRACE_LINE = "line" -TRACE_RETURN = "return" -TRACE_EXCEPTION = "exception" +TRACE_CALL = const("call") +TRACE_LINE = const("line") +TRACE_RETURN = const("return") +TRACE_EXCEPTION = const("exception") + +# Step modes +STEP_INTO = const("into") +STEP_OVER = const("over") +STEP_OUT = const("out") + # Scope types -SCOPE_LOCALS = "locals" -SCOPE_GLOBALS = "globals" +SCOPE_LOCALS = const("locals") +SCOPE_GLOBALS = const("globals") diff --git a/python-ecosys/debugpy/debugpy/common/messaging.py b/python-ecosys/debugpy/debugpy/common/messaging.py index a491578ad..eb7df4dd2 100644 --- a/python-ecosys/debugpy/debugpy/common/messaging.py +++ b/python-ecosys/debugpy/debugpy/common/messaging.py @@ -86,15 +86,36 @@ def recv_message(self): if self.closed: return None + # Quick bail-out: if buffer is empty, do a non-blocking peek to see if data is available + if not self._recv_buffer: + try: + # Try to read a small amount non-blocking to see if anything is available + peek_data = self.sock.recv(1) + if not peek_data: + return None # No data available + # Put the peeked data back into buffer + self._recv_buffer = peek_data + except OSError as e: + # Handle non-blocking socket errors (no data available) + if hasattr(e, "errno") and e.errno in (11, 35): # EAGAIN, EWOULDBLOCK + return None # No data available, quick exit + # Other errors + self.closed = True + return None + + # Cache frequently accessed attributes + recv_buffer = self._recv_buffer + sock_recv = self.sock.recv + try: # Read headers - while b"\r\n\r\n" not in self._recv_buffer: + while b"\r\n\r\n" not in recv_buffer: try: - data = self.sock.recv(1024) + data = sock_recv(1024) if not data: self.closed = True return None - self._recv_buffer += data + recv_buffer += data except OSError as e: # Handle timeout and other socket errors if hasattr(e, "errno") and e.errno in (11, 35): # EAGAIN, EWOULDBLOCK @@ -102,9 +123,9 @@ def recv_message(self): self.closed = True return None - header_end = self._recv_buffer.find(b"\r\n\r\n") - header_str = self._recv_buffer[:header_end].decode("utf-8") - self._recv_buffer = self._recv_buffer[header_end + 4 :] + header_end = recv_buffer.find(b"\r\n\r\n") + header_str = recv_buffer[:header_end].decode("utf-8") + recv_buffer = recv_buffer[header_end + 4 :] # Parse Content-Length content_length = 0 @@ -114,24 +135,26 @@ def recv_message(self): break if content_length == 0: + self._recv_buffer = recv_buffer return None # Read body - while len(self._recv_buffer) < content_length: + while len(recv_buffer) < content_length: try: - data = self.sock.recv(content_length - len(self._recv_buffer)) + data = sock_recv(content_length - len(recv_buffer)) if not data: self.closed = True return None - self._recv_buffer += data + recv_buffer += data except OSError as e: if hasattr(e, "errno") and e.errno in (11, 35): # EAGAIN, EWOULDBLOCK + self._recv_buffer = recv_buffer return None self.closed = True return None - body = self._recv_buffer[:content_length] - self._recv_buffer = self._recv_buffer[content_length:] + body = recv_buffer[:content_length] + self._recv_buffer = recv_buffer[content_length:] # Parse JSON try: diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index c7553a604..79f12b513 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -424,11 +424,13 @@ def _handle_source(self, seq, args): # message=f"Could not read source: {e}" ) - def _trace_function(self, frame, event, arg): + def _trace_function(self, frame, event:str, arg): """Trace function called by sys.settrace.""" + # https://docs.python.org/3/library/sys.html#sys.settrace + global _twiddel # Process any pending DAP messages frequently + self.process_pending_messages() - # Handle breakpoints and stepping if self.pdb.should_stop(frame, event, arg): self._send_stopped_event( @@ -441,6 +443,10 @@ def _trace_function(self, frame, event, arg): # Wait for continue command self.pdb.wait_for_continue() + # The trace function is invoked (with event set to 'call') whenever a new local scope is entered; + # it should return a reference to a local trace function to be used for the new scope, + # or None if the scope shouldn’t be traced. + return self._trace_function def _send_stopped_event(self, reason): diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index 9d242cd6b..9f971363d 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -3,10 +3,13 @@ import sys import time import os -import json +from micropython import const Any = object from ..common.constants import ( + STEP_INTO, + STEP_OUT, + STEP_OVER, TRACE_CALL, TRACE_LINE, TRACE_RETURN, @@ -15,14 +18,14 @@ SCOPE_GLOBALS, ) -VARREF_LOCALS = 1 -VARREF_GLOBALS = 2 -VARREF_LOCALS_SPECIAL = 3 -VARREF_GLOBALS_SPECIAL = 4 +VARREF_LOCALS = const(1) +VARREF_GLOBALS = const(2) +VARREF_LOCALS_SPECIAL = const(3) +VARREF_GLOBALS_SPECIAL = const(4) # New constants for complex variable references -VARREF_COMPLEX_BASE = 10000 # Base for complex variable references -MAX_CACHE_SIZE = 50 # Limit cache size for memory constraints +VARREF_COMPLEX_BASE = const(10000) # Base for complex variable references +MAX_CACHE_SIZE = const(50) # Limit cache size for memory constraints class VariableReferenceCache: @@ -54,17 +57,12 @@ def _cleanup_oldest(self) -> None: """Remove oldest entries to free memory - optimized for MicroPython.""" if not self.cache or not self.insertion_order: return - - # More aggressive cleanup for memory-constrained environments - to_remove = max(1, len(self.cache) // 3) # Remove 1/3 instead of 1/4 - + to_remove = max(1, len(self.cache) // 3) # Direct list slicing is more memory efficient than iteration keys_to_remove = self.insertion_order[:to_remove] - # Batch delete for efficiency for key in keys_to_remove: self.cache.pop(key, None) # Use pop with default to avoid KeyError - # Update insertion order in one operation self.insertion_order = self.insertion_order[to_remove:] @@ -188,16 +186,8 @@ def set_breakpoints(self, filename: str, breakpoints: list[dict]): if local_name != filename: self.breakpoints[local_name] = {} self._debug_print(f"[>>>] Setting breakpoints for local: {local_name}:{line}") - self.breakpoints[local_name][line] = { - "line": line, - "verified": True, - "source": {"path": filename}, - } - self.breakpoints[filename][line] = { - "line": line, - "verified": True, - "source": {"path": filename}, - } + self.breakpoints[local_name][line] = {} + self.breakpoints[filename][line] = {} actual_breakpoints.append( {"line": line, "verified": True, "source": {"path": filename}} ) @@ -208,20 +198,19 @@ def set_breakpoints(self, filename: str, breakpoints: list[dict]): def should_stop(self, frame, event: str, arg): """Determine if execution should stop at this point.""" + # HOT path - no debug printing here self.current_frame = frame self.hit_breakpoint = False - # Get frame information - filename = frame.f_code.co_filename + # Cache frame attributes to reduce lookup overhead + frame_code = frame.f_code + filename = frame_code.co_filename lineno = frame.f_lineno + # Check for exact filename match first if filename in self.breakpoints and lineno in self.breakpoints[filename]: - self._debug_print(f"[PDB] HIT BREAKPOINT (exact match) at {filename}:{lineno}") - # Record the path mapping (in this case, they're already the same) - # self.file_mappings[filename] = self._filename_as_debugger(filename) - self.hit_breakpoint = True - return True - # path/file.py matched - but not the line number - keep running + self.hit_breakpoint = True + return True else: # file not (yet) matched - this is slow so we do not want to do this often. # TODO: use builins - sys.path method to find the file @@ -230,17 +219,17 @@ def should_stop(self, frame, event: str, arg): self.breakpoints[filename] = set() # Ensure the filename is in the breakpoints dict if not filename in self.file_mappings: self.file_mappings[filename] = self._filename_as_debugger(filename) - self._debug_print( - f"[PDB] add mapping for :'{filename}' -> '{self.file_mappings[filename]}'" - ) + # self._debug_print( + # f"[PDB] add mapping for :'{filename}' -> '{self.file_mappings[filename]}'" + # ) # Check stepping - if self.step_mode == "into": + if self.step_mode == STEP_INTO: if event in (TRACE_CALL, TRACE_LINE): self.step_mode = None return True - elif self.step_mode == "over": + elif self.step_mode == STEP_OVER: if event == TRACE_LINE and frame == self.step_frame: self.step_mode = None return True @@ -251,7 +240,7 @@ def should_stop(self, frame, event: str, arg): else: self.step_mode = None - elif self.step_mode == "out": + elif self.step_mode == STEP_OUT: if event == TRACE_RETURN and frame == self.step_frame: self.step_mode = None return True From 64fb03e387bb46f28982f0e34223755c59b5b089 Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Tue, 1 Jul 2025 13:04:07 +0200 Subject: [PATCH 41/74] debugpy: Performance - aviod method access. Signed-off-by: Jos Verlinde --- .../debugpy/debugpy/server/pdb_adapter.py | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index 9f971363d..3064e54c4 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -203,33 +203,32 @@ def should_stop(self, frame, event: str, arg): self.hit_breakpoint = False # Cache frame attributes to reduce lookup overhead - frame_code = frame.f_code - filename = frame_code.co_filename - lineno = frame.f_lineno + _frame_code = frame.f_code + _filename = _frame_code.co_filename + _lineno = frame.f_lineno - # Check for exact filename match first - if filename in self.breakpoints and lineno in self.breakpoints[filename]: + # Optimize dictionary lookups - use .get() to avoid double lookup + file_breakpoints = self.breakpoints.get(_filename) + if file_breakpoints and _lineno in file_breakpoints: self.hit_breakpoint = True return True else: # file not (yet) matched - this is slow so we do not want to do this often. # TODO: use builins - sys.path method to find the file # if we have a path match , but no breakpoints - add it to the file_mappings dict simplify this check - if not filename in self.breakpoints: - self.breakpoints[filename] = set() # Ensure the filename is in the breakpoints dict - if not filename in self.file_mappings: - self.file_mappings[filename] = self._filename_as_debugger(filename) - # self._debug_print( - # f"[PDB] add mapping for :'{filename}' -> '{self.file_mappings[filename]}'" - # ) + if file_breakpoints is None: + self.breakpoints[_filename] = {} # Ensure the filename is in the breakpoints dict + if _filename not in self.file_mappings: + self.file_mappings[_filename] = self._filename_as_debugger(_filename) # Check stepping - if self.step_mode == STEP_INTO: + _step_mode = self.step_mode + if _step_mode == STEP_INTO: if event in (TRACE_CALL, TRACE_LINE): self.step_mode = None return True - elif self.step_mode == STEP_OVER: + elif _step_mode == STEP_OVER: if event == TRACE_LINE and frame == self.step_frame: self.step_mode = None return True @@ -240,7 +239,7 @@ def should_stop(self, frame, event: str, arg): else: self.step_mode = None - elif self.step_mode == STEP_OUT: + elif _step_mode == STEP_OUT: if event == TRACE_RETURN and frame == self.step_frame: self.step_mode = None return True From b6498690af8f94fa83870790048a0f1ce5e55e32 Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Tue, 1 Jul 2025 15:09:28 +0200 Subject: [PATCH 42/74] py-ecosys/debugpy: Add pause functionality. Signed-off-by: Jos Verlinde --- python-ecosys/debugpy/debugpy/server/pdb_adapter.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index 3064e54c4..92cdca715 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -98,6 +98,7 @@ def __init__(self): self.step_mode = None # None, 'over', 'into', 'out' self.step_frame = None self.step_depth = 0 + self.paused = False self.hit_breakpoint = False self.continue_event = False self.variables_cache = {} # frameId -> variables @@ -202,6 +203,14 @@ def should_stop(self, frame, event: str, arg): self.current_frame = frame self.hit_breakpoint = False + # Get frame information + filename = frame.f_code.co_filename + lineno = frame.f_lineno + # Check for exact filename match first + if self.paused or filename in self.breakpoints and lineno in self.breakpoints[filename]: + self._debug_print(f"[PDB] HIT BREAKPOINT (exact match) at {filename}:{lineno}") + # Record the path mapping (in this case, they're already the same) + # self.file_mappings[filename] = self._filename_as_debugger(filename) # Cache frame attributes to reduce lookup overhead _frame_code = frame.f_code _filename = _frame_code.co_filename @@ -271,6 +280,7 @@ def step_out(self): def pause(self): """Pause execution at next opportunity.""" # This is handled by the debug session + self.paused = True def wait_for_continue(self): """Wait for continue command (simplified implementation).""" From 2e858e60e5853788c2684c673aae52c251f94595 Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Tue, 1 Jul 2025 16:54:17 +0200 Subject: [PATCH 43/74] python-ecosys/debugpy: Add set Variable functionality. Signed-off-by: Jos Verlinde --- .../debugpy/debugpy/server/debug_session.py | 82 ++++++++++++------- .../debugpy/debugpy/server/pdb_adapter.py | 47 +++++++++++ 2 files changed, 101 insertions(+), 28 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index 79f12b513..b7a277bf4 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -15,6 +15,7 @@ CMD_STACK_TRACE, CMD_SCOPES, CMD_VARIABLES, + CMD_SET_VARIABLE, CMD_EVALUATE, CMD_DISCONNECT, CMD_CONFIGURATION_DONE, @@ -202,6 +203,8 @@ def _handle_request(self, message): self._handle_scopes(seq, args) elif command == CMD_VARIABLES: self._handle_variables(seq, args) + elif command == CMD_SET_VARIABLE: + self._handle_set_variable(seq, args) elif command == CMD_EVALUATE: self._handle_evaluate(seq, args) elif command == CMD_DISCONNECT: @@ -224,38 +227,39 @@ def _handle_initialize(self, seq, args): """Handle initialize request.""" capabilities = { "supportsConfigurationDoneRequest": True, - "supportsFunctionBreakpoints": False, - "supportsConditionalBreakpoints": False, - "supportsHitConditionalBreakpoints": False, "supportsEvaluateForHovers": True, - "supportsStepBack": False, - "supportsSetVariable": False, - "supportsRestartFrame": False, - "supportsGotoTargetsRequest": False, - "supportsStepInTargetsRequest": False, - "supportsCompletionsRequest": False, - "supportsModulesRequest": False, - "additionalModuleColumns": [], - "supportedChecksumAlgorithms": [], - "supportsRestartRequest": False, - "supportsExceptionOptions": False, - "supportsValueFormattingOptions": False, - "supportsExceptionInfoRequest": False, "supportTerminateDebuggee": True, "supportSuspendDebuggee": True, - "supportsDelayedStackTraceLoading": False, - "supportsLoadedSourcesRequest": False, - "supportsLogPoints": False, - "supportsTerminateThreadsRequest": False, - "supportsSetExpression": False, "supportsTerminateRequest": True, - "supportsDataBreakpoints": False, - "supportsReadMemoryRequest": False, - "supportsWriteMemoryRequest": False, - "supportsDisassembleRequest": False, - "supportsCancelRequest": False, - "supportsBreakpointLocationsRequest": False, - "supportsClipboardContext": False, + "supportsSetVariable": True, + + # "supportsFunctionBreakpoints": False, + # "supportsConditionalBreakpoints": False, + # "supportsHitConditionalBreakpoints": False, + # "supportsStepBack": False, + # "supportsRestartFrame": False, + # "supportsGotoTargetsRequest": False, + # "supportsStepInTargetsRequest": False, + # "supportsCompletionsRequest": False, + # "supportsModulesRequest": False, + # "additionalModuleColumns": [], + # "supportedChecksumAlgorithms": [], + # "supportsRestartRequest": False, + # "supportsExceptionOptions": False, + # "supportsValueFormattingOptions": False, + # "supportsExceptionInfoRequest": False, + # "supportsDelayedStackTraceLoading": False, + # "supportsLoadedSourcesRequest": False, + # "supportsLogPoints": False, + # "supportsTerminateThreadsRequest": False, + # "supportsSetExpression": False, + # "supportsDataBreakpoints": False, + # "supportsReadMemoryRequest": False, + # "supportsWriteMemoryRequest": False, + # "supportsDisassembleRequest": False, + # "supportsCancelRequest": False, + # "supportsBreakpointLocationsRequest": False, + # "supportsClipboardContext": False, } self.channel.send_response(CMD_INITIALIZE, seq, body=capabilities) @@ -364,6 +368,28 @@ def _handle_variables(self, seq, args): variables = self.pdb.get_variables(variables_ref) self.channel.send_response(CMD_VARIABLES, seq, body={"variables": variables}) + def _handle_set_variable(self, seq, args): + """Handle setVariable request.""" + variables_ref = args.get("variablesReference", 0) + name = args.get("name", "") + value = args.get("value", "") + + if not name: + self.channel.send_response( + CMD_SET_VARIABLE, seq, success=False, message="No variable name provided" + ) + return + + self._debug_print(f"[DAP] Processing setVariable request: name={name}, value={value}, ref={variables_ref}") + + try: + updated_variable = self.pdb.set_variable(variables_ref, name, value) + self.channel.send_response(CMD_SET_VARIABLE, seq, body=updated_variable) + except Exception as e: + self.channel.send_response( + CMD_SET_VARIABLE, seq, success=False, message=str(e) + ) + def _handle_evaluate(self, seq, args): """Handle evaluate request.""" expression = args.get("expression", "") diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index 92cdca715..6e0fdcf9b 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -719,3 +719,50 @@ def _lightweight_serialize(self, value): return repr_val except: return f"<{type_name} object>" + + def set_variable(self, variables_ref: int, name: str, value: str) -> dict[str, str | int]: + """Set a variable to a new value and return the updated variable info.""" + # Handle complex variable references (not supported for setting) + if variables_ref >= VARREF_COMPLEX_BASE: + raise Exception("Cannot set variables in complex object expansions") + + frame_id = variables_ref // 1000 + scope_type = variables_ref % 1000 + + if frame_id not in self.variables_cache: + raise Exception("Invalid frame reference") + + frame = self.variables_cache[frame_id] + + # Determine the variable dictionary to modify + if scope_type == VARREF_LOCALS or scope_type == VARREF_LOCALS_SPECIAL: + var_dict = frame.f_locals if hasattr(frame, "f_locals") else {} + elif scope_type == VARREF_GLOBALS or scope_type == VARREF_GLOBALS_SPECIAL: + var_dict = frame.f_globals if hasattr(frame, "f_globals") else {} + else: + raise Exception("Invalid scope reference") + + # Check if variable exists + if name not in var_dict: + raise Exception(f"Variable '{name}' not found in the specified scope") + + try: + # Evaluate the new value in the context of the frame + globals_dict = frame.f_globals if hasattr(frame, "f_globals") else {} + locals_dict = frame.f_locals if hasattr(frame, "f_locals") else {} + + # Try to evaluate the value as a Python expression + try: + new_value = eval(value, globals_dict, locals_dict) + except: + # If evaluation fails, treat as string literal + new_value = value + + # Set the variable + var_dict[name] = new_value + + # Return the updated variable info + return self._get_variable_info(name, new_value) + + except Exception as e: + raise Exception(f"Failed to set variable '{name}': {str(e)}") From f55fffb59e318547dddd2fc494c1d2587bd370c0 Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Tue, 1 Jul 2025 18:09:22 +0200 Subject: [PATCH 44/74] python-ecosys/debugpy: Add local variable modification while debugging. Signed-off-by: Jos Verlinde (cherry picked from commit 215300dad99c2dab2adbf8d48e7044508b17b3e9) Signed-off-by: Jos Verlinde --- .../debugpy/debugpy/common/constants.py | 1 + .../debugpy/debugpy/server/pdb_adapter.py | 78 +++++++++++++------ 2 files changed, 57 insertions(+), 22 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/common/constants.py b/python-ecosys/debugpy/debugpy/common/constants.py index 7f832eea8..bc8a4e382 100644 --- a/python-ecosys/debugpy/debugpy/common/constants.py +++ b/python-ecosys/debugpy/debugpy/common/constants.py @@ -33,6 +33,7 @@ CMD_STACK_TRACE = const("stackTrace") CMD_SCOPES = const("scopes") CMD_VARIABLES = const("variables") +CMD_SET_VARIABLE = const("setVariable") CMD_EVALUATE = const("evaluate") CMD_DISCONNECT = const("disconnect") CMD_CONFIGURATION_DONE = const("configurationDone") diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index 6e0fdcf9b..442b54747 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -721,7 +721,14 @@ def _lightweight_serialize(self, value): return f"<{type_name} object>" def set_variable(self, variables_ref: int, name: str, value: str) -> dict[str, str | int]: - """Set a variable to a new value and return the updated variable info.""" + """Set a variable to a new value and return the updated variable info. + + This function can modify both global and local variables when using a MicroPython + build with settrace and local variable modification support (sys._set_local_var). + + For global variables: Works reliably on all MicroPython builds. + For local variables: Requires MicroPython build with C-level local variable support. + """ # Handle complex variable references (not supported for setting) if variables_ref >= VARREF_COMPLEX_BASE: raise Exception("Cannot set variables in complex object expansions") @@ -729,37 +736,64 @@ def set_variable(self, variables_ref: int, name: str, value: str) -> dict[str, s frame_id = variables_ref // 1000 scope_type = variables_ref % 1000 - if frame_id not in self.variables_cache: - raise Exception("Invalid frame reference") - - frame = self.variables_cache[frame_id] + # Only allow setting variables in the topmost frame (frame_id = 0) + if frame_id != 0: + raise Exception("Variable modification is only allowed in the topmost frame") - # Determine the variable dictionary to modify - if scope_type == VARREF_LOCALS or scope_type == VARREF_LOCALS_SPECIAL: - var_dict = frame.f_locals if hasattr(frame, "f_locals") else {} - elif scope_type == VARREF_GLOBALS or scope_type == VARREF_GLOBALS_SPECIAL: - var_dict = frame.f_globals if hasattr(frame, "f_globals") else {} - else: - raise Exception("Invalid scope reference") + # Use the current frame for modification + frame = self.current_frame + if frame is None: + raise Exception("No current frame available") - # Check if variable exists - if name not in var_dict: - raise Exception(f"Variable '{name}' not found in the specified scope") + # Get the appropriate variable contexts + globals_dict = frame.f_globals if hasattr(frame, "f_globals") else {} + locals_dict = frame.f_locals if hasattr(frame, "f_locals") else {} try: - # Evaluate the new value in the context of the frame - globals_dict = frame.f_globals if hasattr(frame, "f_globals") else {} - locals_dict = frame.f_locals if hasattr(frame, "f_locals") else {} - - # Try to evaluate the value as a Python expression + # Try to evaluate the new value as a Python expression try: new_value = eval(value, globals_dict, locals_dict) except: # If evaluation fails, treat as string literal new_value = value - # Set the variable - var_dict[name] = new_value + if scope_type == VARREF_GLOBALS or scope_type == VARREF_GLOBALS_SPECIAL: + # Check if variable exists in globals + if name not in globals_dict: + raise Exception(f"Global variable '{name}' not found") + + # For global variables, direct assignment works reliably + globals_dict[name] = new_value + self._debug_print(f"[PDB] Successfully set global variable '{name}' = {new_value}") + + elif scope_type == VARREF_LOCALS or scope_type == VARREF_LOCALS_SPECIAL: + # Check if variable exists in locals + if name not in locals_dict: + raise Exception(f"Local variable '{name}' not found") + + # Try to use the frame._set_local method to set local variables + try: + if hasattr(frame, '_set_local'): + # Use the frame._set_local method (CPython-compatible API) + frame._set_local(name, new_value) + self._debug_print(f"[PDB] Successfully set local variable '{name}' = {new_value}") + else: + # Fallback error if the method is not available + raise Exception( + f"Cannot modify local variable '{name}'. " + f"This MicroPython build doesn't support local variable modification. " + f"Please use a MicroPython build with settrace and local variable support." + ) + except Exception as inner_e: + # If frame.set_local fails, provide detailed error + raise Exception( + f"Failed to modify local variable '{name}': {str(inner_e)}. " + f"Local variables in MicroPython are stored in internal code_state->state[] slots. " + f"Consider using global variables for reliable modification during debugging." + ) + + else: + raise Exception("Invalid scope reference") # Return the updated variable info return self._get_variable_info(name, new_value) From 9ef4f771b5b20322c918a38501d54c7fcbfd4d53 Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Tue, 1 Jul 2025 22:21:43 +0200 Subject: [PATCH 45/74] python-ecosys/debugpy: Format code with ruff. Signed-off-by: Jos Verlinde --- python-ecosys/debugpy/dap_monitor.py | 66 ++++++----- .../debugpy/debugpy/common/constants.py | 1 + .../debugpy/debugpy/server/debug_session.py | 23 ++-- .../debugpy/debugpy/server/pdb_adapter.py | 107 +++++++++--------- python-ecosys/debugpy/demo.py | 14 ++- python-ecosys/debugpy/test_vscode.py | 8 +- 6 files changed, 124 insertions(+), 95 deletions(-) diff --git a/python-ecosys/debugpy/dap_monitor.py b/python-ecosys/debugpy/dap_monitor.py index 93d02ddf7..85455c9a6 100644 --- a/python-ecosys/debugpy/dap_monitor.py +++ b/python-ecosys/debugpy/dap_monitor.py @@ -8,8 +8,9 @@ import sys import argparse + class DAPMonitor: - def __init__(self, listen_port=5679, target_host='127.0.0.1', target_port=5678): + def __init__(self, listen_port=5679, target_host="127.0.0.1", target_port=5678): self.disconnect = False self.listen_port = listen_port self.target_host = target_host @@ -26,7 +27,7 @@ def start(self): # Create listening socket listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - listener.bind(('127.0.0.1', self.listen_port)) + listener.bind(("127.0.0.1", self.listen_port)) listener.listen(1) print(f"Listening for VS Code connection on port {self.listen_port}...") @@ -90,11 +91,11 @@ def receive_dap_message(self, sock, source): header += byte # Parse content length - header_str = header.decode('utf-8') + header_str = header.decode("utf-8") content_length = 0 - for line in header_str.split('\r\n'): - if line.startswith('Content-Length:'): - content_length = int(line.split(':', 1)[1].strip()) + for line in header_str.split("\r\n"): + if line.startswith("Content-Length:"): + content_length = int(line.split(":", 1)[1].strip()) break if content_length == 0: @@ -113,7 +114,7 @@ def receive_dap_message(self, sock, source): self.log_dap_message(source, message) # Check for disconnect command if message: - if "disconnect" == message.get('command', message.get('event', 'unknown')): + if "disconnect" == message.get("command", message.get("event", "unknown")): print(f"\n[{source}] Disconnect command received, stopping monitor.") self.disconnect = True return header + content @@ -124,7 +125,7 @@ def receive_dap_message(self, sock, source): def parse_dap(self, source, content): """Parse DAP message and log it.""" try: - message = json.loads(content.decode('utf-8')) + message = json.loads(content.decode("utf-8")) return message except json.JSONDecodeError: print(f"\n[{source}] Invalid JSON: {content}") @@ -132,28 +133,28 @@ def parse_dap(self, source, content): def log_dap_message(self, source, message): """Log DAP message details.""" - msg_type = message.get('type', 'unknown') - command = message.get('command', message.get('event', 'unknown')) - seq = message.get('seq', 0) + msg_type = message.get("type", "unknown") + command = message.get("command", message.get("event", "unknown")) + seq = message.get("seq", 0) print(f"\n[{source}] {msg_type.upper()}: {command} (seq={seq})") - if msg_type == 'request': - args = message.get('arguments', {}) + if msg_type == "request": + args = message.get("arguments", {}) if args: print(f" Arguments: {json.dumps(args, indent=2)}") - elif msg_type == 'response': - success = message.get('success', False) - req_seq = message.get('request_seq', 0) + elif msg_type == "response": + success = message.get("success", False) + req_seq = message.get("request_seq", 0) print(f" Success: {success}, Request Seq: {req_seq}") - body = message.get('body') + body = message.get("body") if body: print(f" Body: {json.dumps(body, indent=2)}") - msg = message.get('message') + msg = message.get("message") if msg: print(f" Message: {msg}") - elif msg_type == 'event': - body = message.get('body', {}) + elif msg_type == "event": + body = message.get("body", {}) if body: print(f" Body: {json.dumps(body, indent=2)}") @@ -171,17 +172,28 @@ def cleanup(self): if self.server_sock: self.server_sock.close() -if __name__ == "__main__": +if __name__ == "__main__": parser = argparse.ArgumentParser(description="DAP protocol monitor proxy") - parser.add_argument("--target-host", "--th", default="127.0.0.1", help="Target debugpy host (default: 127.0.0.1)") - parser.add_argument("--target-port", "--tp", type=int, default=5678, help="Target debugpy port (default: 5678)") - parser.add_argument("--listen-port", "--lp", type=int, default=5679, help="Port to listen for VS Code (default: 5679)") + parser.add_argument( + "--target-host", + "--th", + default="127.0.0.1", + help="Target debugpy host (default: 127.0.0.1)", + ) + parser.add_argument( + "--target-port", "--tp", type=int, default=5678, help="Target debugpy port (default: 5678)" + ) + parser.add_argument( + "--listen-port", + "--lp", + type=int, + default=5679, + help="Port to listen for VS Code (default: 5679)", + ) args = parser.parse_args() monitor = DAPMonitor( - listen_port=args.listen_port, - target_host=args.target_host, - target_port=args.target_port + listen_port=args.listen_port, target_host=args.target_host, target_port=args.target_port ) monitor.start() diff --git a/python-ecosys/debugpy/debugpy/common/constants.py b/python-ecosys/debugpy/debugpy/common/constants.py index bc8a4e382..44c12334c 100644 --- a/python-ecosys/debugpy/debugpy/common/constants.py +++ b/python-ecosys/debugpy/debugpy/common/constants.py @@ -1,4 +1,5 @@ """Constants used throughout debugpy.""" + from micropython import const # Default networking settings diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index b7a277bf4..6869c8790 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -232,7 +232,6 @@ def _handle_initialize(self, seq, args): "supportSuspendDebuggee": True, "supportsTerminateRequest": True, "supportsSetVariable": True, - # "supportsFunctionBreakpoints": False, # "supportsConditionalBreakpoints": False, # "supportsHitConditionalBreakpoints": False, @@ -373,22 +372,22 @@ def _handle_set_variable(self, seq, args): variables_ref = args.get("variablesReference", 0) name = args.get("name", "") value = args.get("value", "") - + if not name: self.channel.send_response( CMD_SET_VARIABLE, seq, success=False, message="No variable name provided" ) return - - self._debug_print(f"[DAP] Processing setVariable request: name={name}, value={value}, ref={variables_ref}") - + + self._debug_print( + f"[DAP] Processing setVariable request: name={name}, value={value}, ref={variables_ref}" + ) + try: updated_variable = self.pdb.set_variable(variables_ref, name, value) self.channel.send_response(CMD_SET_VARIABLE, seq, body=updated_variable) except Exception as e: - self.channel.send_response( - CMD_SET_VARIABLE, seq, success=False, message=str(e) - ) + self.channel.send_response(CMD_SET_VARIABLE, seq, success=False, message=str(e)) def _handle_evaluate(self, seq, args): """Handle evaluate request.""" @@ -450,12 +449,12 @@ def _handle_source(self, seq, args): # message=f"Could not read source: {e}" ) - def _trace_function(self, frame, event:str, arg): + def _trace_function(self, frame, event: str, arg): """Trace function called by sys.settrace.""" # https://docs.python.org/3/library/sys.html#sys.settrace global _twiddel # Process any pending DAP messages frequently - + self.process_pending_messages() # Handle breakpoints and stepping if self.pdb.should_stop(frame, event, arg): @@ -469,8 +468,8 @@ def _trace_function(self, frame, event:str, arg): # Wait for continue command self.pdb.wait_for_continue() - # The trace function is invoked (with event set to 'call') whenever a new local scope is entered; - # it should return a reference to a local trace function to be used for the new scope, + # The trace function is invoked (with event set to 'call') whenever a new local scope is entered; + # it should return a reference to a local trace function to be used for the new scope, # or None if the scope shouldn’t be traced. return self._trace_function diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index 442b54747..4de655f4e 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -3,9 +3,8 @@ import sys import time import os -from micropython import const +from micropython import const # type: ignore[import-untyped] -Any = object from ..common.constants import ( STEP_INTO, STEP_OUT, @@ -18,6 +17,8 @@ SCOPE_GLOBALS, ) +Any = object + VARREF_LOCALS = const(1) VARREF_GLOBALS = const(2) VARREF_LOCALS_SPECIAL = const(3) @@ -57,7 +58,7 @@ def _cleanup_oldest(self) -> None: """Remove oldest entries to free memory - optimized for MicroPython.""" if not self.cache or not self.insertion_order: return - to_remove = max(1, len(self.cache) // 3) + to_remove = max(1, len(self.cache) // 3) # Direct list slicing is more memory efficient than iteration keys_to_remove = self.insertion_order[:to_remove] # Batch delete for efficiency @@ -215,7 +216,7 @@ def should_stop(self, frame, event: str, arg): _frame_code = frame.f_code _filename = _frame_code.co_filename _lineno = frame.f_lineno - + # Optimize dictionary lookups - use .get() to avoid double lookup file_breakpoints = self.breakpoints.get(_filename) if file_breakpoints and _lineno in file_breakpoints: @@ -305,11 +306,11 @@ def get_stack_trace(self): frame = self.current_frame frame_id = 0 - self._debug_print("=" * 40 ) - self._debug_print(f"[PDB] file mappings: {repr(self.file_mappings)} " ) - self._debug_print(f"[PDB] path mappings: {repr(self.path_mappings)}" ) - self._debug_print(f"[PDB] breakpoints: {repr(self.breakpoints)}" ) - self._debug_print("=" * 40 ) + self._debug_print("=" * 40) + self._debug_print(f"[PDB] file mappings: {repr(self.file_mappings)} ") + self._debug_print(f"[PDB] path mappings: {repr(self.path_mappings)}") + self._debug_print(f"[PDB] breakpoints: {repr(self.breakpoints)}") + self._debug_print("=" * 40) while frame: filename = frame.f_code.co_filename @@ -320,7 +321,6 @@ def get_stack_trace(self): else: hint = "normal" - # Use the VS Code path if we have a mapping, otherwise use the original path debugger_path = self._filename_as_debugger(filename) # Create StackFrame info @@ -479,7 +479,7 @@ def _get_variable_info_fast(self, name: str, value: Any) -> dict[str, str | int] if self._is_expandable(value): var_ref = self.var_cache.add_variable(value) preview = self._get_preview(value) # Always use consistent preview - + # Use pre-calculated length for better performance length = 0 try: @@ -525,12 +525,7 @@ def _get_variable_info_fast(self, name: str, value: Any) -> dict[str, str | int] "variablesReference": 0, } except Exception: - return { - "name": name, - "value": "", - "type": "unknown", - "variablesReference": 0 - } + return {"name": name, "value": "", "type": "unknown", "variablesReference": 0} def _expand_complex_variable(self, ref_id: int) -> list[dict[str, str | int]]: """Expand a complex variable into its child elements - optimized for memory.""" @@ -549,24 +544,28 @@ def _expand_complex_variable(self, ref_id: int) -> list[dict[str, str | int]]: key_str = str(key)[:50] # Limit key string length variables.append(self._get_variable_info(key_str, val)) if len(items) > max_items: - variables.append({ - "name": f"<{len(items) - max_items} more items>", - "value": "...", - "type": "info", - "variablesReference": 0, - }) + variables.append( + { + "name": f"<{len(items) - max_items} more items>", + "value": "...", + "type": "info", + "variablesReference": 0, + } + ) elif isinstance(value, (list, tuple)): # Limit list/tuple expansion max_items = min(len(value), 100) # Limit to 100 items max for i in range(max_items): variables.append(self._get_variable_info(f"[{i}]", value[i])) if len(value) > max_items: - variables.append({ - "name": f"<{len(value) - max_items} more items>", - "value": "...", - "type": "info", - "variablesReference": 0, - }) + variables.append( + { + "name": f"<{len(value) - max_items} more items>", + "value": "...", + "type": "info", + "variablesReference": 0, + } + ) elif isinstance(value, set): # Handle set elements with size limit items = list(value) # Convert once @@ -574,20 +573,24 @@ def _expand_complex_variable(self, ref_id: int) -> list[dict[str, str | int]]: for i in range(max_items): variables.append(self._get_variable_info(f"<{i}>", items[i])) if len(items) > max_items: - variables.append({ - "name": f"<{len(items) - max_items} more items>", - "value": "...", - "type": "info", - "variablesReference": 0, - }) + variables.append( + { + "name": f"<{len(items) - max_items} more items>", + "value": "...", + "type": "info", + "variablesReference": 0, + } + ) except Exception as e: # Return error info for debugging - variables.append({ - "name": "error", - "value": f"Failed to expand: {str(e)[:50]}", # Limit error message length - "type": "error", - "variablesReference": 0, - }) + variables.append( + { + "name": "error", + "value": f"Failed to expand: {str(e)[:50]}", # Limit error message length + "type": "error", + "variablesReference": 0, + } + ) return variables @@ -678,10 +681,10 @@ def _lightweight_serialize(self, value): elif isinstance(value, str): # Simple escaping for strings - avoid full JSON complexity if len(value) > 30: - escaped = value[:27].replace('"', '\\"').replace('\n', '\\n') + escaped = value[:27].replace('"', '\\"').replace("\n", "\\n") return f'"{escaped}..."' else: - escaped = value.replace('"', '\\"').replace('\n', '\\n') + escaped = value.replace('"', '\\"').replace("\n", "\\n") return f'"{escaped}"' elif isinstance(value, (list, tuple)): if len(value) == 0: @@ -722,10 +725,10 @@ def _lightweight_serialize(self, value): def set_variable(self, variables_ref: int, name: str, value: str) -> dict[str, str | int]: """Set a variable to a new value and return the updated variable info. - + This function can modify both global and local variables when using a MicroPython build with settrace and local variable modification support (sys._set_local_var). - + For global variables: Works reliably on all MicroPython builds. For local variables: Requires MicroPython build with C-level local variable support. """ @@ -761,22 +764,24 @@ def set_variable(self, variables_ref: int, name: str, value: str) -> dict[str, s # Check if variable exists in globals if name not in globals_dict: raise Exception(f"Global variable '{name}' not found") - + # For global variables, direct assignment works reliably globals_dict[name] = new_value self._debug_print(f"[PDB] Successfully set global variable '{name}' = {new_value}") - + elif scope_type == VARREF_LOCALS or scope_type == VARREF_LOCALS_SPECIAL: # Check if variable exists in locals if name not in locals_dict: raise Exception(f"Local variable '{name}' not found") - + # Try to use the frame._set_local method to set local variables try: - if hasattr(frame, '_set_local'): + if hasattr(frame, "_set_local"): # Use the frame._set_local method (CPython-compatible API) frame._set_local(name, new_value) - self._debug_print(f"[PDB] Successfully set local variable '{name}' = {new_value}") + self._debug_print( + f"[PDB] Successfully set local variable '{name}' = {new_value}" + ) else: # Fallback error if the method is not available raise Exception( @@ -791,7 +796,7 @@ def set_variable(self, variables_ref: int, name: str, value: str) -> dict[str, s f"Local variables in MicroPython are stored in internal code_state->state[] slots. " f"Consider using global variables for reliable modification during debugging." ) - + else: raise Exception("Invalid scope reference") diff --git a/python-ecosys/debugpy/demo.py b/python-ecosys/debugpy/demo.py index 02a927257..fd88c0272 100644 --- a/python-ecosys/debugpy/demo.py +++ b/python-ecosys/debugpy/demo.py @@ -2,16 +2,19 @@ """Simple demo of MicroPython debugpy functionality.""" import sys -sys.path.insert(0, '.') + +sys.path.insert(0, ".") import debugpy + def simple_function(a, b): """A simple function to demonstrate debugging.""" result = a + b print(f"Computing {a} + {b} = {result}") return result + def main(): print("MicroPython debugpy Demo") print("========================") @@ -21,11 +24,11 @@ def main(): print("1. Testing trace functionality:") def trace_function(frame, event, arg): - if event == 'call': + if event == "call": print(f" -> Entering function: {frame.f_code.co_name}") - elif event == 'line': + elif event == "line": print(f" -> Executing line {frame.f_lineno} in {frame.f_code.co_name}") - elif event == 'return': + elif event == "return": print(f" -> Returning from {frame.f_code.co_name} with value: {arg}") return trace_function @@ -46,6 +49,7 @@ def trace_function(frame, event, arg): # Test PDB adapter from debugpy.server.pdb_adapter import PdbAdapter + pdb = PdbAdapter() # Set some mock breakpoints @@ -54,6 +58,7 @@ def trace_function(frame, event, arg): # Test messaging from debugpy.common.messaging import JsonMessageChannel + print(" JsonMessageChannel available") print() @@ -64,5 +69,6 @@ def trace_function(frame, event, arg): print(" - Connect VS Code using the 'Attach to MicroPython' configuration") print(" - Set breakpoints and debug normally") + if __name__ == "__main__": main() diff --git a/python-ecosys/debugpy/test_vscode.py b/python-ecosys/debugpy/test_vscode.py index 9a5672822..1d24fac81 100644 --- a/python-ecosys/debugpy/test_vscode.py +++ b/python-ecosys/debugpy/test_vscode.py @@ -3,13 +3,14 @@ import sys -sys.path.insert(0, '.') +sys.path.insert(0, ".") import debugpy foo = 42 bar = "Hello, MicroPython!" + def fibonacci(n): """Calculate fibonacci number (iterative for efficiency).""" if n <= 1: @@ -19,6 +20,7 @@ def fibonacci(n): a, b = b, a + b return b + def debuggable_code(): """The actual code we want to debug - wrapped in a function so sys.settrace will trace it.""" global foo @@ -33,6 +35,7 @@ def debuggable_code(): print(f"fibonacci({num}) = {result}") print(sys.implementation) import machine + print(dir(machine)) # Test manual breakpoint @@ -42,6 +45,7 @@ def debuggable_code(): print("Test completed successfully!") + def main(): print("MicroPython VS Code Debugging Test") print("==================================") @@ -64,6 +68,7 @@ def main(): # Give VS Code a moment to set breakpoints after attach print("\nGiving VS Code time to set breakpoints...") import time + time.sleep(2) # Call the debuggable code function so it gets traced @@ -74,5 +79,6 @@ def main(): except Exception as e: print(f"Error: {e}") + if __name__ == "__main__": main() From 5cc1ff517ff9a3e8b2f3b9314c9e8d181b61b6d3 Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Tue, 1 Jul 2025 22:38:50 +0200 Subject: [PATCH 46/74] python-ecosys/debugpy: Code cleanup. Signed-off-by: Jos Verlinde --- .../debugpy/debugpy/server/debug_session.py | 39 ++++++++-------- .../debugpy/debugpy/server/pdb_adapter.py | 46 ++++++++----------- 2 files changed, 39 insertions(+), 46 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index 6869c8790..17904fb49 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -1,38 +1,39 @@ """Main debug session handling DAP protocol communication.""" import sys -from ..common.messaging import JsonMessageChannel + from ..common.constants import ( - CMD_INITIALIZE, - CMD_LAUNCH, CMD_ATTACH, - CMD_SET_BREAKPOINTS, + CMD_CONFIGURATION_DONE, CMD_CONTINUE, + CMD_DISCONNECT, + CMD_EVALUATE, + CMD_INITIALIZE, + CMD_LAUNCH, CMD_NEXT, - CMD_STEP_IN, - CMD_STEP_OUT, CMD_PAUSE, - CMD_STACK_TRACE, CMD_SCOPES, - CMD_VARIABLES, + CMD_SET_BREAKPOINTS, CMD_SET_VARIABLE, - CMD_EVALUATE, - CMD_DISCONNECT, - CMD_CONFIGURATION_DONE, - CMD_THREADS, CMD_SOURCE, + CMD_STACK_TRACE, + CMD_STEP_IN, + CMD_STEP_OUT, + CMD_THREADS, + CMD_VARIABLES, + EVENT_CONTINUED, EVENT_INITIALIZED, EVENT_STOPPED, - EVENT_CONTINUED, EVENT_TERMINATED, STOP_REASON_BREAKPOINT, - STOP_REASON_STEP, STOP_REASON_PAUSE, + STOP_REASON_STEP, TRACE_CALL, + TRACE_EXCEPTION, TRACE_LINE, TRACE_RETURN, - TRACE_EXCEPTION, ) +from ..common.messaging import JsonMessageChannel from .pdb_adapter import PdbAdapter @@ -43,7 +44,7 @@ def __init__(self, client_socket): self.debug_logging = False # Initialize first self.channel = JsonMessageChannel(client_socket, self._debug_print) self.pdb = PdbAdapter() - self.pdb._debug_session = self # Allow PDB to process messages during wait # type: ignore + self.pdb._debug_session = self # Allow PDB to process messages during wait # type: ignore[assignment] self.initialized = False self.connected = True self.thread_id = 1 # Simple single-thread model @@ -393,7 +394,7 @@ def _handle_evaluate(self, seq, args): """Handle evaluate request.""" expression = args.get("expression", "") frame_id = args.get("frameId") - context = args.get("context", "watch") + # context = args.get("context", "watch") if not expression: self.channel.send_response( CMD_EVALUATE, seq, success=False, message="No expression provided" @@ -429,7 +430,7 @@ def _handle_source(self, seq, args): source = args.get("source", {}) source_path = source.get("path", "") if self._baremetal or not source_path: - # BUGBUG: unable to read the source on ESP32 + # BUG: unable to read the source on ESP32 # Possible an effect of the import / inialization sequence ? # Nothe that other source files ( other.py) do not seem to get requested in the same way self.channel.send_response(CMD_SOURCE, seq, success=False) @@ -470,7 +471,7 @@ def _trace_function(self, frame, event: str, arg): # The trace function is invoked (with event set to 'call') whenever a new local scope is entered; # it should return a reference to a local trace function to be used for the new scope, - # or None if the scope shouldn’t be traced. + # or None if the scope shouldn't be traced. return self._trace_function diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index 4de655f4e..73d4c3ce6 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -1,20 +1,21 @@ """PDB adapter for integrating with MicroPython's trace system.""" +import os import sys import time -import os + from micropython import const # type: ignore[import-untyped] from ..common.constants import ( + SCOPE_GLOBALS, + SCOPE_LOCALS, STEP_INTO, STEP_OUT, STEP_OVER, TRACE_CALL, + TRACE_EXCEPTION, TRACE_LINE, TRACE_RETURN, - TRACE_EXCEPTION, - SCOPE_LOCALS, - SCOPE_GLOBALS, ) Any = object @@ -92,9 +93,8 @@ class PdbAdapter: """Adapter between DAP protocol and MicroPython's sys.settrace functionality.""" def __init__(self): - self.breakpoints: dict[ - str, dict[int, dict] - ] = {} # filename -> {line_no: breakpoint_info} # todo - simplify - reduce info stored + self.breakpoints: dict[str, dict[int, dict]] = {} + # filename -> {line_no: breakpoint_info} # todo - simplify self.current_frame = None self.step_mode = None # None, 'over', 'into', 'out' self.step_frame = None @@ -105,16 +105,14 @@ def __init__(self): self.variables_cache = {} # frameId -> variables self.var_cache = VariableReferenceCache() # Enhanced variable reference cache self.frame_id_counter = 1 - self.path_mappings: list[ - tuple[str, str] - ] = [] # runtime_path -> vscode_path mapping # todo: move to session level - self.file_mappings: dict[ - str, str - ] = {} # runtime_path -> vscode_path mapping # todo : merge with .breakpoints + self.path_mappings: list[tuple[str, str]] = [] + # list of [runtime_path -> vscode_path mapping] + self.file_mappings: dict[str, str] = {} + # runtime_path -> vscode_path mapping # todo : merge with .breakpoints def _debug_print(self, message): """Print debug message only if debug logging is enabled.""" - if hasattr(self, "_debug_session") and self._debug_session.debug_logging: # type: ignore + if hasattr(self, "_debug_session") and self._debug_session.debug_logging: # type: ignore[attr-defined] print(message) def _normalize_path(self, path: str): @@ -208,7 +206,7 @@ def should_stop(self, frame, event: str, arg): filename = frame.f_code.co_filename lineno = frame.f_lineno # Check for exact filename match first - if self.paused or filename in self.breakpoints and lineno in self.breakpoints[filename]: + if self.paused or (filename in self.breakpoints and lineno in self.breakpoints[filename]): self._debug_print(f"[PDB] HIT BREAKPOINT (exact match) at {filename}:{lineno}") # Record the path mapping (in this case, they're already the same) # self.file_mappings[filename] = self._filename_as_debugger(filename) @@ -224,7 +222,7 @@ def should_stop(self, frame, event: str, arg): return True else: # file not (yet) matched - this is slow so we do not want to do this often. - # TODO: use builins - sys.path method to find the file + # TODO: use sys.path[] method to find the file, does not work for frozen .... # if we have a path match , but no breakpoints - add it to the file_mappings dict simplify this check if file_breakpoints is None: self.breakpoints[_filename] = {} # Ensure the filename is in the breakpoints dict @@ -294,7 +292,7 @@ def wait_for_continue(self): while not self.continue_event: # Process any pending DAP messages (scopes, variables, etc.) if hasattr(self, "_debug_session"): - self._debug_session.process_pending_messages() # type: ignore + self._debug_session.process_pending_messages() # type: ignore[arg-type] time.sleep(0.01) def get_stack_trace(self): @@ -306,12 +304,6 @@ def get_stack_trace(self): frame = self.current_frame frame_id = 0 - self._debug_print("=" * 40) - self._debug_print(f"[PDB] file mappings: {repr(self.file_mappings)} ") - self._debug_print(f"[PDB] path mappings: {repr(self.path_mappings)}") - self._debug_print(f"[PDB] breakpoints: {repr(self.breakpoints)}") - self._debug_print("=" * 40) - while frame: filename = frame.f_code.co_filename name = frame.f_code.co_name @@ -483,7 +475,7 @@ def _get_variable_info_fast(self, name: str, value: Any) -> dict[str, str | int] # Use pre-calculated length for better performance length = 0 try: - length = len(value) # type: ignore + length = len(value) # type: ignore[arg-type] except: pass @@ -670,7 +662,7 @@ def cleanup(self): if hasattr(sys, "settrace"): sys.settrace(None) - def _lightweight_serialize(self, value): + def _lightweight_serialize(self, value): # noqa: PLR0911 """Lightweight serialization optimized for MicroPython memory constraints.""" if value is None: return "None" @@ -792,7 +784,7 @@ def set_variable(self, variables_ref: int, name: str, value: str) -> dict[str, s except Exception as inner_e: # If frame.set_local fails, provide detailed error raise Exception( - f"Failed to modify local variable '{name}': {str(inner_e)}. " + f"Failed to modify local variable '{name}': {inner_e}. " f"Local variables in MicroPython are stored in internal code_state->state[] slots. " f"Consider using global variables for reliable modification during debugging." ) @@ -804,4 +796,4 @@ def set_variable(self, variables_ref: int, name: str, value: str) -> dict[str, s return self._get_variable_info(name, new_value) except Exception as e: - raise Exception(f"Failed to set variable '{name}': {str(e)}") + raise Exception(f"Failed to set variable '{name}': {e}") From 1efe7290ad72fad77625e605bd6a432759cb1a4e Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Mon, 14 Jul 2025 17:07:36 +0200 Subject: [PATCH 47/74] debugpy: Capitalise the "Special" variables scope name. Signed-off-by: Jos Verlinde --- python-ecosys/debugpy/debugpy/server/pdb_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index 73d4c3ce6..b55772f6d 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -592,7 +592,7 @@ def _var_error(name: str): @staticmethod def _special_vars(varref: int): - return {"name": "special", "value": "", "variablesReference": varref} + return {"name": "Special", "value": "", "variablesReference": varref} def get_variables(self, variables_ref): """Get variables for a scope with enhanced complex variable support.""" From 01665cd4f2746e41c80db188167b0e36691dd606 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Sun, 5 Jul 2026 21:33:47 +1000 Subject: [PATCH 48/74] debugpy: Reassemble DAP messages split across recv() calls. recv_message() stripped the header from the receive buffer as soon as the CRLF/CRLF terminator was found, but only persisted buffer state on some partial-read paths. When a message body arrived in a later read than its header, the parsed-header state was lost and framing desynchronised for the rest of the connection. Keep the header and body together in the buffer until the whole message (header + Content-Length bytes) is present, then slice it off. Treat an empty recv as a peer close and EAGAIN/EWOULDBLOCK as "try later". Signed-off-by: Andrew Leech --- .../debugpy/debugpy/common/messaging.py | 131 +++++++----------- 1 file changed, 53 insertions(+), 78 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/common/messaging.py b/python-ecosys/debugpy/debugpy/common/messaging.py index eb7df4dd2..7d704f6a8 100644 --- a/python-ecosys/debugpy/debugpy/common/messaging.py +++ b/python-ecosys/debugpy/debugpy/common/messaging.py @@ -82,94 +82,69 @@ def send_event(self, event, **kwargs): self.send_message(MSG_TYPE_EVENT, event, **kwargs) def recv_message(self): - """Receive a DAP message.""" + """Receive a DAP message, or None if a full one isn't available yet. + + Called repeatedly against a socket with a short recv timeout (see + `DebugSession.process_pending_messages`), so a single message's + header and body routinely arrive across several calls. Everything + read so far - including an already-located header - is kept in + `self._recv_buffer` verbatim until the *entire* message (header + + `Content-Length` body bytes) is available, and only then is it + parsed and sliced off. Parsing the header again on each call is + cheap and avoids having to separately persist "header already + parsed, N body bytes still outstanding" state between calls: a + prior version stripped the header out of the buffer as soon as it + was found, which discarded that state and desynchronised framing + for the rest of the connection whenever the body arrived in a + later read than the header. + """ if self.closed: return None - # Quick bail-out: if buffer is empty, do a non-blocking peek to see if data is available - if not self._recv_buffer: - try: - # Try to read a small amount non-blocking to see if anything is available - peek_data = self.sock.recv(1) - if not peek_data: - return None # No data available - # Put the peeked data back into buffer - self._recv_buffer = peek_data - except OSError as e: - # Handle non-blocking socket errors (no data available) - if hasattr(e, "errno") and e.errno in (11, 35): # EAGAIN, EWOULDBLOCK - return None # No data available, quick exit - # Other errors + # Non-blocking top-up: pull in whatever is available right now + # without blocking if there's nothing new yet. + try: + data = self.sock.recv(4096) + if not data: + # A truly empty read (as opposed to EAGAIN/EWOULDBLOCK, + # handled below) means the peer closed the connection. + self.closed = True + return None + self._recv_buffer += data + except OSError as e: + if not (hasattr(e, "errno") and e.errno in (11, 35)): # EAGAIN, EWOULDBLOCK self.closed = True return None + # No new data available right now - fall through and try to + # parse a complete message out of whatever is already buffered. - # Cache frequently accessed attributes recv_buffer = self._recv_buffer - sock_recv = self.sock.recv + header_end = recv_buffer.find(b"\r\n\r\n") + if header_end < 0: + return None # Header not fully received yet. - try: - # Read headers - while b"\r\n\r\n" not in recv_buffer: - try: - data = sock_recv(1024) - if not data: - self.closed = True - return None - recv_buffer += data - except OSError as e: - # Handle timeout and other socket errors - if hasattr(e, "errno") and e.errno in (11, 35): # EAGAIN, EWOULDBLOCK - return None # No data available - self.closed = True - return None - - header_end = recv_buffer.find(b"\r\n\r\n") - header_str = recv_buffer[:header_end].decode("utf-8") - recv_buffer = recv_buffer[header_end + 4 :] - - # Parse Content-Length - content_length = 0 - for line in header_str.split("\r\n"): - if line.startswith("Content-Length:"): - content_length = int(line.split(":", 1)[1].strip()) - break - - if content_length == 0: - self._recv_buffer = recv_buffer - return None + header_str = recv_buffer[:header_end].decode("utf-8") + content_length = 0 + for line in header_str.split("\r\n"): + if line.startswith("Content-Length:"): + content_length = int(line.split(":", 1)[1].strip()) + break - # Read body - while len(recv_buffer) < content_length: - try: - data = sock_recv(content_length - len(recv_buffer)) - if not data: - self.closed = True - return None - recv_buffer += data - except OSError as e: - if hasattr(e, "errno") and e.errno in (11, 35): # EAGAIN, EWOULDBLOCK - self._recv_buffer = recv_buffer - return None - self.closed = True - return None - - body = recv_buffer[:content_length] - self._recv_buffer = recv_buffer[content_length:] - - # Parse JSON - try: - message = json.loads(body.decode("utf-8")) - self._debug_print( - f"[DAP] Successfully received message: {message.get('type')} {message.get('command', message.get('event', 'unknown'))}" - ) - return message - except (ValueError, UnicodeDecodeError) as e: - print(f"[DAP] JSON parse error: {e}") - return None + body_start = header_end + 4 + if len(recv_buffer) < body_start + content_length: + return None # Body not fully received yet. - except OSError as e: - print(f"[DAP] Socket error in recv_message: {e}") - self.closed = True + body = recv_buffer[body_start : body_start + content_length] + self._recv_buffer = recv_buffer[body_start + content_length :] + + try: + message = json.loads(body.decode("utf-8")) + self._debug_print( + f"[DAP] Successfully received message: {message.get('type')} {message.get('command', message.get('event', 'unknown'))}" + ) + return message + except (ValueError, UnicodeDecodeError) as e: + print(f"[DAP] JSON parse error: {e}") return None def close(self): From e4c8169f5e050237c970fbc8277c7bb025e31e2f Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Sun, 5 Jul 2026 21:33:47 +1000 Subject: [PATCH 49/74] debugpy: Add wait_for_client, a capability probe and read-only locals. - wait_for_client() blocks until the DAP client sends configurationDone, draining the socket so breakpoints set beforehand are honoured; it replaces a fixed sleep. Bounded timeout, logged rather than silent. - A runtime capability probe (settrace / save_names / set_local / f_back) derived by exercising the interpreter, never inferred from a build or variant name; exposed via get_capabilities(). - Local variables are marked read-only (DAP presentationHint) when the firmware lacks frame._set_local, so clients do not offer an edit that cannot work; globals stay editable. - listen() resolves the actually-bound port and never advertises port 0. Signed-off-by: Andrew Leech --- python-ecosys/debugpy/debugpy/__init__.py | 13 ++- .../debugpy/debugpy/common/constants.py | 5 + python-ecosys/debugpy/debugpy/public_api.py | 54 ++++++++-- .../debugpy/debugpy/server/debug_session.py | 98 ++++++++++++++++++- .../debugpy/debugpy/server/pdb_adapter.py | 40 +++++--- 5 files changed, 186 insertions(+), 24 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/__init__.py b/python-ecosys/debugpy/debugpy/__init__.py index 3912a49a5..cce6cb870 100644 --- a/python-ecosys/debugpy/debugpy/__init__.py +++ b/python-ecosys/debugpy/debugpy/__init__.py @@ -7,7 +7,15 @@ __version__ = "0.1.0" -from .public_api import listen, wait_for_client, breakpoint, debug_this_thread +from .public_api import ( + breakpoint, + debug_this_thread, + disconnect, + get_capabilities, + is_client_connected, + listen, + wait_for_client, +) from .common.constants import DEFAULT_HOST, DEFAULT_PORT __all__ = [ @@ -15,6 +23,9 @@ "DEFAULT_PORT", "breakpoint", "debug_this_thread", + "disconnect", + "get_capabilities", + "is_client_connected", "listen", "wait_for_client", ] diff --git a/python-ecosys/debugpy/debugpy/common/constants.py b/python-ecosys/debugpy/debugpy/common/constants.py index 44c12334c..5d9a52204 100644 --- a/python-ecosys/debugpy/debugpy/common/constants.py +++ b/python-ecosys/debugpy/debugpy/common/constants.py @@ -67,3 +67,8 @@ # Scope types SCOPE_LOCALS = const("locals") SCOPE_GLOBALS = const("globals") + +# Bounded wait for the DAP client to send configurationDone (seconds). There is +# no server thread, so a hang here would spin forever with no diagnostic; a +# timeout with a clear message replaces a silent guessed delay. +WAIT_FOR_CLIENT_TIMEOUT_S = const(30) diff --git a/python-ecosys/debugpy/debugpy/public_api.py b/python-ecosys/debugpy/debugpy/public_api.py index 06b928965..59577a44e 100644 --- a/python-ecosys/debugpy/debugpy/public_api.py +++ b/python-ecosys/debugpy/debugpy/public_api.py @@ -37,7 +37,23 @@ def listen(port=DEFAULT_PORT, host=DEFAULT_HOST): listener.bind(addr) listener.listen(1) - # getsockname not available in MicroPython, use original values + # Resolve the actual bound port (needed when the caller asked for port 0 / + # auto). Not every MicroPython port implements getsockname(). + requested_port = port + try: + bound_addr = listener.getsockname() + if isinstance(bound_addr, (tuple, list)) and len(bound_addr) >= 2: + port = bound_addr[1] + except Exception: + pass + if requested_port == 0 and port == 0: + # The caller asked for an OS-assigned port and this port has no way + # to report what the OS actually picked. Advertising port 0 in the + # handshake would tell the client to connect to a port that can + # never accept a connection, so fall back to the documented default + # instead - it is at least a real, well-known port to try. + port = DEFAULT_PORT + print(f"Debugpy listening on {host}:{port}") # Wait for connection @@ -69,7 +85,9 @@ def listen(port=DEFAULT_PORT, host=DEFAULT_HOST): client_sock.close() _debug_session = None finally: - # Only close the listener, not the client connection + # The accepted client socket is independent of the listener; closing + # the listener does not affect it. This is a single-connection server, + # so stop listening once the client is accepted. listener.close() return (host, port) @@ -97,11 +115,35 @@ def format_client_addr(client_addr): return str(client_addr) -def wait_for_client(): - """Wait for the debugger client to connect and initialize.""" +def wait_for_client(timeout_s=None): + """Block until the DAP client has finished configuring (configurationDone). + + Replaces a fixed sleep after debug_this_thread(): breakpoints the client + sets before configurationDone are honoured because this drains the socket + the whole time it waits. Returns True once configurationDone arrives, + False after a bounded timeout (logged, not silent) or if no session is + listening. + """ global _debug_session - if _debug_session: - _debug_session.wait_for_client() + if _debug_session is None: + print("[DAP] wait_for_client: no debug session is listening, nothing to wait for") + return False + if timeout_s is None: + return _debug_session.wait_for_client() + return _debug_session.wait_for_client(timeout_s) + + +def get_capabilities(): + """Return the firmware capability dict (settrace/save_names/set_local/f_back). + + Uses the active session's probe result if a session exists, otherwise + probes directly. Values always come from probing the running + interpreter, never from a build/variant name. + """ + global _debug_session + if _debug_session is not None: + return _debug_session.capabilities + return DebugSession.probe_capabilities() def breakpoint(): diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index 17904fb49..575faa794 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -1,6 +1,7 @@ """Main debug session handling DAP protocol communication.""" import sys +import time from ..common.constants import ( CMD_ATTACH, @@ -32,11 +33,24 @@ TRACE_EXCEPTION, TRACE_LINE, TRACE_RETURN, + WAIT_FOR_CLIENT_TIMEOUT_S, ) from ..common.messaging import JsonMessageChannel from .pdb_adapter import PdbAdapter +def _is_placeholder_local_name(name): + """True if `name` is a positional `local_N` placeholder, not a real name. + + Without MICROPY_PY_SYS_SETTRACE_SAVE_NAMES, frame.f_locals synthesizes + names as `local_1`, `local_2`, ... (see py/profile.c). This is the only + reliable signal that separates the two cases at runtime. + """ + if not name.startswith("local_"): + return False + return name[len("local_") :].isdigit() + + class DebugSession: """Manages a debugging session with a DAP client.""" @@ -50,6 +64,10 @@ def __init__(self, client_socket): self.thread_id = 1 # Simple single-thread model self.stepping = False self.paused = False + self.configuration_done = False + # Probed once at session start; never inferred from a build/variant name. + self.capabilities = self.probe_capabilities() + self.pdb.capabilities = self.capabilities def _debug_print(self, message): """Print debug message only if debug logging is enabled.""" @@ -60,6 +78,53 @@ def _debug_print(self, message): def _baremetal(self) -> bool: return sys.platform not in ("linux") # to be expanded + @staticmethod + def probe_capabilities(): + """Probe what the running firmware actually supports. + + Returns a dict with at least `settrace`, `save_names`, `set_local` and + `f_back`, each derived by exercising the real interpreter - never by + reading a build/variant name, which does not reliably reflect what a + given firmware image supports (see BACKGROUND.md). Safe to call on + both the unix port and bare-metal builds; never raises. + """ + caps = { + "settrace": hasattr(sys, "settrace"), + "f_back": False, + "save_names": False, + "set_local": False, + } + if not caps["settrace"]: + return caps + + try: + frame = sys._getframe() + except Exception: + return caps + + try: + caps["f_back"] = hasattr(frame, "f_back") + except Exception: + pass + + try: + caps["set_local"] = hasattr(frame, "_set_local") + except Exception: + pass + + try: + local_names = list(frame.f_locals.keys()) + # An empty locals dict (e.g. probing from module scope) proves + # nothing either way; only trust the signal when there is at + # least one local name to inspect for the placeholder pattern. + caps["save_names"] = bool(local_names) and not any( + _is_placeholder_local_name(n) for n in local_names + ) + except Exception: + pass + + return caps + def start(self): """Start the debug session message loop.""" try: @@ -417,6 +482,7 @@ def _handle_configuration_done(self, seq, args): """Handle configurationDone request.""" # This indicates that the client has finished configuring breakpoints # and is ready to start debugging + self.configuration_done = True self.channel.send_response(CMD_CONFIGURATION_DONE, seq) def _handle_threads(self, seq, args): @@ -481,10 +547,34 @@ def _send_stopped_event(self, reason): EVENT_STOPPED, reason=reason, threadId=self.thread_id, allThreadsStopped=True ) - def wait_for_client(self): - """Wait for client to initialize.""" - # This is a simplified version - in a real implementation - # we might want to wait for specific initialization steps + def wait_for_client(self, timeout_s=WAIT_FOR_CLIENT_TIMEOUT_S): + """Block until the client has sent configurationDone, or time out. + + Same busy-poll shape as PdbAdapter.wait_for_continue(): there is no + server thread, so nothing services the socket unless this loop drains + it. Replaces a fixed sleep with a deterministic handshake - breakpoints + set before configurationDone are already applied by the time this + returns because process_pending_messages() has drained them too. + Returns True once configurationDone arrives, False if the bounded + timeout elapses first (a hard failure is worse than continuing with a + clear log message: a client that never configures is a client bug or + a dropped connection, not something to hang on forever). + """ + start = time.ticks_ms() + while not self.configuration_done: + self.process_pending_messages() + if not self.connected or self.channel.closed: + print("[DAP] wait_for_client: connection closed before configurationDone") + return False + if time.ticks_diff(time.ticks_ms(), start) > timeout_s * 1000: + print( + "[DAP] wait_for_client: timed out after {}s waiting for configurationDone".format( + timeout_s + ) + ) + return False + time.sleep(0.01) + return True def trigger_breakpoint(self): """Trigger a manual breakpoint.""" diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index b55772f6d..1988c83e8 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -109,6 +109,9 @@ def __init__(self): # list of [runtime_path -> vscode_path mapping] self.file_mappings: dict[str, str] = {} # runtime_path -> vscode_path mapping # todo : merge with .breakpoints + self.capabilities: dict = {} + # set by DebugSession at session start (see DebugSession.probe_capabilities); + # empty dict here means "not yet probed", treated as no set_local support def _debug_print(self, message): """Print debug message only if debug logging is enabled.""" @@ -358,7 +361,7 @@ def get_scopes(self, frame_id): ] return scopes - def _process_special_variables(self, var_dict): + def _process_special_variables(self, var_dict, read_only=False): """Process special variables (those starting and ending with __).""" variables = [] for name, value in var_dict.items(): @@ -367,19 +370,20 @@ def _process_special_variables(self, var_dict): # Use lightweight serialization instead of json.dumps value_str = self._lightweight_serialize(value) type_str = type(value).__name__ - variables.append( - { - "name": name, - "value": value_str, - "type": type_str, - "variablesReference": 0, - } - ) + info = { + "name": name, + "value": value_str, + "type": type_str, + "variablesReference": 0, + } + if read_only: + info["presentationHint"] = {"attributes": ["readOnly"]} + variables.append(info) except Exception: variables.append(self._var_error(name)) return variables - def _process_regular_variables(self, var_dict): + def _process_regular_variables(self, var_dict, read_only=False): """Process regular variables (excluding special ones) - optimized.""" variables = [] for name, value in var_dict.items(): @@ -387,7 +391,10 @@ def _process_regular_variables(self, var_dict): if name.startswith("__") and name.endswith("__"): continue # Use fast path for variable info generation - variables.append(self._get_variable_info_fast(name, value)) + info = self._get_variable_info_fast(name, value) + if read_only: + info["presentationHint"] = {"attributes": ["readOnly"]} + variables.append(info) return variables def _is_expandable(self, value: Any) -> bool: @@ -608,10 +615,16 @@ def get_variables(self, variables_ref): frame = self.variables_cache[frame_id] + # Locals are read-only in DAP when this firmware has no _set_local + # (STORY-1.3): the edit affordance is greyed out client-side instead + # of setVariable failing with an error after the fact. Globals always + # stay editable - global write-back works on every firmware. + locals_read_only = not self.capabilities.get("set_local", False) + # Handle special scope types first if scope_type == VARREF_LOCALS_SPECIAL: var_dict = frame.f_locals if hasattr(frame, "f_locals") else {} - return self._process_special_variables(var_dict) + return self._process_special_variables(var_dict, read_only=locals_read_only) elif scope_type == VARREF_GLOBALS_SPECIAL: var_dict = frame.f_globals if hasattr(frame, "f_globals") else {} return self._process_special_variables(var_dict) @@ -629,7 +642,8 @@ def get_variables(self, variables_ref): return [] # Add regular variables with enhanced processing - variables.extend(self._process_regular_variables(var_dict)) + read_only = locals_read_only if scope_type == VARREF_LOCALS else False + variables.extend(self._process_regular_variables(var_dict, read_only=read_only)) return variables def evaluate_expression(self, expression, frame_id=None): From 6318df32e11a802c7b2faa46293e8eb30389afef Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Wed, 15 Jul 2026 05:43:21 +1000 Subject: [PATCH 50/74] debugpy: Execute statements from repl and clipboard evaluate contexts. DAP `evaluate` requests carry a `context` field (`watch`, `hover`, `repl`, `clipboard`, ...) that `_handle_evaluate` read but discarded, so every request went through `eval()` only; a statement such as `x = 5` or `def f(): ...` typed into the Debug Console failed with a syntax error instead of running. `evaluate_expression` now dispatches on `context`: `watch`/`hover` (and any other or absent context) keep the original eval-only, read-only contract unchanged. `repl`/`clipboard` try `eval()` first, so a plain expression like `1 + 1` still returns a value, and only fall back to `exec(expression, globals_dict)` when `eval()` raises `SyntaxError`. The exec namespace is globals-only, on purpose: `exec(code, g, l)` binds a top-level assignment into `l`, and here `l` is a throwaway copy of the paused frame's `f_locals` snapshot handed back to the caller and then discarded, so the assignment would silently vanish instead of taking effect. Passing only `globals_dict` makes a statement's assignments land in the running module namespace, where they are visible to the target program after `continue`. That globals-only exec creates a shadowing hazard: assigning a name that is also a LOCAL of the paused frame changes the global but leaves the local exactly as it was, which looks like a no-op from the Debug Console's perspective. `_shadowed_local_warning` detects the common case (a simple `name = ...` or `name op= ...` at the start of the statement) and appends a warning to the result so the mismatch is visible rather than silently misleading; it does not attempt to parse multi-target assignment, unpacking, attribute/subscript targets, or `def`/`class`/`for` bindings, and a `None` result from `_assigned_name` means "not proven safe", never "proven no shadowing". Signed-off-by: Andrew Leech --- .../debugpy/debugpy/server/debug_session.py | 12 ++- .../debugpy/debugpy/server/pdb_adapter.py | 101 +++++++++++++++++- 2 files changed, 107 insertions(+), 6 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index 575faa794..483864411 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -456,17 +456,23 @@ def _handle_set_variable(self, seq, args): self.channel.send_response(CMD_SET_VARIABLE, seq, success=False, message=str(e)) def _handle_evaluate(self, seq, args): - """Handle evaluate request.""" + """Handle evaluate request. + + `context` selects the contract PdbAdapter.evaluate_expression applies: + `repl`/`clipboard` (Debug Console, "Copy as Expression") may execute a + statement when `expression` isn't a valid expression; `watch`/`hover` + and any other or absent context stay read-only eval. + """ expression = args.get("expression", "") frame_id = args.get("frameId") - # context = args.get("context", "watch") + context = args.get("context", "watch") if not expression: self.channel.send_response( CMD_EVALUATE, seq, success=False, message="No expression provided" ) return try: - result = self.pdb.evaluate_expression(expression, frame_id) + result = self.pdb.evaluate_expression(expression, frame_id, context) self.channel.send_response( CMD_EVALUATE, seq, body={"result": str(result), "variablesReference": 0} ) diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index 1988c83e8..da9144d06 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -89,6 +89,68 @@ def ends_with_path(full_path: str, relative_path: str): return full_parts[-len(rel_parts) :] == rel_parts +# Augmented-assignment operators checked longest-first so e.g. "**=" is not +# mistaken for "*=" followed by stray text. +_AUG_ASSIGN_OPS = ("**=", "//=", ">>=", "<<=", "+=", "-=", "*=", "/=", "%=", "&=", "|=", "^=", "=") + + +def _is_ident_char(ch: str) -> bool: + """True for `[A-Za-z0-9_]` - MicroPython's `str` has no `.isalnum()`.""" + return ch.isalpha() or ch.isdigit() or ch == "_" + + +def _assigned_name(statement: str): + """Return the target name of a simple top-level assignment, or None. + + Recognises only `...` where the identifier is the very + first token and `` is `=` or an augmented-assignment operator. This + is a deliberately narrow, best-effort check - it does NOT catch: + multi-target assignment (`a = b = 1`, only `a` is seen), tuple/list + unpacking (`a, b = 1, 2`), attribute/subscript targets (`obj.x = 1`, + `d[k] = 1`), `def`/`class` statements (which also bind a name), a + `for`/`with ... as` binding, or an assignment that is not the first + statement on the line (e.g. after `;`). Those forms pass through + undetected; callers must treat a `None` result as "not proven safe", + never as "proven no shadowing". + """ + stripped = statement.strip() + if not stripped or stripped[0].isdigit() or not _is_ident_char(stripped[0]): + return None + i = 1 + n = len(stripped) + while i < n and _is_ident_char(stripped[i]): + i += 1 + name = stripped[:i] + rest = stripped[i:].lstrip() + for op in _AUG_ASSIGN_OPS: + if rest.startswith(op): + if op == "=" and rest[1:2] == "=": + return None # `==`, a comparison, not an assignment + return name + return None + + +def _shadowed_local_warning(statement: str, locals_dict): + """Build the honesty-rule warning for `statement`, or None if it doesn't apply. + + Fires when `_assigned_name` recognises a top-level assignment whose + target name is also a key in `locals_dict` (the paused frame's + `f_locals` snapshot): that name is about to be rebound in `f_globals` + only, so the LOCAL of the same name stays exactly as it was. On + firmware without local-name capture (`save_names` capability False), + `locals_dict` keys are synthetic `local_N` placeholders rather than + real identifiers, so a real name can never match and this warning + silently cannot fire there - a known limitation, not a bug. + """ + name = _assigned_name(statement) + if name and name in locals_dict: + return ( + f"Warning: '{name}' also exists as a LOCAL in this frame; " + "the local is unchanged (statement ran against globals only)." + ) + return None + + class PdbAdapter: """Adapter between DAP protocol and MicroPython's sys.settrace functionality.""" @@ -646,8 +708,28 @@ def get_variables(self, variables_ref): variables.extend(self._process_regular_variables(var_dict, read_only=read_only)) return variables - def evaluate_expression(self, expression, frame_id=None): - """Evaluate an expression in the context of a frame.""" + def evaluate_expression(self, expression, frame_id=None, context="watch"): + """Evaluate a DAP `evaluate` request in the context of a frame. + + `watch`/`hover` (and any other/absent `context`) keep the original, + read-only contract: `eval()` only - a statement is a `SyntaxError`, + surfaced as an evaluation error, exactly as before this method + gained statement support. + + `repl`/`clipboard` add statement execution: `eval()` is tried first + (so a plain expression like `1 + 1` still returns a value); a + `SyntaxError` falls back to `exec(expression, globals_dict)` against + the frame's live `f_globals` only. The locals snapshot is + deliberately never passed to `exec` as a namespace - `exec(code, g, + l)` binds a top-level assignment into `l`, and `l` here is a + disposable copy handed back to the caller and then discarded, so + the assignment would silently vanish instead of taking effect. Only + `globals_dict` is live, so a statement's top-level assignments land + in the running module namespace and are visible to the target + program after `continue`. See `_shadowed_local_warning` for the + honesty-rule warning this implies when the assigned name also + exists as a frame LOCAL. + """ if frame_id is not None and frame_id in self.variables_cache: frame = self.variables_cache[frame_id] globals_dict = frame.f_globals if hasattr(frame, "f_globals") else {} @@ -661,13 +743,26 @@ def evaluate_expression(self, expression, frame_id=None): else: globals_dict = globals() locals_dict = {} + try: - # Evaluate the expression result = eval(expression, globals_dict, locals_dict) return result + except SyntaxError as e: + if context not in ("repl", "clipboard"): + raise Exception(f"Evaluation error: {e}") except Exception as e: raise Exception(f"Evaluation error: {e}") + # Only repl/clipboard reach here, and only after eval() raised a + # SyntaxError - try `expression` as a statement instead. + try: + exec(expression, globals_dict) + except Exception as e: + raise Exception(f"Evaluation error: {e}") + + warning = _shadowed_local_warning(expression, locals_dict) + return warning if warning else "" + def cleanup(self): """Clean up resources with enhanced cache management.""" self.variables_cache.clear() From 7d15d4e93e1557121d5a2b91e00d0e8b092b714e Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Wed, 5 Aug 2026 18:06:34 +1000 Subject: [PATCH 51/74] debugpy: Return the bound endpoint from listen() before accepting. listen() bound the socket, blocked in accept() and handled the client's initialize request before returning, so a caller could only learn the endpoint after a client had already connected to it - unusable for any orchestration that has to read the address in order to attach. listen() now returns as soon as the socket is bound. The accept and the initialize handshake move into wait_for_client(), which creates the session. This matches CPython debugpy, where listen() reports the endpoint and wait_for_client() blocks. port=0 now raises instead of substituting DEFAULT_PORT when the target's getsockname() cannot report the assigned port: callers act on the returned endpoint, so naming an address the socket is not bound to sends them somewhere nothing is listening. Signed-off-by: Andrew Leech --- python-ecosys/debugpy/debugpy/public_api.py | 80 ++++++++++++++------- 1 file changed, 54 insertions(+), 26 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/public_api.py b/python-ecosys/debugpy/debugpy/public_api.py index 59577a44e..6379d4523 100644 --- a/python-ecosys/debugpy/debugpy/public_api.py +++ b/python-ecosys/debugpy/debugpy/public_api.py @@ -7,21 +7,31 @@ from .server.debug_session import DebugSession _debug_session = None +# Bound-but-not-yet-accepted socket, held between listen() and the accept that +# wait_for_client() performs. +_listener = None def listen(port=DEFAULT_PORT, host=DEFAULT_HOST): - """Start listening for debugger connections. + """Bind a listening socket and return the address it is bound to. + + Returns as soon as the socket is bound, WITHOUT waiting for a client, so + the caller can publish the endpoint that a client then connects to. The + accept and the `initialize` handshake happen in `wait_for_client()`. This + matches CPython debugpy, where `listen()` reports the endpoint and + `wait_for_client()` blocks. Args: - port: Port number to listen on (default: 5678) + port: Port number to listen on, or 0 to let the system choose + (default: 5678) host: Host address to bind to (default: "127.0.0.1") Returns: - (host, port) tuple of the actual listening address + (host, port) tuple of the actual bound address """ - global _debug_session + global _listener - if _debug_session is not None: + if _listener is not None or _debug_session is not None: raise RuntimeError("Already listening for debugger") # Create listening socket @@ -47,25 +57,42 @@ def listen(port=DEFAULT_PORT, host=DEFAULT_HOST): except Exception: pass if requested_port == 0 and port == 0: - # The caller asked for an OS-assigned port and this port has no way - # to report what the OS actually picked. Advertising port 0 in the - # handshake would tell the client to connect to a port that can - # never accept a connection, so fall back to the documented default - # instead - it is at least a real, well-known port to try. - port = DEFAULT_PORT + # Callers act on the endpoint this returns, so reporting a port + # nothing can connect to would be worse than refusing: substituting + # DEFAULT_PORT here would advertise an address the socket is not + # bound to. Ask for an explicit port on a target whose getsockname() + # cannot report the OS-assigned one. + listener.close() + raise OSError( + "port=0 needs getsockname() to report the assigned port, which " + "this target does not implement; pass an explicit port" + ) + _listener = listener print(f"Debugpy listening on {host}:{port}") + return (host, port) + + +def _accept_and_initialize(): + """Accept the pending connection and handle the client's `initialize`. + + Split out of `listen()` so the endpoint can be published before a client + exists. Returns True once a session is ready. + """ + global _debug_session, _listener + + if _listener is None: + print("[DAP] no listening socket; call listen() first") + return False - # Wait for connection + listener, _listener = _listener, None client_sock = None try: client_sock, client_addr = listener.accept() print(f"Debugger connected from {format_client_addr(client_addr)}") - # Create debug session _debug_session = DebugSession(client_sock) - # Handle just the initialize request, then return immediately print("[DAP] Waiting for initialize request...") init_message = _debug_session.channel.recv_message() if init_message and init_message.get("command") == "initialize": @@ -78,20 +105,20 @@ def listen(port=DEFAULT_PORT, host=DEFAULT_HOST): _debug_session.channel.sock.settimeout(0.001) print("[DAP] Debug session ready - all other messages will be handled in trace function") + return True except Exception as e: print(f"[DAP] Connection error: {e}") if client_sock: client_sock.close() - _debug_session = None + _debug_session = None + return False finally: # The accepted client socket is independent of the listener; closing # the listener does not affect it. This is a single-connection server, # so stop listening once the client is accepted. listener.close() - return (host, port) - def format_client_addr(client_addr): """Format client address using socket module methods""" @@ -116,17 +143,18 @@ def format_client_addr(client_addr): def wait_for_client(timeout_s=None): - """Block until the DAP client has finished configuring (configurationDone). - - Replaces a fixed sleep after debug_this_thread(): breakpoints the client - sets before configurationDone are honoured because this drains the socket - the whole time it waits. Returns True once configurationDone arrives, - False after a bounded timeout (logged, not silent) or if no session is - listening. + """Block until a client has attached and finished configuring. + + Accepts the connection and handles `initialize` (both deferred by + `listen()` so the endpoint can be published first), then waits for + `configurationDone`. Breakpoints the client sets before then are honoured + because this drains the socket the whole time it waits. Returns True once + configurationDone arrives, False after a bounded timeout (logged, not + silent) or if nothing is listening. """ global _debug_session - if _debug_session is None: - print("[DAP] wait_for_client: no debug session is listening, nothing to wait for") + if _debug_session is None and not _accept_and_initialize(): + print("[DAP] wait_for_client: nothing is listening, nothing to wait for") return False if timeout_s is None: return _debug_session.wait_for_client() From 9c91034384c7c377b8f9d81c3e4f94249afa7b8e Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Thu, 6 Aug 2026 17:18:01 +1000 Subject: [PATCH 52/74] debugpy: Stop nested message pumps from clobbering the socket timeout. process_pending_messages() set a 1 ms socket timeout and restored blocking mode in its finally. The trace function calls it on entry to every new frame, so handling a message re-enters it, and the inner call's finally put the socket back into blocking mode underneath the outer loop. That loop's next recv() then waited for a message the client will not send until it has seen an event the loop itself is what produces - a deadlock between the two sides. It only bites when the clobber lands inside the window after configurationDone, which is why it presented as a load-sensitive flake: the session hangs before wait_for_client() returns, so the target never runs and no stopped event is ever produced. The nesting is tracked rather than the timeout saved and restored, because MicroPython sockets have no gettimeout(). Measured on the wrapper repo's harness: the previously worst-affected file went from 4 clean runs in 6 to 6 in 6, and the full suite from 0 clean in 3 to 3 in 4. Signed-off-by: Andrew Leech --- .../debugpy/debugpy/server/debug_session.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index 483864411..8a594a246 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -65,6 +65,7 @@ def __init__(self, client_socket): self.stepping = False self.paused = False self.configuration_done = False + self._pumping = False # Probed once at session start; never inferred from a build/variant name. self.capabilities = self.probe_capabilities() self.pdb.capabilities = self.capabilities @@ -201,7 +202,19 @@ def initialize_connection(self): print(f"[DAP] Initialization error: {e}") def process_pending_messages(self): - """Process any pending DAP messages without blocking.""" + """Process any pending DAP messages without blocking. + + Not re-entered: the trace function calls this on entry to every new + frame, so handling a message here can call it again. A nested call + must not touch the socket timeout, because its `finally` would put the + socket back into blocking mode underneath the outer loop, whose next + recv() then waits for a message the client will not send until it has + seen an event this loop is what produces. MicroPython sockets have no + gettimeout(), so the nesting is tracked rather than the timeout saved. + """ + if self._pumping: + return + self._pumping = True try: # Set socket to non-blocking mode for message processing self.channel.sock.settimeout(0.001) # Very short timeout @@ -218,6 +231,7 @@ def process_pending_messages(self): finally: # Reset to blocking mode self.channel.sock.settimeout(None) + self._pumping = False def _handle_message(self, message): """Handle incoming DAP messages.""" From 1544fad05190ba8083852d8213364f3f3271a07a Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Sun, 9 Aug 2026 08:36:01 +1000 Subject: [PATCH 53/74] debugpy: Make debug_session.py match tools/codeformat.py. `ruff format --diff .` - the second command in this repository's ruff CI job - reports this file as unformatted, so the job is red on this branch. The over-length line carries two trailing comments. Parenthesising the value, which is what the formatter does, is the worse of the two readings; the prose comment moves above the assignment instead, and the `# type: ignore` stays on the line it applies to. `tools/codeformat.py` then leaves the tree unchanged. Signed-off-by: Andrew Leech --- python-ecosys/debugpy/debugpy/server/debug_session.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index 8a594a246..619759339 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -58,7 +58,8 @@ def __init__(self, client_socket): self.debug_logging = False # Initialize first self.channel = JsonMessageChannel(client_socket, self._debug_print) self.pdb = PdbAdapter() - self.pdb._debug_session = self # Allow PDB to process messages during wait # type: ignore[assignment] + # Lets the adapter pump DAP messages while it waits. + self.pdb._debug_session = self # type: ignore[assignment] self.initialized = False self.connected = True self.thread_id = 1 # Simple single-thread model From 033bc85f0379cf8551a582696e29ba19ada0956c Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Sun, 9 Aug 2026 08:36:01 +1000 Subject: [PATCH 54/74] debugpy: Fix the EXE001 shebangs on the three top-level scripts. `ruff check .` reports EXE001 on all three, and the rule is pointing at a real inconsistency rather than a style preference. `demo.py` and `test_vscode.py` are run as `micropython demo.py`, so `#!/usr/bin/env python3` names an interpreter that cannot run them; the line is removed. `dap_monitor.py` is a host-side CPython tool whose shebang is correct, so it becomes executable instead. Signed-off-by: Andrew Leech --- python-ecosys/debugpy/dap_monitor.py | 0 python-ecosys/debugpy/demo.py | 1 - python-ecosys/debugpy/test_vscode.py | 1 - 3 files changed, 2 deletions(-) mode change 100644 => 100755 python-ecosys/debugpy/dap_monitor.py diff --git a/python-ecosys/debugpy/dap_monitor.py b/python-ecosys/debugpy/dap_monitor.py old mode 100644 new mode 100755 diff --git a/python-ecosys/debugpy/demo.py b/python-ecosys/debugpy/demo.py index fd88c0272..03f32701d 100644 --- a/python-ecosys/debugpy/demo.py +++ b/python-ecosys/debugpy/demo.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """Simple demo of MicroPython debugpy functionality.""" import sys diff --git a/python-ecosys/debugpy/test_vscode.py b/python-ecosys/debugpy/test_vscode.py index 1d24fac81..309bc7e18 100644 --- a/python-ecosys/debugpy/test_vscode.py +++ b/python-ecosys/debugpy/test_vscode.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """Test script for VS Code debugging with MicroPython debugpy.""" import sys From 4da4c340bca0e417eb99affcf353a168cdbd3028 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Sun, 9 Aug 2026 08:39:16 +1000 Subject: [PATCH 55/74] debugpy: Fix the four typos codespell reports in debug_session.py. The repository's codespell job fails on this branch, and all four hits are in this file's comments: "debugee" twice, "inialization", "Nothe". Signed-off-by: Andrew Leech --- python-ecosys/debugpy/debugpy/server/debug_session.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index 619759339..11d3cbe87 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -361,9 +361,9 @@ def _handle_attach(self, seq, args): f"[DAP] Debug logging {'enabled' if self.debug_logging else 'disabled'} (logToFile={self.debug_logging})" ) - # get debugger root and debugee root from pathMappings + # get debugger root and debuggee root from pathMappings for pm in args.get("pathMappings", []): - # debugee - debugger + # debuggee - debugger self.pdb.path_mappings.append((pm.get("remoteRoot", "./"), pm.get("localRoot", "./"))) # # TODO: justMyCode, debugOptions , @@ -518,8 +518,8 @@ def _handle_source(self, seq, args): source_path = source.get("path", "") if self._baremetal or not source_path: # BUG: unable to read the source on ESP32 - # Possible an effect of the import / inialization sequence ? - # Nothe that other source files ( other.py) do not seem to get requested in the same way + # Possible an effect of the import / initialization sequence ? + # Note that other source files ( other.py) do not seem to get requested in the same way self.channel.send_response(CMD_SOURCE, seq, success=False) return self._debug_print(f"[DAP] Processing source request for path: {source}") From c7d973e47aa8f3e1a59dd615a2fef77aeab9b86d Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Mon, 10 Aug 2026 07:53:54 +1000 Subject: [PATCH 56/74] debugpy: Start the debugger around the sample, not inside it. `ruff check .` reports four T100 (flake8-debugger) failures in this package, all of them in the two sample scripts: an `import debugpy` apiece, and `test_vscode.py`'s calls to `listen`, `breakpoint` and `wait_for_client`. The rule is right about these files. A program that starts its own debug server is not the case this package exists for: on a device the client is at the other end of a link, and something has to bind the socket and wait for it before the program runs. So `test_vscode.py` is now just a program to debug - no debugpy import, no manual breakpoint, a `main()` for a launcher to call after `listen()` and `wait_for_client()` - and the README and the development guide give the command that does that. Two incidental repairs come with it: the loop imported `machine`, which does not exist on the unix port the guide tells you to run this on, and the README named a test file that is not in the package and a build directory that its own build command does not produce. `demo.py`'s `import debugpy` was dead: it reaches the package through `from debugpy.server ...` imports, so the bare import just goes. Signed-off-by: Andrew Leech --- python-ecosys/debugpy/README.md | 13 +++-- python-ecosys/debugpy/demo.py | 21 +++++--- python-ecosys/debugpy/development_guide.md | 27 +++++++--- python-ecosys/debugpy/test_vscode.py | 62 ++++++---------------- 4 files changed, 56 insertions(+), 67 deletions(-) diff --git a/python-ecosys/debugpy/README.md b/python-ecosys/debugpy/README.md index 70859b974..87af4da8e 100644 --- a/python-ecosys/debugpy/README.md +++ b/python-ecosys/debugpy/README.md @@ -78,21 +78,26 @@ Create a `.vscode/launch.json` file in your project: ### Testing -1. Build the MicroPython Unix coverage port: +1. Build the MicroPython Unix port with tracing enabled: ```bash cd ports/unix make CFLAGS_EXTRA="-DMICROPY_PY_SYS_SETTRACE=1" ``` -2. Run the test script: +2. Run the sample program under a debug server. The program itself contains + no debugpy calls, so the server is started around it: ```bash cd lib/micropython-lib/python-ecosys/debugpy - ../../../../ports/unix/build-coverage/micropython test_debugpy.py + ../../../../ports/unix/build-standard/micropython -c "import debugpy; \ + debugpy.listen(); debugpy.wait_for_client(); \ + debugpy.debug_this_thread(); import test_vscode; test_vscode.main()" ``` + It waits for a client to attach and finish configuring before running + anything, so breakpoints set now are in place for the first line. 3. In VS Code, open the debugpy folder and press F5 to attach the debugger -4. Set breakpoints in the test script and observe debugging functionality +4. Set breakpoints in `test_vscode.py` and observe debugging functionality ## API Reference diff --git a/python-ecosys/debugpy/demo.py b/python-ecosys/debugpy/demo.py index 03f32701d..a1f94c2b7 100644 --- a/python-ecosys/debugpy/demo.py +++ b/python-ecosys/debugpy/demo.py @@ -1,11 +1,15 @@ -"""Simple demo of MicroPython debugpy functionality.""" +"""Simple demo of MicroPython debugpy functionality. + +Exercises the pieces a session is built from without starting one: the +firmware's trace hook, and the two package internals that sit on top of it. +Starting a server is the launcher's job, not a sample's. +""" import sys +# The package is a sibling of this file, not installed. sys.path.insert(0, ".") -import debugpy - def simple_function(a, b): """A simple function to demonstrate debugging.""" @@ -62,11 +66,12 @@ def trace_function(frame, event, arg): print() print("3. debugpy is ready for VS Code integration!") - print(" To use with VS Code:") - print(" - Import debugpy in your script") - print(" - Call debugpy.listen() to start the debug server") - print(" - Connect VS Code using the 'Attach to MicroPython' configuration") - print(" - Set breakpoints and debug normally") + print(" To debug a program with VS Code:") + print(" - Start the server first: listen(), then wait_for_client()") + print(" - Import and run the program from there, so the client's") + print(" breakpoints are already set when it starts") + print(" - Attach with the 'Attach to MicroPython' configuration") + print(" - See development_guide.md for the command") if __name__ == "__main__": diff --git a/python-ecosys/debugpy/development_guide.md b/python-ecosys/debugpy/development_guide.md index 94f06b420..81c546b14 100644 --- a/python-ecosys/debugpy/development_guide.md +++ b/python-ecosys/debugpy/development_guide.md @@ -1,11 +1,25 @@ # Debugging MicroPython debugpy with VS Code +## Starting a session + +Nothing in the program being debugged talks to the debugger. The server is +started around it: `listen()`, then `wait_for_client()`, and only then is the +program imported and run, so the breakpoints the client sent are already in +place when its first line executes. `test_vscode.py` is such a program - no +debugpy import, no manual breakpoint - and this runs it: + +```bash +/micropython -c "import debugpy; debugpy.listen(); \ + debugpy.wait_for_client(); debugpy.debug_this_thread(); \ + import test_vscode; test_vscode.main()" +``` + +`wait_for_client()` blocks until a client has attached and sent +`configurationDone`, so there is no race to connect and no sleep to tune. + ## Method 1: Direct Connection with Enhanced Logging -1. **Start MicroPython with enhanced logging:** - ```bash - ~/micropython2/ports/unix/build-standard/micropython test_vscode.py - ``` +1. **Start the session** as above, in a terminal you can read. This will now show detailed DAP protocol messages like: ``` @@ -22,10 +36,7 @@ ## Method 2: Using DAP Monitor (Recommended for detailed analysis) -1. **Start MicroPython debugpy server:** - ```bash - ~/micropython2/ports/unix/build-standard/micropython test_vscode.py - ``` +1. **Start the session** as above. 2. **In another terminal, start the DAP monitor:** ```bash diff --git a/python-ecosys/debugpy/test_vscode.py b/python-ecosys/debugpy/test_vscode.py index 309bc7e18..18dc20aed 100644 --- a/python-ecosys/debugpy/test_vscode.py +++ b/python-ecosys/debugpy/test_vscode.py @@ -1,10 +1,14 @@ -"""Test script for VS Code debugging with MicroPython debugpy.""" +"""A program to debug: ordinary MicroPython that knows nothing about debugpy. -import sys - -sys.path.insert(0, ".") +Something else starts the session and then runs this. A launcher calls +`debugpy.listen()` and `debugpy.wait_for_client()`, imports this module and +calls `main()`; `development_guide.md` gives the command. That is the case +worth demonstrating - stopping in code with no debugger calls in it - and the +only one available when the program being debugged is on a device and the +client is not. +""" -import debugpy +import sys foo = 42 bar = "Hello, MicroPython!" @@ -21,26 +25,17 @@ def fibonacci(n): def debuggable_code(): - """The actual code we want to debug - wrapped in a function so sys.settrace will trace it.""" + """A call to step into, a global to watch, and a loop to break inside.""" global foo print("Starting debuggable code...") - # Test data - set breakpoint here (using smaller numbers to avoid slow fibonacci) + # Small numbers: fibonacci is here to be stepped through, not benchmarked. numbers = [3, 4, 5] for i, num in enumerate(numbers): - print(f"Calculating fibonacci({num})...") - result = fibonacci(num) # <-- SET BREAKPOINT HERE (line 26) + print(f"[{i}] Calculating fibonacci({num})...") + result = fibonacci(num) # <-- SET BREAKPOINT HERE foo += result # Modify foo to see if it gets traced print(f"fibonacci({num}) = {result}") - print(sys.implementation) - import machine - - print(dir(machine)) - - # Test manual breakpoint - print("\nTriggering manual breakpoint...") - debugpy.breakpoint() - print("Manual breakpoint triggered!") print("Test completed successfully!") @@ -48,35 +43,8 @@ def debuggable_code(): def main(): print("MicroPython VS Code Debugging Test") print("==================================") - - # Start debug server - try: - debugpy.listen() - print("Debug server attached on 127.0.0.1:5678") - print("Connecting back to VS Code debugger now...") - # print("Set a breakpoint on line 26: 'result = fibonacci(num)'") - # print("Press Enter to continue after connecting debugger...") - # try: - # input() - # except: - # pass - - # Enable debugging for this thread - debugpy.debug_this_thread() - - # Give VS Code a moment to set breakpoints after attach - print("\nGiving VS Code time to set breakpoints...") - import time - - time.sleep(2) - - # Call the debuggable code function so it gets traced - debuggable_code() - - except KeyboardInterrupt: - print("\nTest interrupted by user") - except Exception as e: - print(f"Error: {e}") + print(sys.implementation) + debuggable_code() if __name__ == "__main__": From 2f36b80d65faceabb0ef8bc64d8a65bfb6a6ba58 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Fri, 7 Aug 2026 21:13:20 +1000 Subject: [PATCH 57/74] debugpy: Run the DAP channel over a stream, not only a TCP socket. `StreamTransport` presents the four things `messaging.py` asks of a socket - send, recv, close, and the settimeout the pumps mutate from outside - over a reader/writer pair, polled rather than threaded because the server is driven from the trace callback. `listen_stream()` starts a session on one; the TCP path is unchanged. `recv()` reads a byte at a time, re-polling between bytes: MicroPython's read/readinto loop internally until the buffer is full, so on a stream backed by a genuinely blocking read the second internal call waits for bytes that may never arrive. A device stream that returns short without blocking would prefer a chunked read; that cannot be validated without hardware. `wait_for_continue()` drops `sys.settrace` and resumes when the channel is gone, instead of spinning forever - a target stopped at a breakpoint when its client disappears would otherwise need a power cycle. `caps` gains `serial_dap`, reporting whether the board routes DAP to a dedicated interface. No port implements that detection, so it is always false and no board can yet choose a stream over TCP. Signed-off-by: Andrew Leech --- python-ecosys/debugpy/debugpy/__init__.py | 2 + .../debugpy/common/stream_transport.py | 106 ++++++++++++++++++ python-ecosys/debugpy/debugpy/public_api.py | 43 +++++-- .../debugpy/debugpy/server/debug_session.py | 27 ++++- .../debugpy/debugpy/server/pdb_adapter.py | 26 +++-- 5 files changed, 185 insertions(+), 19 deletions(-) create mode 100644 python-ecosys/debugpy/debugpy/common/stream_transport.py diff --git a/python-ecosys/debugpy/debugpy/__init__.py b/python-ecosys/debugpy/debugpy/__init__.py index cce6cb870..a9a75b628 100644 --- a/python-ecosys/debugpy/debugpy/__init__.py +++ b/python-ecosys/debugpy/debugpy/__init__.py @@ -14,6 +14,7 @@ get_capabilities, is_client_connected, listen, + listen_stream, wait_for_client, ) from .common.constants import DEFAULT_HOST, DEFAULT_PORT @@ -27,5 +28,6 @@ "get_capabilities", "is_client_connected", "listen", + "listen_stream", "wait_for_client", ] diff --git a/python-ecosys/debugpy/debugpy/common/stream_transport.py b/python-ecosys/debugpy/debugpy/common/stream_transport.py new file mode 100644 index 000000000..0e7c269b5 --- /dev/null +++ b/python-ecosys/debugpy/debugpy/common/stream_transport.py @@ -0,0 +1,106 @@ +"""Socket-shaped adapter for running the DAP channel over a stream, not a socket.""" + +import select +import time + + +class StreamTransport: + """Presents a reader/writer stream pair as a socket to `JsonMessageChannel`. + + For boards with a dedicated CDC interface for DAP (no shared REPL/stdout + byte stream to demux) - the interface `messaging.py` and `debug_session.py` + need is exactly `recv`/`send`/`settimeout`/`close`, so this wraps the + stream to present that shape rather than changing anything downstream of + `self.sock`. + + `recv` raises `OSError(11)` (EAGAIN) when no data has arrived within the + current timeout, matching a non-blocking socket, and returns `b""` once + the stream is at EOF. `send` raises `OSError(110)` (ETIMEDOUT) if the + writer won't take the rest of the data within the current timeout, rather + than returning having written only part of it - a short write here would + desync `messaging.py`'s Content-Length framing with no way to resync. + `settimeout` is mutated repeatedly at runtime by + `public_api.py`/`debug_session.py` and governs both directions, matching + a real socket; `poll(ms)` is called fresh on every `recv`/`send` rather + than cached, so each change takes effect immediately. + """ + + def __init__(self, reader, writer=None): + self._reader = reader + self._writer = writer if writer is not None else reader + self._timeout = None # seconds, or None = block forever + self._eof = False + self._poller = select.poll() + self._poller.register(self._reader, select.POLLIN) + self._write_poller = select.poll() + self._write_poller.register(self._writer, select.POLLOUT) + self._buf = bytearray() # grown to the largest `n` seen, reused across recv() calls + + def settimeout(self, seconds): + self._timeout = seconds + + def recv(self, n): + if self._eof: + return b"" + timeout_ms = None if self._timeout is None else max(0, int(self._timeout * 1000)) + if not self._poller.poll(timeout_ms): + raise OSError(11) # EAGAIN: no data within the timeout + + # `.read()`/`.readinto()` loop internally until the buffer is full + # (py/stream.c's mp_stream_rw) - on a stream backed by a genuinely + # blocking read (a plain POSIX file, as opposed to a UART/CDC driver + # that always returns immediately), a second internal call would + # block waiting for bytes that may never come. Reading one byte at a + # time, re-checking readiness with a zero-timeout poll before every + # further byte, never makes a read the poller hasn't already + # confirmed data for. + if len(self._buf) < n: + self._buf = bytearray(n) + mv = memoryview(self._buf) + got = 0 + while got < n: + r = self._reader.readinto(mv[got : got + 1]) + if r is None: + break # raced the poll result - not actually ready + if not r: + self._eof = True + break + got += r + if got < n and not self._poller.poll(0): + break + + if got == 0: + if self._eof: + return b"" + raise OSError(11) + return bytes(mv[:got]) + + def send(self, data): + mv = memoryview(data) + off = 0 + total = len(mv) + if self._timeout is None: + deadline = None + else: + deadline = time.ticks_add(time.ticks_ms(), max(0, int(self._timeout * 1000))) + while off < total: + if deadline is None: + self._write_poller.poll() + else: + remaining_ms = time.ticks_diff(deadline, time.ticks_ms()) + if remaining_ms <= 0 or not self._write_poller.poll(max(0, remaining_ms)): + raise OSError(110) # ETIMEDOUT: writer never drained within the timeout + written = self._writer.write(mv[off:]) + if written: + off += written + + def close(self): + try: + self._reader.close() + except OSError: + pass + if self._writer is not self._reader: + try: + self._writer.close() + except OSError: + pass diff --git a/python-ecosys/debugpy/debugpy/public_api.py b/python-ecosys/debugpy/debugpy/public_api.py index 6379d4523..3b075dccf 100644 --- a/python-ecosys/debugpy/debugpy/public_api.py +++ b/python-ecosys/debugpy/debugpy/public_api.py @@ -4,6 +4,7 @@ import struct import sys from .common.constants import DEFAULT_HOST, DEFAULT_PORT +from .common.stream_transport import StreamTransport from .server.debug_session import DebugSession _debug_session = None @@ -73,6 +74,26 @@ def listen(port=DEFAULT_PORT, host=DEFAULT_HOST): return (host, port) +def listen_stream(reader, writer=None): + """Start a debug session directly on an already-open stream, no TCP. + + For a board with a second CDC interface dedicated to DAP: `reader`/`writer` + are that interface's stream (the same object for both if it is one + read/write file, as `open(dev, "r+b")` gives on the unix port). Unlike + `listen()`, the stream is already connected - there is no bind/accept + step, so `wait_for_client()` goes straight to the initialize/ + configurationDone handshake. + """ + global _listener + + if _listener is not None or _debug_session is not None: + raise RuntimeError("Already listening for debugger") + + _listener = StreamTransport(reader, writer) + print("Debugpy listening on stream") + return _listener + + def _accept_and_initialize(): """Accept the pending connection and handle the client's `initialize`. @@ -86,10 +107,17 @@ def _accept_and_initialize(): return False listener, _listener = _listener, None + # A stream transport (listen_stream()) is already connected - there is no + # separate client socket to accept, and no separate listener to close off + # once accepted (it IS the connection). + is_stream = isinstance(listener, StreamTransport) client_sock = None try: - client_sock, client_addr = listener.accept() - print(f"Debugger connected from {format_client_addr(client_addr)}") + if is_stream: + client_sock = listener + else: + client_sock, client_addr = listener.accept() + print(f"Debugger connected from {format_client_addr(client_addr)}") _debug_session = DebugSession(client_sock) @@ -109,15 +137,16 @@ def _accept_and_initialize(): except Exception as e: print(f"[DAP] Connection error: {e}") - if client_sock: + if client_sock is not None: client_sock.close() _debug_session = None return False finally: - # The accepted client socket is independent of the listener; closing - # the listener does not affect it. This is a single-connection server, - # so stop listening once the client is accepted. - listener.close() + # This is a single-connection server, so stop listening once the + # client is accepted. Not for a stream transport: it IS the client + # socket, still needed by the session this just started. + if not is_stream: + listener.close() def format_client_addr(client_addr): diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index 11d3cbe87..200b25176 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -80,21 +80,38 @@ def _debug_print(self, message): def _baremetal(self) -> bool: return sys.platform not in ("linux") # to be expanded + @staticmethod + def _probe_serial_dap(): + """Whether this board exposes a second CDC interface dedicated to DAP. + + No port implements the board-specific detection yet (which TinyUSB + CDC instance is DAP's, if any, is per-board configuration - e.g. rp2's + second `machine.USBDevice` CDC), so this always reports `False`; a + board's own boot script never gets to choose `listen_stream()` over + `listen()` on that basis until a real check lands here. Kept as its + own probe (mirroring `f_back`/`save_names`/`set_local` above) so the + one place a board-specific check would go is unambiguous, and so the + capability is always present in `caps` rather than only sometimes. + """ + return False + @staticmethod def probe_capabilities(): """Probe what the running firmware actually supports. - Returns a dict with at least `settrace`, `save_names`, `set_local` and - `f_back`, each derived by exercising the real interpreter - never by - reading a build/variant name, which does not reliably reflect what a - given firmware image supports (see BACKGROUND.md). Safe to call on - both the unix port and bare-metal builds; never raises. + Returns a dict with at least `settrace`, `save_names`, `set_local`, + `f_back` and `serial_dap`, each derived by exercising the real + interpreter - never by reading a build/variant name, which does not + reliably reflect what a given firmware image supports (see + BACKGROUND.md). Safe to call on both the unix port and bare-metal + builds; never raises. """ caps = { "settrace": hasattr(sys, "settrace"), "f_back": False, "save_names": False, "set_local": False, + "serial_dap": DebugSession._probe_serial_dap(), } if not caps["settrace"]: return caps diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index da9144d06..2a64aa19b 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -347,17 +347,29 @@ def pause(self): self.paused = True def wait_for_continue(self): - """Wait for continue command (simplified implementation).""" - # In a real implementation, this would block until continue - # For MicroPython, we'll use a simple polling approach + """Busy-poll until a continue/step command arrives, or the client is gone. + + No server thread services the socket, so this loop is what drains it + while the target sits stopped. If the channel disappears (bridge + killed, board reset) while stopped, waiting forever would wedge the + target - `sys.settrace(None)` is dropped and the loop exits so the + target resumes and the process/session can end cleanly instead of + requiring a power cycle. + """ self.continue_event = False - # Process DAP messages while waiting for continue self._debug_print("[PDB] Waiting for continue command...") while not self.continue_event: - # Process any pending DAP messages (scopes, variables, etc.) - if hasattr(self, "_debug_session"): - self._debug_session.process_pending_messages() # type: ignore[arg-type] + session = getattr(self, "_debug_session", None) + if session is None: + break + if not session.connected or session.channel.closed: + self._debug_print("[PDB] wait_for_continue: connection lost, resuming target") + if hasattr(sys, "settrace"): + sys.settrace(None) + self.continue_event = True + break + session.process_pending_messages() # type: ignore[arg-type] time.sleep(0.01) def get_stack_trace(self): From e367c15b83b260594d3d649639f3eac3427d662a Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Sat, 8 Aug 2026 10:56:02 +1000 Subject: [PATCH 58/74] debugpy: Derive serial_dap from the channel in use. `caps["serial_dap"]` reports whether this session's DAP channel is a stream rather than a TCP socket, which is knowable exactly and is what a host deciding whether to bridge actually needs. Probing the board for a spare CDC instead could only ever be answered after the device had already chosen its channel, and could disagree with that choice. Signed-off-by: Andrew Leech --- python-ecosys/debugpy/debugpy/public_api.py | 15 ++++++-- .../debugpy/debugpy/server/debug_session.py | 36 ++++++++----------- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/public_api.py b/python-ecosys/debugpy/debugpy/public_api.py index 3b075dccf..70f14e8b6 100644 --- a/python-ecosys/debugpy/debugpy/public_api.py +++ b/python-ecosys/debugpy/debugpy/public_api.py @@ -119,7 +119,7 @@ def _accept_and_initialize(): client_sock, client_addr = listener.accept() print(f"Debugger connected from {format_client_addr(client_addr)}") - _debug_session = DebugSession(client_sock) + _debug_session = DebugSession(client_sock, is_stream) print("[DAP] Waiting for initialize request...") init_message = _debug_session.channel.recv_message() @@ -195,12 +195,21 @@ def get_capabilities(): Uses the active session's probe result if a session exists, otherwise probes directly. Values always come from probing the running - interpreter, never from a build/variant name. + interpreter, never from a build/variant name - except `serial_dap`, + which comes from whichever of `_debug_session`/`_listener` exists: the + boot script calls this between `listen()`/`listen_stream()` and + `wait_for_client()` (before a session exists), so `_listener` is the + only place the channel choice is recorded yet. + + Call this after `listen()`/`listen_stream()`; called before either, or + after `disconnect()`/a `wait_for_client()` timeout has cleared both + globals, `serial_dap` reports `False` regardless of which channel a + prior session used. """ global _debug_session if _debug_session is not None: return _debug_session.capabilities - return DebugSession.probe_capabilities() + return DebugSession.probe_capabilities(isinstance(_listener, StreamTransport)) def breakpoint(): diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index 200b25176..218a2ad62 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -54,7 +54,7 @@ def _is_placeholder_local_name(name): class DebugSession: """Manages a debugging session with a DAP client.""" - def __init__(self, client_socket): + def __init__(self, client_socket, is_stream): self.debug_logging = False # Initialize first self.channel = JsonMessageChannel(client_socket, self._debug_print) self.pdb = PdbAdapter() @@ -67,8 +67,11 @@ def __init__(self, client_socket): self.paused = False self.configuration_done = False self._pumping = False + # is_stream comes from the caller (_accept_and_initialize already + # knows which kind of channel client_socket is) rather than being + # re-derived here, so there is exactly one place that decides it. # Probed once at session start; never inferred from a build/variant name. - self.capabilities = self.probe_capabilities() + self.capabilities = self.probe_capabilities(is_stream) self.pdb.capabilities = self.capabilities def _debug_print(self, message): @@ -81,37 +84,28 @@ def _baremetal(self) -> bool: return sys.platform not in ("linux") # to be expanded @staticmethod - def _probe_serial_dap(): - """Whether this board exposes a second CDC interface dedicated to DAP. - - No port implements the board-specific detection yet (which TinyUSB - CDC instance is DAP's, if any, is per-board configuration - e.g. rp2's - second `machine.USBDevice` CDC), so this always reports `False`; a - board's own boot script never gets to choose `listen_stream()` over - `listen()` on that basis until a real check lands here. Kept as its - own probe (mirroring `f_back`/`save_names`/`set_local` above) so the - one place a board-specific check would go is unambiguous, and so the - capability is always present in `caps` rather than only sometimes. - """ - return False - - @staticmethod - def probe_capabilities(): + def probe_capabilities(is_stream): """Probe what the running firmware actually supports. Returns a dict with at least `settrace`, `save_names`, `set_local`, `f_back` and `serial_dap`, each derived by exercising the real interpreter - never by reading a build/variant name, which does not reliably reflect what a given firmware image supports (see - BACKGROUND.md). Safe to call on both the unix port and bare-metal - builds; never raises. + BACKGROUND.md). `serial_dap` is the one exception to "probe, don't + ask": whether the DAP channel is a stream rather than a TCP socket + is a fact about *this session*, decided by whichever boot script + called `listen_stream()` vs `listen()`. `is_stream` is that fact - + required, not defaulted, so a caller cannot silently report "not a + stream" by forgetting the argument - passed in by the caller rather + than guessed here, so the two can never disagree. Safe to call on + both the unix port and bare-metal builds; never raises. """ caps = { "settrace": hasattr(sys, "settrace"), "f_back": False, "save_names": False, "set_local": False, - "serial_dap": DebugSession._probe_serial_dap(), + "serial_dap": is_stream, } if not caps["settrace"]: return caps From df0a46ebe0f71c55edabf6992cc44247b3ea8724 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Sat, 8 Aug 2026 18:03:51 +1000 Subject: [PATCH 59/74] debugpy: Probe save_names on code the firmware compiled. Local names belong to the code object that declares them, so reading this module's own frame reports whichever compiler produced debugpy. mpy-cross does not persist names into .mpy (LOCALNAMES_PERSIST is off because it corrupts line numbers), so an installed .mpy copy reported save_names False on firmware that supports it, and the launcher's capability cross-check then rejected a correct manifest. Compile a throwaway function at probe time and read that frame instead, falling back to the caller's frame where there is no exec to compile with. Signed-off-by: Andrew Leech --- .../debugpy/debugpy/server/debug_session.py | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index 218a2ad62..6066b9695 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -51,6 +51,38 @@ def _is_placeholder_local_name(name): return name[len("local_") :].isdigit() +# Compiled by the running firmware at probe time; see _probe_local_names. +_LOCAL_NAMES_PROBE_SRC = """ +def _probe(): + _mpdbg_probe_local = 1 + return list(sys._getframe().f_locals.keys()) +""" + + +def _probe_local_names(frame): + """Local names as seen in a frame the *running firmware* compiled. + + Names are attached to a code object when that object is compiled, so + reading this module's own frame measures whichever compiler produced + this module, not the firmware. Those differ whenever debugpy is + installed cross-compiled: mpy-cross only persists names into .mpy with + MICROPY_PY_SYS_SETTRACE_LOCALNAMES_PERSIST, which is off by default (it + corrupts line numbers), so an .mpy install always reports placeholders + however the firmware was built. + + Compiling a throwaway function here measures the firmware's own + compiler, which is what the `save_names` capability claims. A firmware + without `exec` cannot compile source at all, so there `frame` - the + caller's own - is the only frame available and the honest answer. + """ + namespace = {"sys": sys} + try: + exec(_LOCAL_NAMES_PROBE_SRC, namespace) + except Exception: + return list(frame.f_locals.keys()) + return namespace["_probe"]() + + class DebugSession: """Manages a debugging session with a DAP client.""" @@ -99,6 +131,10 @@ def probe_capabilities(is_stream): stream" by forgetting the argument - passed in by the caller rather than guessed here, so the two can never disagree. Safe to call on both the unix port and bare-metal builds; never raises. + + `save_names` is measured on code the firmware compiles here rather + than on this module's own frame, so it reports the firmware and not + how debugpy itself was deployed (see _probe_local_names). """ caps = { "settrace": hasattr(sys, "settrace"), @@ -126,7 +162,7 @@ def probe_capabilities(is_stream): pass try: - local_names = list(frame.f_locals.keys()) + local_names = _probe_local_names(frame) # An empty locals dict (e.g. probing from module scope) proves # nothing either way; only trust the signal when there is at # least one local name to inspect for the placeholder pattern. From b2d0a6a5c1ed7ef040121f056e3f324ff4040dd1 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Sat, 8 Aug 2026 18:21:34 +1000 Subject: [PATCH 60/74] debugpy: Accept every idle errno, and finish every socket write. `JsonMessageChannel` polls a socket that always carries a timeout, so an idle poll always ends in an error rather than in data. Which error is a property of the network stack: the unix port raises EAGAIN, lwIP - every WiFi board - raises ETIMEDOUT once the timeout elapses. Only EAGAIN and EWOULDBLOCK were accepted, so on a board the channel closed on its first idle poll, which arrives immediately after `initialize` drops the timeout to 1 ms. Measured on a PYBD-SF6W over WiFi: `recv` with no data raises 110 at any non-zero timeout and 11 only at timeout 0. `send_message` also called `sock.send()` once and ignored the returned count. A socket that accepts a prefix truncates the frame mid-`Content-Length`, which has no protocol-level symptom - the client waits forever for a message that will never complete and misreads every later frame as its body. Writes are now driven to completion with a stall budget, bounded because the write runs inside the trace callback. `StreamTransport.send` returns a byte count to match the socket contract it presents. Signed-off-by: Andrew Leech --- .../debugpy/debugpy/common/messaging.py | 59 +++++++++++++++++-- .../debugpy/common/stream_transport.py | 10 ++-- 2 files changed, 59 insertions(+), 10 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/common/messaging.py b/python-ecosys/debugpy/debugpy/common/messaging.py index 7d704f6a8..cf2f94991 100644 --- a/python-ecosys/debugpy/debugpy/common/messaging.py +++ b/python-ecosys/debugpy/debugpy/common/messaging.py @@ -1,8 +1,24 @@ """JSON message handling for DAP protocol.""" import json +import time + from .constants import MSG_TYPE_REQUEST, MSG_TYPE_RESPONSE, MSG_TYPE_EVENT +# "Nothing to read / no room to write right now" on a socket carrying a +# timeout, which this channel always does (see DebugSession's settimeout +# calls). Which errno says that depends on the network stack, not on the +# situation: the unix port raises EAGAIN, and lwIP - every WiFi board - +# raises ETIMEDOUT once the timeout elapses. None of them means the peer went +# away, so none may close the channel. A stack whose errno is missing here +# drops the session on its first idle poll, which is immediately. +_WOULD_BLOCK = (11, 35, 110) # EAGAIN, EWOULDBLOCK, ETIMEDOUT + +# How long a single DAP frame may take to drain into the socket before the +# peer is treated as gone. Generous: it is a stall budget, not a latency +# target, and the loop returns as soon as the bytes are written. +_SEND_DEADLINE_MS = 5000 + class JsonMessageChannel: """Handles JSON message communication over a socket using DAP format.""" @@ -47,10 +63,41 @@ def send_message(self, msg_type, command=None, **kwargs): content = json_str.encode("utf-8") header = f"Content-Length: {len(content)}\r\n\r\n".encode("utf-8") - try: - self.sock.send(header + content) - except OSError: - self.closed = True + self._send_all(header + content) + + def _send_all(self, data): + """Write every byte of one frame, or close the channel trying. + + `sock.send()` is not `sendall()`: it may accept a prefix and return + the count, and on a socket carrying a timeout it reports a full + transmit buffer as an error instead of blocking. Either one truncates + a DAP frame mid-`Content-Length`, which desynchronises the stream for + the rest of the session rather than failing visibly. Both are ordinary + on a board - a `variables` response is easily larger than lwIP's + window - so the write is driven to completion here. + + `sendall()` is not the answer: MicroPython does not support it on a + socket with a timeout, which is the only kind this channel has. + """ + view = memoryview(data) + sent = 0 + start = time.ticks_ms() + while sent < len(view): + try: + sent += self.sock.send(view[sent:]) + except OSError as e: + if getattr(e, "errno", None) not in _WOULD_BLOCK: + self.closed = True + return + if time.ticks_diff(time.ticks_ms(), start) > _SEND_DEADLINE_MS: + self._debug_print( + "[DAP] send stalled with {} of {} bytes written; closing".format( + sent, len(view) + ) + ) + self.closed = True + return + time.sleep(0.001) def send_request(self, command, **kwargs): """Send a request message.""" @@ -106,13 +153,13 @@ def recv_message(self): try: data = self.sock.recv(4096) if not data: - # A truly empty read (as opposed to EAGAIN/EWOULDBLOCK, + # A truly empty read (as opposed to a _WOULD_BLOCK errno, # handled below) means the peer closed the connection. self.closed = True return None self._recv_buffer += data except OSError as e: - if not (hasattr(e, "errno") and e.errno in (11, 35)): # EAGAIN, EWOULDBLOCK + if getattr(e, "errno", None) not in _WOULD_BLOCK: self.closed = True return None # No new data available right now - fall through and try to diff --git a/python-ecosys/debugpy/debugpy/common/stream_transport.py b/python-ecosys/debugpy/debugpy/common/stream_transport.py index 0e7c269b5..69be1f8e3 100644 --- a/python-ecosys/debugpy/debugpy/common/stream_transport.py +++ b/python-ecosys/debugpy/debugpy/common/stream_transport.py @@ -15,10 +15,11 @@ class StreamTransport: `recv` raises `OSError(11)` (EAGAIN) when no data has arrived within the current timeout, matching a non-blocking socket, and returns `b""` once - the stream is at EOF. `send` raises `OSError(110)` (ETIMEDOUT) if the - writer won't take the rest of the data within the current timeout, rather - than returning having written only part of it - a short write here would - desync `messaging.py`'s Content-Length framing with no way to resync. + the stream is at EOF. `send` returns the byte count a socket would, but + only ever the whole buffer: it raises `OSError(110)` (ETIMEDOUT) if the + writer won't take the rest of the data within the current timeout rather + than reporting a partial write, since a truncated frame would desync + `messaging.py`'s Content-Length framing with no way to resync. `settimeout` is mutated repeatedly at runtime by `public_api.py`/`debug_session.py` and governs both directions, matching a real socket; `poll(ms)` is called fresh on every `recv`/`send` rather @@ -93,6 +94,7 @@ def send(self, data): written = self._writer.write(mv[off:]) if written: off += written + return total # the socket contract: how many bytes went out def close(self): try: From f448206b45555f2061d3165fa63e5514a1410f46 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Sat, 8 Aug 2026 19:01:30 +1000 Subject: [PATCH 61/74] debugpy: Replace both keys a file's breakpoints are stored under. `set_breakpoints` keeps each file's set under two keys: the path the client sent, and the name the debuggee reports for the same file, because `should_stop` matches on `frame.f_code.co_filename`. The client key was replaced unconditionally, the debuggee key only from inside the per-breakpoint loop. DAP has no remove-breakpoint request - a `setBreakpoints` carrying an empty list is how a client removes them - so a removal never entered the loop and left the debuggee key armed, and the program kept stopping at breakpoints the client had cleared. With several breakpoints in one file the key was also reset on each pass, leaving only the last of them. The two keys are the same string unless a path mapping is configured, so this is invisible until a launch config sets `pathMappings`. Signed-off-by: Andrew Leech --- .../debugpy/debugpy/server/pdb_adapter.py | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index 2a64aa19b..b43b73062 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -238,21 +238,30 @@ def _filename_as_debugger(self, path: str): return path def set_breakpoints(self, filename: str, breakpoints: list[dict]): - """Set breakpoints for a file.""" - self.breakpoints[filename] = {} + """Replace the breakpoint set for one file. + + DAP sends the whole set for a source on every request, so this + replaces rather than adds, and an empty list is how a client removes + every breakpoint in a file. + + The set is stored under both the path the client used and the name + the debuggee knows the same file by, because `should_stop` matches + whatever `frame.f_code.co_filename` reports. Both are replaced + together: clearing only one leaves the other still armed. + """ local_name = self._filename_as_debugee(filename) self.file_mappings[local_name] = filename + self.breakpoints[filename] = {} + self.breakpoints[local_name] = {} + actual_breakpoints = [] - self._debug_print(f"[PDB] Setting breakpoints for file: {filename}") + self._debug_print(f"[PDB] Setting breakpoints for file: {filename} (as {local_name})") for bp in breakpoints: line = bp.get("line") if line: - if local_name != filename: - self.breakpoints[local_name] = {} - self._debug_print(f"[>>>] Setting breakpoints for local: {local_name}:{line}") - self.breakpoints[local_name][line] = {} self.breakpoints[filename][line] = {} + self.breakpoints[local_name][line] = {} actual_breakpoints.append( {"line": line, "verified": True, "source": {"path": filename}} ) From 7746778f2084d9423ad55feeab734df0ff79b854 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Sat, 8 Aug 2026 19:48:40 +1000 Subject: [PATCH 62/74] debugpy: Give StreamTransport.send the socket short-write contract. `send` writes once and returns how many bytes went out, raising EAGAIN only when nothing could be written. `messaging.py`'s `_send_all` already drives a whole frame out from that count. A whole-buffer contract cannot express a partial write: it loops internally and raises OSError 110 when the timeout expires mid-frame, discarding the count of what has already gone out, so `_send_all` retries from offset 0 and resends that prefix and the peer's Content-Length framing can never resynchronise. On stm32 that hit every DAP message over 1024 bytes: `USB_VCP.write` takes at most `MICROPY_HW_USB_CDC_TX_DATA_SIZE` per call and reports the short count, and `public_api.py` drops the channel timeout to 1 ms once `initialize` is answered. Measured on a PYBD-SF6W: a 16 KB `evaluate` response over the second CDC in 0.20 s, 81.7 kB/s. Signed-off-by: Andrew Leech --- .../debugpy/common/stream_transport.py | 54 +++++++++---------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/common/stream_transport.py b/python-ecosys/debugpy/debugpy/common/stream_transport.py index 69be1f8e3..a3034a419 100644 --- a/python-ecosys/debugpy/debugpy/common/stream_transport.py +++ b/python-ecosys/debugpy/debugpy/common/stream_transport.py @@ -1,7 +1,6 @@ """Socket-shaped adapter for running the DAP channel over a stream, not a socket.""" import select -import time class StreamTransport: @@ -15,15 +14,14 @@ class StreamTransport: `recv` raises `OSError(11)` (EAGAIN) when no data has arrived within the current timeout, matching a non-blocking socket, and returns `b""` once - the stream is at EOF. `send` returns the byte count a socket would, but - only ever the whole buffer: it raises `OSError(110)` (ETIMEDOUT) if the - writer won't take the rest of the data within the current timeout rather - than reporting a partial write, since a truncated frame would desync - `messaging.py`'s Content-Length framing with no way to resync. - `settimeout` is mutated repeatedly at runtime by - `public_api.py`/`debug_session.py` and governs both directions, matching - a real socket; `poll(ms)` is called fresh on every `recv`/`send` rather - than cached, so each change takes effect immediately. + the stream is at EOF. `send` writes once and reports how much went out, + also matching a socket, raising EAGAIN only when nothing could be + written; driving a whole frame out is `messaging.py`'s `_send_all`, the + one place that knows where the frame boundary is. `settimeout` is mutated + repeatedly at runtime by `public_api.py`/`debug_session.py` and governs + both directions, matching a real socket; `poll(ms)` is called fresh on + every `recv`/`send` rather than cached, so each change takes effect + immediately. """ def __init__(self, reader, writer=None): @@ -77,24 +75,24 @@ def recv(self, n): return bytes(mv[:got]) def send(self, data): - mv = memoryview(data) - off = 0 - total = len(mv) - if self._timeout is None: - deadline = None - else: - deadline = time.ticks_add(time.ticks_ms(), max(0, int(self._timeout * 1000))) - while off < total: - if deadline is None: - self._write_poller.poll() - else: - remaining_ms = time.ticks_diff(deadline, time.ticks_ms()) - if remaining_ms <= 0 or not self._write_poller.poll(max(0, remaining_ms)): - raise OSError(110) # ETIMEDOUT: writer never drained within the timeout - written = self._writer.write(mv[off:]) - if written: - off += written - return total # the socket contract: how many bytes went out + # A short write is normal, not exceptional: a CDC interface takes at + # most its transmit buffer per call (1024 bytes on stm32) and reports + # the count, exactly as a socket reports a short send. Reporting that + # count is what lets the caller resume from the right offset - a + # whole-buffer contract would have to raise once the timeout expired + # mid-frame, throwing away the count of what had already gone out, and + # the retry would then re-send that prefix and desynchronise the + # Content-Length framing it was meant to protect. + timeout_ms = None if self._timeout is None else max(0, int(self._timeout * 1000)) + if not self._write_poller.poll(timeout_ms): + raise OSError(11) # EAGAIN: no room within the timeout + # `write()` answers a non-blocking stream that took nothing with None + # rather than 0, so both are treated as "came back not ready after + # all" - the poll above can only promise room at the moment it ran. + written = self._writer.write(memoryview(data)) + if not written: + raise OSError(11) + return written def close(self): try: From 66ed177fef578b7487893011a70dc5879fc59870 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Sun, 9 Aug 2026 01:28:42 +1000 Subject: [PATCH 63/74] debugpy: Make DAP path translation symmetric and boundary-aware. `_filename_as_debugee` and `_filename_as_debugger` are now exact inverses of each other: one mapping list, first match wins in both directions, and a mapping matches a path only at a path-separator boundary. A bare string prefix let a sibling directory sharing the root's name (`/home/dev/src-old` under root `/home/dev/src`) be rewritten into a device path that cannot exist, and `_filename_as_debugee` kept rewriting after its first match while its inverse stopped at one, so a second mapping could be applied to a path the first had already translated. `_filename_as_debugger`'s two loops were identical, making the second unreachable; one remains. Signed-off-by: Andrew Leech --- .../debugpy/debugpy/server/pdb_adapter.py | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index b43b73062..5886e396a 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -202,38 +202,46 @@ def set_trace_function(self, trace_func): raise RuntimeError("sys.settrace not available") def _filename_as_debugee(self, path: str): + """Translate an IDE-side (vscode) path to the runtime's own path. + + The first mapping whose `vscode_path` names `path` itself or a + directory containing it wins - matched on a path-separator boundary, + not a bare string prefix, so a sibling directory that merely shares + the root's name (`/home/dev/src-old` against root `/home/dev/src`) + is left untranslated instead of being rewritten into a device path + that cannot exist. First-match-wins makes this the exact inverse of + `_filename_as_debugger` below. + """ # check if we have a 1:1 file mapping for this path if self.file_mappings.get(path): return self.file_mappings[path] - # Check if we have a folder mapping for this path for runtime_path, vscode_path in self.path_mappings: - if path.startswith(vscode_path): - path = path.replace(vscode_path, runtime_path, 1) + if path == vscode_path or path.startswith(vscode_path + "/"): + path = runtime_path + path[len(vscode_path) :] if path.startswith("//"): path = path[1:] + return path # If no mapping found, return the original path return path def _filename_as_debugger(self, path: str): - """Convert a file path to the debugger's expected format.""" + """Translate a runtime path (`frame.f_code.co_filename`) to the IDE's path. + + Inverse of `_filename_as_debugee`: same first-match, boundary-aware + rule, applied in the other direction. + """ path = path or "" if not path: return path if path.startswith("<"): # Special case for or similar return path - # Check if we have a 1:1 file mapping for this path for runtime_path, vscode_path in self.path_mappings: - if path.startswith(runtime_path): - path = path.replace(runtime_path, vscode_path, 1) - return path - - # Check if we have a folder mapping for this path - for runtime_path, vscode_path in self.path_mappings: - if path.startswith(runtime_path): - path = path.replace(runtime_path, vscode_path, 1) + if path == runtime_path or path.startswith(runtime_path + "/"): + path = vscode_path + path[len(runtime_path) :] if path.startswith("//"): path = path[1:] + return path # If no mapping found, return the original path return path From caa6bbcb3aeeab688da939d938310ad54222efc4 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Sun, 9 Aug 2026 01:28:50 +1000 Subject: [PATCH 64/74] debugpy: Keep the message pump from raising into the debuggee. `process_pending_messages` restored the channel's blocking mode in a `finally`, and the loop it guards is what closes that channel: a DAP `disconnect` request handled there runs the whole session teardown, so on the way out of that request the socket is gone and `settimeout` raises EBADF. Every caller is `_trace_function`, so that exception landed in whichever line of the debugged program was being traced and killed it with a traceback naming an errno belonging to the debug channel. Deterministic for every session that ends the ordinary way, and downstream it made mpremote's mount teardown read the dead program's traceback as a device fault. The restore now happens only against a channel that is still open, and a channel found closed ends the session rather than leaving a trace function installed to pump it. Signed-off-by: Andrew Leech --- .../debugpy/debugpy/server/debug_session.py | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index 6066b9695..da4f7a0c1 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -259,6 +259,12 @@ def process_pending_messages(self): recv() then waits for a message the client will not send until it has seen an event this loop is what produces. MicroPython sockets have no gettimeout(), so the nesting is tracked rather than the timeout saved. + + Nothing here may raise: every caller is `_trace_function`, so an + exception escaping this method lands in whichever line of the + debugged program was being traced and kills that program with an + errno belonging to the debug channel, not to anything the program + did. """ if self._pumping: return @@ -277,9 +283,23 @@ def process_pending_messages(self): # No messages available or socket error pass finally: - # Reset to blocking mode - self.channel.sock.settimeout(None) self._pumping = False + # Reset to blocking mode - but only against a channel that is + # still there. Restoring the timeout is a socket operation like + # any other and fails on a closed socket, and the loop above is + # exactly what closes it: a `disconnect` request handled there + # runs the whole session teardown, so on the way out of that + # request this socket is already gone. A client that vanishes + # without sending `disconnect` reaches the same place without + # setting the flag, hence the guard as well as the check. + # Either way the session is over, so end it here rather than + # leaving a trace function installed that pumps a dead channel. + if self.channel.closed: + return + try: + self.channel.sock.settimeout(None) + except Exception: + self.disconnect() def _handle_message(self, message): """Handle incoming DAP messages.""" @@ -410,8 +430,13 @@ def _handle_attach(self, seq, args): # get debugger root and debuggee root from pathMappings for pm in args.get("pathMappings", []): - # debuggee - debugger - self.pdb.path_mappings.append((pm.get("remoteRoot", "./"), pm.get("localRoot", "./"))) + # debuggee - debugger. Trailing slashes are stripped so "/remote" + # and "/remote/" name the same root: pdb_adapter's translation + # matches a mapping on a path-separator boundary it adds itself, + # and a root that already carries one would double it up. + remote_root = pm.get("remoteRoot", "./").rstrip("/") + local_root = pm.get("localRoot", "./").rstrip("/") + self.pdb.path_mappings.append((remote_root, local_root)) # # TODO: justMyCode, debugOptions , # Enable trace function From 9457f2d0a67702c978e380db45f31aae10f224da Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Sun, 9 Aug 2026 02:39:56 +1000 Subject: [PATCH 65/74] debugpy: Stop at a breakpoint only on the events that precede the line. MicroPython's return event reports the last line the frame executed, where CPython's reports the line that ended the frame. should_stop compared the line against the breakpoint table for every event, so a breakpoint on a function's final line stopped twice per call: once as the line event about to run it, once as the return event on the way out, on a frame that has already produced its value and where stepping or inspecting locals means nothing. A hit is now recognised for call and line events only. call has to stay: it reports the def line, so it is the only event that can ever match a breakpoint placed there. The dead first half of the check goes with it. It re-read the same two frame attributes, tested `self.paused` and the breakpoint table, and then only printed - reaching no return of its own and falling through to the real check below. `paused` is written in three places and read in no stop decision anywhere, so the DAP pause request answers success and never stops the target; that is a separate gap, not something this line was providing. Verified against a real event stream under a built unix firmware by the mpy-debugpy host harness, which asserts that a return event was delivered at the breakpoint line, so the test cannot pass vacuously on a firmware that never produced one. Signed-off-by: Andrew Leech --- .../debugpy/debugpy/server/pdb_adapter.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index 5886e396a..bd6be08ad 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -284,14 +284,6 @@ def should_stop(self, frame, event: str, arg): self.current_frame = frame self.hit_breakpoint = False - # Get frame information - filename = frame.f_code.co_filename - lineno = frame.f_lineno - # Check for exact filename match first - if self.paused or (filename in self.breakpoints and lineno in self.breakpoints[filename]): - self._debug_print(f"[PDB] HIT BREAKPOINT (exact match) at {filename}:{lineno}") - # Record the path mapping (in this case, they're already the same) - # self.file_mappings[filename] = self._filename_as_debugger(filename) # Cache frame attributes to reduce lookup overhead _frame_code = frame.f_code _filename = _frame_code.co_filename @@ -300,8 +292,16 @@ def should_stop(self, frame, event: str, arg): # Optimize dictionary lookups - use .get() to avoid double lookup file_breakpoints = self.breakpoints.get(_filename) if file_breakpoints and _lineno in file_breakpoints: - self.hit_breakpoint = True - return True + # Only an event that is about to run this line counts as a hit. + # `return` reports the last line the frame executed, so a breakpoint + # on a function's final line would otherwise stop a second time on + # the way out, on a frame that has already produced its value. + # `call` reports the `def` line, which is the only event that can + # ever match a breakpoint placed there. + if event in (TRACE_CALL, TRACE_LINE): + self._debug_print(f"[PDB] HIT BREAKPOINT (exact match) at {_filename}:{_lineno}") + self.hit_breakpoint = True + return True else: # file not (yet) matched - this is slow so we do not want to do this often. # TODO: use sys.path[] method to find the file, does not work for frozen .... From 877ccfd1fc3da7adee9237ea12bd30d851016ddb Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Sun, 9 Aug 2026 02:40:11 +1000 Subject: [PATCH 66/74] debugpy: Honour a DAP restart, and add a target console channel. A client's restart button now re-runs the program in the same session instead of being refused. The session outlives the restart deliberately: breakpoints live in the adapter, and a client that sends restart rather than reconnecting does not re-send them, so keeping one session alive is what makes them still bind on the next run, at no re-attach cost. restart is offered only when the code that owns the run loop has said it can re-run its target (enable_restart(), before wait_for_client(), since the capability is answered during initialize). Without that, supportsRestartRequest is false and a restart request is refused with a message saying why - a restart button that silently does nothing is worse than one that is absent. The unwind is a RestartRequest raised from the trace function, because a restart cannot wait for the target to return: the ordinary embedded shape is a main loop that never does, and with no second thread and a message pump that runs inside the trace function, a deliberate raise is the only mechanism there is. It derives from BaseException so a target's own `except Exception:` cannot swallow it and leave the restart silently undone. The handler releases a target parked at a breakpoint and clears any pending step, so it does not stop again on its way out, and sends `continued` so the client's UI does not stay stopped on a frame that is about to cease to exist. wait_for_restart() pumps messages between runs, nothing else being able to read the socket then, and reports the client leaving as False, so a session with nobody left to restart for ends rather than spinning. console() sends text to the client's debug console as a DAP output event. It is the only route a target's own notes have to the user on a transport where device stdout never reaches the host: a mounted serial session's filesystem pump discards everything the device prints. `terminated` is still never sent between runs - a client that sees it tears the session down, which is the opposite of what a re-runnable session is for. Signed-off-by: Andrew Leech --- python-ecosys/debugpy/debugpy/__init__.py | 8 ++ .../debugpy/debugpy/common/constants.py | 1 + python-ecosys/debugpy/debugpy/public_api.py | 43 +++++- .../debugpy/debugpy/server/debug_session.py | 123 +++++++++++++++++- 4 files changed, 171 insertions(+), 4 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/__init__.py b/python-ecosys/debugpy/debugpy/__init__.py index a9a75b628..ed2494e0e 100644 --- a/python-ecosys/debugpy/debugpy/__init__.py +++ b/python-ecosys/debugpy/debugpy/__init__.py @@ -8,26 +8,34 @@ __version__ = "0.1.0" from .public_api import ( + RestartRequest, breakpoint, + console, debug_this_thread, disconnect, + enable_restart, get_capabilities, is_client_connected, listen, listen_stream, wait_for_client, + wait_for_restart, ) from .common.constants import DEFAULT_HOST, DEFAULT_PORT __all__ = [ "DEFAULT_HOST", "DEFAULT_PORT", + "RestartRequest", "breakpoint", + "console", "debug_this_thread", "disconnect", + "enable_restart", "get_capabilities", "is_client_connected", "listen", "listen_stream", "wait_for_client", + "wait_for_restart", ] diff --git a/python-ecosys/debugpy/debugpy/common/constants.py b/python-ecosys/debugpy/debugpy/common/constants.py index 5d9a52204..a0fd245c8 100644 --- a/python-ecosys/debugpy/debugpy/common/constants.py +++ b/python-ecosys/debugpy/debugpy/common/constants.py @@ -37,6 +37,7 @@ CMD_SET_VARIABLE = const("setVariable") CMD_EVALUATE = const("evaluate") CMD_DISCONNECT = const("disconnect") +CMD_RESTART = const("restart") CMD_CONFIGURATION_DONE = const("configurationDone") CMD_THREADS = const("threads") CMD_SOURCE = const("source") diff --git a/python-ecosys/debugpy/debugpy/public_api.py b/python-ecosys/debugpy/debugpy/public_api.py index 70f14e8b6..b65b4c561 100644 --- a/python-ecosys/debugpy/debugpy/public_api.py +++ b/python-ecosys/debugpy/debugpy/public_api.py @@ -5,12 +5,15 @@ import sys from .common.constants import DEFAULT_HOST, DEFAULT_PORT from .common.stream_transport import StreamTransport -from .server.debug_session import DebugSession +from .server.debug_session import DebugSession, RestartRequest _debug_session = None # Bound-but-not-yet-accepted socket, held between listen() and the accept that # wait_for_client() performs. _listener = None +# Set by enable_restart() before a session exists, because the capability it +# controls is answered during the `initialize` that starts one. +_restart_supported = False def listen(port=DEFAULT_PORT, host=DEFAULT_HOST): @@ -119,7 +122,7 @@ def _accept_and_initialize(): client_sock, client_addr = listener.accept() print(f"Debugger connected from {format_client_addr(client_addr)}") - _debug_session = DebugSession(client_sock, is_stream) + _debug_session = DebugSession(client_sock, is_stream, _restart_supported) print("[DAP] Waiting for initialize request...") init_message = _debug_session.channel.recv_message() @@ -190,6 +193,42 @@ def wait_for_client(timeout_s=None): return _debug_session.wait_for_client(timeout_s) +def enable_restart(): + """Declare that this process can re-run its target, so `restart` is offered. + + Must be called before `wait_for_client()`: the capability is answered in the + `initialize` request, which that accepts and handles. Only the code that + owns the run loop can honour a restart, so nothing else turns this on. + """ + global _restart_supported + _restart_supported = True + if _debug_session is not None: + _debug_session.restart_supported = True + + +def wait_for_restart(): + """Block between runs until a restart is requested, or the client leaves. + + Returns True if the target should be run again, False if there is no client + left to run it for. The caller must not be traced while it waits (see + DebugSession.wait_for_restart). + """ + if _debug_session is None: + return False + return _debug_session.wait_for_restart() + + +def console(text): + """Show `text` in the client's debug console, if a session is connected. + + A no-op with no session, so a target can report progress unconditionally + without caring whether it is being debugged. See DebugSession.console for + why anything worth saying goes here as well as to stdout. + """ + if _debug_session is not None: + _debug_session.console(text) + + def get_capabilities(): """Return the firmware capability dict (settrace/save_names/set_local/f_back). diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index da4f7a0c1..6dd188076 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -13,6 +13,7 @@ CMD_LAUNCH, CMD_NEXT, CMD_PAUSE, + CMD_RESTART, CMD_SCOPES, CMD_SET_BREAKPOINTS, CMD_SET_VARIABLE, @@ -24,6 +25,7 @@ CMD_VARIABLES, EVENT_CONTINUED, EVENT_INITIALIZED, + EVENT_OUTPUT, EVENT_STOPPED, EVENT_TERMINATED, STOP_REASON_BREAKPOINT, @@ -39,6 +41,24 @@ from .pdb_adapter import PdbAdapter +class RestartRequest(BaseException): + """Raised inside the debugged program to unwind it back to its launcher. + + A restart cannot wait for the target to return - the ordinary embedded + shape is a main loop that never does - and nothing else on the device can + interrupt it: there is no second thread, and the DAP message pump runs + inside the trace function, where raising would kill the program with an + error belonging to the debug channel. A deliberate raise from the trace + function is the one remaining mechanism, and it is how CPython debuggers + unwind a target too. + + Derived from BaseException so a target's own `except Exception` does not + swallow the unwind and leave the restart silently undone. A target that + catches BaseException (or uses a bare `except:`) can still swallow it; + there is no mechanism behind that, only documentation. + """ + + def _is_placeholder_local_name(name): """True if `name` is a positional `local_N` placeholder, not a real name. @@ -86,7 +106,7 @@ def _probe_local_names(frame): class DebugSession: """Manages a debugging session with a DAP client.""" - def __init__(self, client_socket, is_stream): + def __init__(self, client_socket, is_stream, restart_supported=False): self.debug_logging = False # Initialize first self.channel = JsonMessageChannel(client_socket, self._debug_print) self.pdb = PdbAdapter() @@ -99,6 +119,11 @@ def __init__(self, client_socket, is_stream): self.paused = False self.configuration_done = False self._pumping = False + # Whether the launcher can actually re-run the target. Only it knows, + # so a session never assumes it: `restart` is refused, and + # supportsRestartRequest not advertised, unless it was told otherwise. + self.restart_supported = restart_supported + self.restart_requested = False # is_stream comes from the caller (_accept_and_initialize already # knows which kind of channel client_socket is) rather than being # re-derived here, so there is exactly one place that decides it. @@ -355,6 +380,8 @@ def _handle_request(self, message): self._handle_set_variable(seq, args) elif command == CMD_EVALUATE: self._handle_evaluate(seq, args) + elif command == CMD_RESTART: + self._handle_restart(seq, args) elif command == CMD_DISCONNECT: self._handle_disconnect(seq, args) elif command == CMD_CONFIGURATION_DONE: @@ -391,7 +418,10 @@ def _handle_initialize(self, seq, args): # "supportsModulesRequest": False, # "additionalModuleColumns": [], # "supportedChecksumAlgorithms": [], - # "supportsRestartRequest": False, + # Advertised only when the launcher runs the target in a loop it + # can re-enter; a client that sees this offers a restart button + # and expects the debuggee to come back, not the session to end. + "supportsRestartRequest": self.restart_supported, # "supportsExceptionOptions": False, # "supportsValueFormattingOptions": False, # "supportsExceptionInfoRequest": False, @@ -566,6 +596,90 @@ def _handle_evaluate(self, seq, args): except Exception as e: self.channel.send_response(CMD_EVALUATE, seq, success=False, message=str(e)) + def _handle_restart(self, seq, args): + """Handle restart request: unwind the target so the launcher re-runs it. + + The session outlives the restart deliberately. Breakpoints live in + `self.pdb`, and a client that sent `restart` rather than reconnecting + does not re-send them, so keeping the one session alive is what makes + them still bind on the next run - and it costs no re-attach round trip. + + Only the flag is set here. Whatever the target is doing, it is doing it + somewhere below this call: this runs inside the trace function, so the + unwind happens where that returns to, not here. + """ + if not self.restart_supported: + self.channel.send_response( + CMD_RESTART, + seq, + success=False, + message="the target was not launched in a loop that can re-run it", + ) + return + + self.restart_requested = True + # A target stopped at a breakpoint is inside wait_for_continue(); it has + # to be let go before it can be unwound. Stepping is cleared with it, so + # a pending step does not stop the target again on its way out. + self.stepping = False + self.pdb.step_mode = None + self.pdb.paused = False + self.pdb.continue_event = True + + self.channel.send_response(CMD_RESTART, seq) + # The client last heard `stopped`; without this its UI stays stopped on + # a frame that is about to cease to exist. + self.channel.send_event(EVENT_CONTINUED, threadId=self.thread_id, allThreadsContinued=True) + + def _raise_if_restarting(self): + """Unwind the target if a restart arrived, clearing the request. + + Cleared here, not by the launcher, so the exception itself is the whole + signal: one restart unwinds one run, and a second request during the + unwind is a fresh one rather than a repeat of this. + """ + if self.restart_requested: + self.restart_requested = False + raise RestartRequest + + def console(self, text): + """Show `text` in the client's debug console (a DAP `output` event). + + The only route a target's own notes have to the user on a transport + where device stdout never reaches the host: a mounted serial session's + filesystem pump discards everything the device prints. Run-boundary + markers go through here as well as to stdout so they are visible on + every transport, not just the ones with a readable console. + """ + self.channel.send_event(EVENT_OUTPUT, category="console", output=text) + + def wait_for_restart(self): + """Pump DAP messages between runs, until a restart or the client leaves. + + A target that returned normally is no longer generating trace events, so + nothing would service the socket and a restart request would sit unread + in it. Returns True if a restart arrived, False if the client went away, + in which case there is nothing left to restart for. + + `terminated` is deliberately not what marks the end of a run: a client + that sees it tears the session down, which is the opposite of what loop + mode exists for. An `output` event says the same thing without ending + anything, and is the only notice a client gets that the program has + finished and a restart is what comes next. + + Callers must not be traced while they wait here: a restart handled by + the pump below would otherwise unwind the caller itself. + """ + self.console("Target finished; waiting for a restart request.\n") + while True: + if self.restart_requested: + self.restart_requested = False + return True + if not self.connected or self.channel.closed: + return False + self.process_pending_messages() + time.sleep(0.01) + def _handle_disconnect(self, seq, args): """Handle disconnect request.""" self.channel.send_response(CMD_DISCONNECT, seq) @@ -616,6 +730,9 @@ def _trace_function(self, frame, event: str, arg): # Process any pending DAP messages frequently self.process_pending_messages() + # Before any breakpoint work: a restart that arrived above wants this + # program gone, not stopped somewhere else on the way out. + self._raise_if_restarting() # Handle breakpoints and stepping if self.pdb.should_stop(frame, event, arg): self._send_stopped_event( @@ -627,6 +744,8 @@ def _trace_function(self, frame, event: str, arg): ) # Wait for continue command self.pdb.wait_for_continue() + # A restart is one of the things that ends that wait. + self._raise_if_restarting() # The trace function is invoked (with event set to 'call') whenever a new local scope is entered; # it should return a reference to a local trace function to be used for the new scope, From f179e80dd1151f49341c30523a94c163e7b85821 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Sun, 9 Aug 2026 03:09:37 +1000 Subject: [PATCH 67/74] debugpy: Correct the README's claims and the placeholder format. The feature list advertised "Pause/continue execution" and "locals generally not supported". Neither holds: `pause` sets a flag that no stop decision reads, so a running target is never stopped by it, and locals are readable whenever the firmware reports `save_names` (under real names) or not (as positional placeholders). The requirements section now also names the macro that decides which of those two a build gives. `_is_placeholder_local_name`'s docstring described the placeholder format as `local_1`, `local_2` and attributed it to `MICROPY_PY_SYS_SETTRACE_SAVE_NAMES`. The format is `local_%02d`, 0-based (`py/profile.c:183,207`), and the macro on this lineage is `MICROPY_PY_SYS_SETTRACE_LOCALNAMES`. The check itself tests the digits rather than the width, so it was already correct for either spelling; that is now stated instead of implied. Signed-off-by: Andrew Leech --- python-ecosys/debugpy/README.md | 14 +++++++++++--- .../debugpy/debugpy/server/debug_session.py | 10 ++++++---- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/python-ecosys/debugpy/README.md b/python-ecosys/debugpy/README.md index 87af4da8e..f765d1dc2 100644 --- a/python-ecosys/debugpy/README.md +++ b/python-ecosys/debugpy/README.md @@ -10,13 +10,21 @@ such as VS Code debugging support. - Breakpoints - Step over/into/out - Stack trace inspection - - Variable inspection (globals, locals generally not supported) + - Variable inspection: globals, and locals when the firmware reports + `save_names` (otherwise positional `local_NN` placeholders). Locals are + read-only; no MicroPython build implements local-variable write-back. - Expression evaluation - - Pause/continue execution + - Continue + +`pause` is accepted and answered, but does not stop a running target: nothing +in the trace hook consults the flag it sets. Use a breakpoint instead. ## Requirements -- MicroPython with `sys.settrace` support (enabled with `MICROPY_PY_SYS_SETTRACE`) +- MicroPython with `sys.settrace` support (enabled with `MICROPY_PY_SYS_SETTRACE`). + Real local-variable names additionally need + `MICROPY_PY_SYS_SETTRACE_LOCALNAMES`, where the build provides it; without it + locals are reported as positional placeholders. - Socket support for network communication - JSON support for DAP message parsing diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index 6dd188076..507418d5b 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -60,11 +60,13 @@ class RestartRequest(BaseException): def _is_placeholder_local_name(name): - """True if `name` is a positional `local_N` placeholder, not a real name. + """True if `name` is a positional `local_NN` placeholder, not a real name. - Without MICROPY_PY_SYS_SETTRACE_SAVE_NAMES, frame.f_locals synthesizes - names as `local_1`, `local_2`, ... (see py/profile.c). This is the only - reliable signal that separates the two cases at runtime. + Without MICROPY_PY_SYS_SETTRACE_LOCALNAMES, frame.f_locals synthesizes + names as `local_00`, `local_01`, ... (`local_%02d` in py/profile.c). The + digit test below stays width-agnostic so a build that numbers them + differently is still recognised; this is the only reliable signal that + separates the two cases at runtime. """ if not name.startswith("local_"): return False From 9c2af1abccfefc8e49a7530f37334f0151875799 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Sun, 9 Aug 2026 03:32:03 +1000 Subject: [PATCH 68/74] debugpy: Make the development guide match a measured session. The guide is what someone reads to find out how to watch the DAP conversation. Three of its claims did not hold, each checked against a live session rather than by reading: - The server's `[DAP] RECV`/`SEND` trace was presented as unconditional. It is gated on `debug_logging`, which is set from the `attach` request's `logToFile`, so without that flag only a handful of unconditional `[DAP]` progress lines appear - and because the flag arrives with `attach`, nothing up to and including that request is ever traced. The first logged line of a session is the attach response, and the example now shows what a run actually printed. - The expected sequence put `configurationDone` before `attach`. No order is enforced, but that one misleads about what gates the run: `configurationDone` is what releases the debuggee, which is why breakpoints sent before it are in place when the program starts. - `attach` was implied to be what makes tracing happen. It carries `pathMappings`; `debug_this_thread()` installs the same trace function, so a client that never attaches still stops at breakpoints, it just gets no path translation. Confirmed by removing the attach request from a session: it still stopped. The guide also led with the standalone `dap_monitor.py`, which needs a second port and a re-pointed client. `mpremote debug --dap-log` records every frame for every transport with neither, so it leads now; the server's own logging follows as the method that needs nothing but the firmware, and the monitor stays as the way to drive a session without mpremote. The host-specific paths are gone. Signed-off-by: Andrew Leech --- python-ecosys/debugpy/development_guide.md | 189 ++++++++++++--------- 1 file changed, 108 insertions(+), 81 deletions(-) diff --git a/python-ecosys/debugpy/development_guide.md b/python-ecosys/debugpy/development_guide.md index 81c546b14..d28ae17ce 100644 --- a/python-ecosys/debugpy/development_guide.md +++ b/python-ecosys/debugpy/development_guide.md @@ -1,12 +1,15 @@ # Debugging MicroPython debugpy with VS Code +For working on this module: how to see the DAP conversation, and what a healthy +one looks like. + ## Starting a session Nothing in the program being debugged talks to the debugger. The server is started around it: `listen()`, then `wait_for_client()`, and only then is the program imported and run, so the breakpoints the client sent are already in -place when its first line executes. `test_vscode.py` is such a program - no -debugpy import, no manual breakpoint - and this runs it: +place when its first line executes. `mpremote debug` does that with a boot +script; by hand, on the unix port, the same sequence is: ```bash /micropython -c "import debugpy; debugpy.listen(); \ @@ -14,82 +17,106 @@ debugpy import, no manual breakpoint - and this runs it: import test_vscode; test_vscode.main()" ``` -`wait_for_client()` blocks until a client has attached and sent -`configurationDone`, so there is no race to connect and no sleep to tune. - -## Method 1: Direct Connection with Enhanced Logging - -1. **Start the session** as above, in a terminal you can read. - - This will now show detailed DAP protocol messages like: - ``` - [DAP] RECV: request initialize (seq=1) - [DAP] args: {...} - [DAP] SEND: response initialize (req_seq=1, success=True) - ``` - -2. **Connect VS Code debugger:** - - Use the launch configuration in `.vscode/launch.json` - - Or manually attach to `127.0.0.1:5678` - -3. **Look for issues in the terminal output** - you'll see all DAP message exchanges - -## Method 2: Using DAP Monitor (Recommended for detailed analysis) - -1. **Start the session** as above. - -2. **In another terminal, start the DAP monitor:** - ```bash - python3 dap_monitor.py - ``` - - The monitor listens on port 5679 and forwards to port 5678 - -3. **Connect VS Code to the monitor:** - - Modify your VS Code launch config to connect to port `5679` instead of `5678` - - Or create a new launch config: - ```json - { - "name": "Debug via Monitor", - "type": "python", - "request": "attach", - "connect": { - "host": "127.0.0.1", - "port": 5679 - } - } - ``` - -4. **Analyze the complete DAP conversation** in the monitor terminal - -## VS Code Debug Logging - -Enable VS Code's built-in DAP logging: - -1. **Open VS Code settings** (Ctrl+,) -2. **Search for:** `debug.console.verbosity` -3. **Set to:** `verbose` -4. **Also set:** `debug.allowBreakpointsEverywhere` to `true` - -## Common Issues to Look For - -1. **Missing required DAP capabilities** - check the `initialize` response -2. **Breakpoint verification failures** - look for `setBreakpoints` exchanges -3. **Thread/stack frame issues** - check `stackTrace` and `scopes` responses -4. **Evaluation problems** - monitor `evaluate` request/response pairs - -## Expected DAP Sequence - -A successful debug session should show this sequence: - -1. `initialize` request → response with capabilities -2. `initialized` event -3. `setBreakpoints` request → response with verified breakpoints -4. `configurationDone` request → response -5. `attach` request → response -6. When execution hits breakpoint: `stopped` event -7. `stackTrace` request → response with frames -8. `scopes` request → response with local/global scopes -9. `continue` request → response to resume - -If any step fails or is missing, that's where the issue lies. \ No newline at end of file +`test_vscode.py` is a sample program with no debugpy import and no manual +breakpoint in it. `wait_for_client()` blocks until a client has attached and +sent `configurationDone`, so there is no race to connect and no sleep to tune. + +## Method 1: `--dap-log` (recommended) + +If you start sessions with `mpremote debug`, pass `--dap-log`. It interposes a +proxy between the client and the device, records every complete frame in both +directions as JSONL, and reports the proxy's endpoint in place of the device's +so the client cannot bypass it: + +```bash +mpremote debug --dap-log --dap-log-file dap.jsonl +``` + +Without `--dap-log-file` it writes a timestamped file in the current directory. +This works for every transport, including serial, and needs no second port or +alternate launch configuration. + +## Method 2: the server's own logging + +Start a session by hand, as above, and read the server's own trace from its +output. Connect to `127.0.0.1:5678` with `"logToFile": true` in the +configuration - that flag, read +from the `attach` request, is what turns on the per-message trace, so without it +you get only the handful of unconditional `[DAP]` progress lines +(`vscode_launch_example.json` sets it). From then on each message is printed as +it is handled: + +``` +[DAP] SEND: response attach (req_seq=2, success=True) +[DAP] RECV: request setBreakpoints (seq=3) +[DAP] SEND: response setBreakpoints (req_seq=3, success=True) +[DAP] RECV: request configurationDone (seq=4) +``` + +Because the flag arrives with `attach`, nothing up to and including that request +is logged - the attach *response* is the first line, as above. +This is the one method that needs nothing but the firmware, so it is what to +reach for when the question is whether the server received something at all. + +## Method 3: `dap_monitor.py` + +A standalone host-side proxy: it listens on 5679, forwards to 5678, and prints +the conversation. `--dap-log` supersedes it and is the supported route; this +module is kept for driving a session without mpremote. Using it means pointing +the client at 5679 instead of the port the server reports. + +```bash +python3 dap_monitor.py +``` + +## VS Code debug logging + +VS Code can log its own side: + +1. Open settings (Ctrl+,) +2. Search for `debug.console.verbosity` and set it to `verbose` +3. Set `debug.allowBreakpointsEverywhere` to `true` + +## Expected DAP sequence + +``` +1. initialize request -> response with capabilities, then an `initialized` event +2. attach request -> response +3. setBreakpoints request -> response with verified breakpoints +4. configurationDone request -> response + ... the debuggee now runs +5. stopped event when execution reaches a breakpoint +6. stackTrace request -> response with frames +7. scopes / variables requests -> response with the frame's scopes +8. continue request -> response, and the program resumes +``` + +What each step is actually responsible for: + +- **`configurationDone` is what releases the debuggee.** `wait_for_client()` + blocks until it arrives, draining requests while it waits, so breakpoints sent + before it are already applied when the program starts. Nothing is missed while + a client connects. This is the only ordering the server enforces. +- **`attach` carries `pathMappings`** and installs the trace function. It is not + what makes tracing happen when a boot script is in play: `debug_this_thread()` + installs the same function, so a client that skips `attach` still stops at + breakpoints - it just gets no path translation, so a breakpoint set under a + host path will not match what the debuggee reports. +- **`launch` is answered with success and does nothing.** This module is + attach-only. + +Breakpoints can be sent at any point, before or after `configurationDone`, and +`setBreakpoints` does not require `attach` first. + +## Common issues to look for + +1. **No `stopped` event at all** - check the capability probe first. Without + `sys.settrace` in the firmware there is no debugger; see the `caps` dict in + the launcher's handshake line. +2. **Missing DAP capabilities** - check the `initialize` response. +3. **Breakpoint verification failures** - look at the `setBreakpoints` exchange, + and at whether the path the client sent matches what the debuggee reports. +4. **Locals show as `local_00`, `local_01`, ...** - the firmware was built + without `MICROPY_PY_SYS_SETTRACE_LOCALNAMES`; the names are not recoverable + at runtime. +5. **Evaluation problems** - check the `evaluate` request/response pairs. From a1649abf62dd62113e9ed56e54cdf4758ff3f355 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Mon, 10 Aug 2026 11:47:12 +1000 Subject: [PATCH 69/74] debugpy: Let a stream say the host has gone when it cannot reach EOF. `listen_stream` takes an optional `is_connected` callable, and a stream transport that has one treats it going false as EOF, so a session whose client disappears ends instead of waiting for a message that cannot arrive. A USB CDC interface never reaches EOF: an idle one and one whose host has vanished both read as no bytes. A target stopped at a breakpoint therefore sat in `wait_for_continue` until the board was power-cycled, even though that function already handles a closed channel. On stm32 the signal the interface has instead is `USB_VCP.isconnected()`, its DTR line, which the host raises when it opens the port and the kernel drops when the last opener goes away. The callable is the caller's because it is port-specific, and it counts only once the channel has carried a byte: nobody holds the interface between `listen_stream()` and the client's first connect, and a host may open it briefly beforehand just to check that it can, so down on its own says nothing. A stream with a real EOF passes nothing and is unchanged. Signed-off-by: Andrew Leech --- .../debugpy/common/stream_transport.py | 33 ++++++++++++++++++- python-ecosys/debugpy/debugpy/public_api.py | 8 +++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/common/stream_transport.py b/python-ecosys/debugpy/debugpy/common/stream_transport.py index a3034a419..8eda1342f 100644 --- a/python-ecosys/debugpy/debugpy/common/stream_transport.py +++ b/python-ecosys/debugpy/debugpy/common/stream_transport.py @@ -22,13 +22,18 @@ class StreamTransport: both directions, matching a real socket; `poll(ms)` is called fresh on every `recv`/`send` rather than cached, so each change takes effect immediately. + + `is_connected`, when the runtime has something to offer, is a callable + reporting whether the host still holds the other end - see `_peer_gone`. """ - def __init__(self, reader, writer=None): + def __init__(self, reader, writer=None, is_connected=None): self._reader = reader self._writer = writer if writer is not None else reader self._timeout = None # seconds, or None = block forever self._eof = False + self._is_connected = is_connected + self._had_traffic = False self._poller = select.poll() self._poller.register(self._reader, select.POLLIN) self._write_poller = select.poll() @@ -38,9 +43,34 @@ def __init__(self, reader, writer=None): def settimeout(self, seconds): self._timeout = seconds + def _peer_gone(self): + """Has the host held this channel and then let go of it? + + A USB CDC interface never reaches EOF. An idle one and one whose host + has vanished read identically - no bytes - so a session stopped at a + breakpoint would wait for a `continue` that cannot come, and the board + would need a power cycle. `is_connected` is whatever the runtime has + instead: on stm32 it is `pyb.USB_VCP.isconnected()`, the interface's + DTR line, raised by the host when it opens the port and dropped by the + kernel when the last opener goes away. + + It counts only once the channel has carried a byte. The line goes up + and down for reasons that are not a session ending: nobody holds the + interface between `listen_stream()` and the client's first connect, + and a host may open it briefly beforehand just to check that it can. + Down on its own therefore says nothing; down after the two ends were + talking is the peer leaving. + """ + if self._is_connected is None or not self._had_traffic: + return False + return not self._is_connected() + def recv(self, n): if self._eof: return b"" + if self._peer_gone(): + self._eof = True + return b"" timeout_ms = None if self._timeout is None else max(0, int(self._timeout * 1000)) if not self._poller.poll(timeout_ms): raise OSError(11) # EAGAIN: no data within the timeout @@ -72,6 +102,7 @@ def recv(self, n): if self._eof: return b"" raise OSError(11) + self._had_traffic = True return bytes(mv[:got]) def send(self, data): diff --git a/python-ecosys/debugpy/debugpy/public_api.py b/python-ecosys/debugpy/debugpy/public_api.py index b65b4c561..2f11de548 100644 --- a/python-ecosys/debugpy/debugpy/public_api.py +++ b/python-ecosys/debugpy/debugpy/public_api.py @@ -77,7 +77,7 @@ def listen(port=DEFAULT_PORT, host=DEFAULT_HOST): return (host, port) -def listen_stream(reader, writer=None): +def listen_stream(reader, writer=None, is_connected=None): """Start a debug session directly on an already-open stream, no TCP. For a board with a second CDC interface dedicated to DAP: `reader`/`writer` @@ -86,13 +86,17 @@ def listen_stream(reader, writer=None): `listen()`, the stream is already connected - there is no bind/accept step, so `wait_for_client()` goes straight to the initialize/ configurationDone handshake. + + `is_connected` is how a stream that cannot reach EOF says the host has + gone, and belongs to the caller because it is port-specific; see + `StreamTransport._peer_gone`. A stream with a real EOF needs none. """ global _listener if _listener is not None or _debug_session is not None: raise RuntimeError("Already listening for debugger") - _listener = StreamTransport(reader, writer) + _listener = StreamTransport(reader, writer, is_connected) print("Debugpy listening on stream") return _listener From d15c876b071b6987d6a3753330b83790853f46cc Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Mon, 10 Aug 2026 13:54:45 +1000 Subject: [PATCH 70/74] debugpy: Stop the target when the client asks it to pause. `should_stop` consumes a pending pause at the next `line` event and reports the stop as `pause`, so a client that presses pause gets a target that has actually stopped. It did not before. `_handle_pause` set two `paused` flags and answered success, and no stop decision read either, so the client's UI went to the stopped state while the program ran on - and then asked for `stackTrace`, `scopes` and `variables`, all answered from whatever frame the last trace event had left in `current_frame`. The user was shown a frame the program had already left. Only `line` events: `call` reports the `def` line before the body has run and `return` reports a frame that has already produced its value, the same distinction the breakpoint check makes. A pending step is dropped with the pause, since a step that outlived a user interrupt would fire later at a point nobody asked for. `wait_for_continue` clears the flag on the way out, so a pause that arrives while the target is already stopped does not stop it again one line into its next run. The stop reason now comes from the adapter, which is what made the decision - a stop is a breakpoint, a consumed pause, or a landed step, and nothing else. That makes `DebugSession.paused` and `DebugSession.stepping` removable rather than write-only: between them they were assigned twelve times and read once, which is what let a request that did nothing read as implemented. A pause still only lands where Python is running. A target blocked in `time.sleep`, waiting on a socket, or inside a long-running C function produces no trace event, so the request stays pending until one comes. The README says so in place of the entry that said pause does nothing. Signed-off-by: Andrew Leech --- python-ecosys/debugpy/README.md | 7 +++- .../debugpy/debugpy/server/debug_session.py | 30 ++++++-------- .../debugpy/debugpy/server/pdb_adapter.py | 39 +++++++++++++++++-- 3 files changed, 52 insertions(+), 24 deletions(-) diff --git a/python-ecosys/debugpy/README.md b/python-ecosys/debugpy/README.md index f765d1dc2..6cbbd45bd 100644 --- a/python-ecosys/debugpy/README.md +++ b/python-ecosys/debugpy/README.md @@ -15,9 +15,12 @@ such as VS Code debugging support. read-only; no MicroPython build implements local-variable write-back. - Expression evaluation - Continue + - Pause -`pause` is accepted and answered, but does not stop a running target: nothing -in the trace hook consults the flag it sets. Use a breakpoint instead. +A pause takes effect at the next line the target executes, because the trace +hook is the only thing that can interrupt it. A target that is executing no +Python - blocked in `time.sleep`, or inside a long-running C function - stays +running until it does; the request is not lost, it is pending. ## Requirements diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index 507418d5b..3a8f95dae 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -117,8 +117,6 @@ def __init__(self, client_socket, is_stream, restart_supported=False): self.initialized = False self.connected = True self.thread_id = 1 # Simple single-thread model - self.stepping = False - self.paused = False self.configuration_done = False self._pumping = False # Whether the launcher can actually re-run the target. Only it knows, @@ -497,35 +495,30 @@ def _handle_set_breakpoints(self, seq, args): def _handle_continue(self, seq, args): """Handle continue request.""" - self.stepping = False - self.paused = False self.pdb.continue_execution() self.channel.send_response(CMD_CONTINUE, seq) def _handle_next(self, seq, args): """Handle next (step over) request.""" - self.stepping = True - self.paused = False self.pdb.step_over() self.channel.send_response(CMD_NEXT, seq) def _handle_step_in(self, seq, args): """Handle stepIn request.""" - self.stepping = True - self.paused = False self.pdb.step_into() self.channel.send_response(CMD_STEP_IN, seq) def _handle_step_out(self, seq, args): """Handle stepOut request.""" - self.stepping = True - self.paused = False self.pdb.step_out() self.channel.send_response(CMD_STEP_OUT, seq) def _handle_pause(self, seq, args): - """Handle pause request.""" - self.paused = True + """Handle pause request. + + The response says the request was accepted, not that the target has + stopped; `stopped` follows when the trace function consumes it. + """ self.pdb.pause() self.channel.send_response(CMD_PAUSE, seq) @@ -621,9 +614,8 @@ def _handle_restart(self, seq, args): self.restart_requested = True # A target stopped at a breakpoint is inside wait_for_continue(); it has - # to be let go before it can be unwound. Stepping is cleared with it, so - # a pending step does not stop the target again on its way out. - self.stepping = False + # to be let go before it can be unwound. A pending step or pause is + # cleared with it, so neither stops the target again on its way out. self.pdb.step_mode = None self.pdb.paused = False self.pdb.continue_event = True @@ -735,14 +727,16 @@ def _trace_function(self, frame, event: str, arg): # Before any breakpoint work: a restart that arrived above wants this # program gone, not stopped somewhere else on the way out. self._raise_if_restarting() - # Handle breakpoints and stepping + # Handle breakpoints, pauses and stepping. The reason comes from the + # adapter because the adapter is what decided: a stop is a breakpoint, + # a consumed pause, or a landed step, and those are the only three. if self.pdb.should_stop(frame, event, arg): self._send_stopped_event( STOP_REASON_BREAKPOINT if self.pdb.hit_breakpoint - else STOP_REASON_STEP - if self.stepping else STOP_REASON_PAUSE + if self.pdb.hit_pause + else STOP_REASON_STEP ) # Wait for continue command self.pdb.wait_for_continue() diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py index bd6be08ad..da8118345 100644 --- a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -161,8 +161,9 @@ def __init__(self): self.step_mode = None # None, 'over', 'into', 'out' self.step_frame = None self.step_depth = 0 - self.paused = False + self.paused = False # a pause request waiting for the next line event self.hit_breakpoint = False + self.hit_pause = False self.continue_event = False self.variables_cache = {} # frameId -> variables self.var_cache = VariableReferenceCache() # Enhanced variable reference cache @@ -279,10 +280,17 @@ def set_breakpoints(self, filename: str, breakpoints: list[dict]): return actual_breakpoints def should_stop(self, frame, event: str, arg): - """Determine if execution should stop at this point.""" + """Determine if execution should stop at this point. + + Returns True for exactly three reasons - a breakpoint, a pending + pause, or a landed step - and records which one in `hit_breakpoint` / + `hit_pause` so the caller can name it without keeping its own copy of + the decision. + """ # HOT path - no debug printing here self.current_frame = frame self.hit_breakpoint = False + self.hit_pause = False # Cache frame attributes to reduce lookup overhead _frame_code = frame.f_code @@ -311,6 +319,18 @@ def should_stop(self, frame, event: str, arg): if _filename not in self.file_mappings: self.file_mappings[_filename] = self._filename_as_debugger(_filename) + # A pause request that arrived while the target was running is consumed + # here. Only on `line`: `call` reports the `def` line before the body + # has run and `return` reports a frame that has already produced its + # value, so neither is a place the user asked to stop at. + if self.paused and event == TRACE_LINE: + self.paused = False + # A pause is a user interrupt; a step that outlived it would fire + # later at a point nobody asked for. + self.step_mode = None + self.hit_pause = True + return True + # Check stepping _step_mode = self.step_mode if _step_mode == STEP_INTO: @@ -359,8 +379,14 @@ def step_out(self): self.continue_event = True def pause(self): - """Pause execution at next opportunity.""" - # This is handled by the debug session + """Request a stop at the next line event. + + Nothing stops here: the target is running, and the only code that can + interrupt it is the trace function. `should_stop` consumes this. A + target executing no traced bytecode - blocked in `time.sleep`, inside + a C-level loop, or between runs of a `--loop` session - produces no + trace event, so the request simply stays pending until one comes. + """ self.paused = True def wait_for_continue(self): @@ -389,6 +415,11 @@ def wait_for_continue(self): session.process_pending_messages() # type: ignore[arg-type] time.sleep(0.01) + # A pause that arrived during the wait is about a target that was + # already stopped. Dropping it here is what keeps it from stopping the + # target again one line after the user's next continue. + self.paused = False + def get_stack_trace(self): """Get the current stack trace.""" if not self.current_frame: From 33af8afbdc181d9d93bf2e8e9c816191753c0e9a Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Mon, 10 Aug 2026 22:06:10 +1000 Subject: [PATCH 71/74] debugpy: Share the REPL's stream with the DAP channel. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A board with one UART and no network has one way in, and it is already carrying the REPL. `ReplMux` splits that stream into two façades: the program's console, and the byte stream `debugpy.listen_stream()` wants. Whichever object the runtime diverts stdout into gets the console side; the DAP side is handed to `listen_stream` unchanged, so nothing above this file knows the stream is shared. The wire reuses `mpremote mount`'s marker and code namespace so one demux point can eventually serve both: `0x18` keeps its meaning, codes 1..13 stay the filesystem RPC's, and a DAP frame is code 14 with an explicit two-byte length. Carrying the length rather than reading the DAP message's own `Content-Length` keeps the reader from ever inspecting a payload, so the only content assumption on the wire is about `0x18`, and doubling that byte in console output removes it. Inbound bytes are credited back to the sender. The receive path this rides on is a fixed ring that discards the tail of a packet it has no room for instead of exerting back-pressure, and a target inside `time.sleep()` drains nothing, so a sender that ignored the credit would lose the middle of a `setBreakpoints`. Blocking instead is recoverable. `_emit` never raises. The console façade sits in the runtime's stdout diversion, and on the ports this reaches an exception out of a diverted `write()` deactivates the diversion - which on a single-stream board removes the console and the debug channel in the same instant. A short write leaves the rest queued. Both façades are `io.IOBase` and answer `MP_STREAM_GET_FILENO` with `-EINVAL`. Answering with a number would let a port built with `MICROPY_PY_SELECT_POSIX_OPTIMISATIONS` poll that descriptor directly and never call the façade's `ioctl` again, which is the only place the demux is pumped. Signed-off-by: Andrew Leech --- .../debugpy/debugpy/common/repl_mux.py | 462 ++++++++++++++++++ 1 file changed, 462 insertions(+) create mode 100644 python-ecosys/debugpy/debugpy/common/repl_mux.py diff --git a/python-ecosys/debugpy/debugpy/common/repl_mux.py b/python-ecosys/debugpy/debugpy/common/repl_mux.py new file mode 100644 index 000000000..f39931669 --- /dev/null +++ b/python-ecosys/debugpy/debugpy/common/repl_mux.py @@ -0,0 +1,462 @@ +"""Framing that lets the DAP channel share the REPL's one byte stream. + +For a board with a single UART and no network, the stream carrying the REPL is +the only way in. `mpremote mount` already puts a second protocol on that stream +by marking its exchanges with `0x18`, and this reuses the marker and the code +namespace so one demux point can eventually serve both: codes 1..13 belong to +mount's filesystem RPC and are never emitted here. + +The wire is the same in both directions: + + 0x18 0x18 one literal 0x18 in the plain byte stream + 0x18 a framed message of `lo | hi << 8` bytes + anything else plain bytes - program stdout one way, + program stdin the other + +`Demux` is the reader for that wire and is used unmodified at both ends, so +"the program's output reaches the terminal" and "a program calling `input()` +gets what the user typed" are the same code path, tested once. The length is +explicit rather than reusing the DAP message's own `Content-Length` header so +that a reader never inspects a payload to find where it ends: the only content +assumption on the wire is about `0x18`, and escaping removes that one. + +Inbound bytes are credited (`CMD_DAP_ACK`), because the receive path this rides +on is a fixed ring that discards the tail of a packet when it overflows rather +than exerting back-pressure - the device drains it only when something polls, +and a target sitting in `time.sleep()` polls nothing. A sender that respects +the credit cannot overrun a receiver that has stopped listening; it blocks +instead, which is recoverable. +""" + +import errno +import io +import select + +# `mp_stream_p_t` request numbers, as `io.IOBase` hands them to `ioctl`. +_IOCTL_POLL = 3 +_IOCTL_GET_FILENO = 10 + +# What both façades answer to `MP_STREAM_GET_FILENO`: a negative return from a +# Python `ioctl` is `MP_STREAM_ERROR` with this as its errno. Neither of them +# is a file, and saying so matters rather than being pedantry - a port built +# with `MICROPY_PY_SELECT_POSIX_OPTIMISATIONS` (the unix one) asks every object +# registered with `select.poll` for a descriptor and, when it gets a number, +# polls *that* and never calls the object's own `ioctl` again. A façade that +# answered 0 would have the poller watching stdin while the demux that actually +# holds the bytes was never pumped. +_NOT_A_FILE = -errno.EINVAL + +MARKER = 0x18 + +# Shared with `mpremote`'s `fs_hook_cmds`, which owns 1..13. +CMD_DAP = 14 +CMD_DAP_ACK = 15 +# The DAP channel is over. Sent because the streams this rides on have no EOF +# of their own - an idle USB CDC interface and one nobody is holding both read +# as no bytes - so without it a host would wait out a finished session. Only +# the device sends it: a host that goes away drops DTR, which the device reads +# through `isconnected()`, and a client that merely detaches is the DAP +# protocol's own business. +CMD_DAP_EOF = 16 + +# Wire bytes a sender may have outstanding before the peer credits them. The +# smallest receive ring in the ports this runs on is 256 bytes +# (`MICROPY_HW_USB_CDC_RX_DATA_SIZE`), so this leaves room for a partly-drained +# one. Both ends must agree: `mpremote`'s copy is checked against this. +RX_CREDIT = 192 + +# Bytes of payload per frame. Bounds the reader's per-frame buffer, and is +# capped so that one whole frame always fits inside the credit window: a frame +# bigger than `RX_CREDIT` could never be sent at all, since the sender would +# wait for credit that only its own unsent bytes could earn. A DAP message +# longer than this is split across frames. +MAX_PAYLOAD = 128 + +# Consumed bytes a receiver may leave uncredited before it sends an ack. Small +# enough that a sender at the credit limit is released within one exchange, +# large enough that a steady stream does not ack every frame. A receiver that +# has drained everything acks regardless, so a sender is never left waiting on +# a threshold that will not be reached. +ACK_THRESHOLD = 64 + +# Outbound bytes queued before the DAP side reports itself unwritable. Reached +# only when the peer has stopped reading; the DAP layer then retries, which is +# back-pressure, where growing without limit would be a slow memory error on a +# device that has a few tens of kilobytes to spend. +MAX_OUTBOX = 4096 + +# Milliseconds a stalled console write waits for room before giving up and +# leaving the rest queued. +_STALL_MS = 100 + + +def frame(code, payload): + """One framed message: marker, code, two-byte length, payload.""" + n = len(payload) + return bytes((MARKER, code, n & 0xFF, (n >> 8) & 0xFF)) + bytes(payload) + + +def _consume(buf, n): + """Drop the first `n` bytes of `buf`, in place. + + `del buf[:n]` is the obvious spelling and raises on a MicroPython + bytearray, which implements slice assignment but not slice deletion. + Emptying it outright is the common case - a reader almost always takes + everything queued - and is the one path here that allocates nothing. + """ + if n >= len(buf): + buf[:] = b"" + else: + buf[:] = buf[n:] + + +def escape(data): + """Plain bytes with the marker doubled, so a reader never mistakes one. + + A program that prints `0x18` is the case this exists for; it costs one + extra byte per occurrence and nothing at all for output that has none. + """ + data = bytes(data) + if data.find(bytes((MARKER,))) < 0: + return data + return data.replace(bytes((MARKER,)), bytes((MARKER, MARKER))) + + +class Demux: + """Reads the wire above into plain bytes and framed payload. + + `plain` and `dap` are drained by the caller; `credited` accumulates what + the peer has reported consuming. `unknown_code` records the first code + this reader has no handler for, which on a single-UART link means the two + ends disagree about the protocol - worth reporting rather than skipping, + since every later byte is then suspect. + + Feeding is incremental: a marker, a length or a payload split across two + reads resumes where it left off, which is the normal case on a stream that + delivers whatever a USB packet happened to contain. + """ + + _PLAIN, _CODE, _LEN0, _LEN1, _BODY = 0, 1, 2, 3, 4 + + def __init__(self): + self.plain = bytearray() + self.dap = bytearray() + self.credited = 0 + self.eof = False + self.unknown_code = None + self._state = self._PLAIN + self._code = 0 + self._need = 0 + self._body = bytearray() + + def feed(self, data): + data = bytes(data) + i = 0 + end = len(data) + while i < end: + if self._state == self._PLAIN: + # Copy up to the next marker in one go: plain bytes are the + # bulk of the traffic in the stdout direction and scanning + # them one at a time is the whole cost of the layer. + j = data.find(bytes((MARKER,)), i) + if j < 0: + self.plain += data[i:] + return + self.plain += data[i:j] + i = j + 1 + self._state = self._CODE + continue + + c = data[i] + i += 1 + if self._state == self._CODE: + if c == MARKER: + self.plain.append(MARKER) # escaped literal + self._state = self._PLAIN + else: + self._code = c + self._state = self._LEN0 + elif self._state == self._LEN0: + self._need = c + self._state = self._LEN1 + elif self._state == self._LEN1: + self._need |= c << 8 + self._body = bytearray() + self._state = self._BODY if self._need else self._PLAIN + if not self._need: + self._deliver() + else: # _BODY + self._body.append(c) + if len(self._body) == self._need: + self._deliver() + self._state = self._PLAIN + + def _deliver(self): + if self._code == CMD_DAP: + self.dap += self._body + elif self._code == CMD_DAP_ACK: + if len(self._body) >= 2: + self.credited += self._body[0] | (self._body[1] << 8) + elif self._code == CMD_DAP_EOF: + self.eof = True + elif self.unknown_code is None: + self.unknown_code = self._code + + def take_plain(self, n): + out = bytes(self.plain[:n]) + _consume(self.plain, len(out)) + return out + + def take_dap(self, n): + out = bytes(self.dap[:n]) + _consume(self.dap, len(out)) + return out + + +class ReplMux: + """Owns the REPL stream and hands out the two façades that share it. + + `console` goes wherever the runtime's stdout is diverted - a `dupterm` + slot on the ports where that reaches every byte - and `dap` is handed to + `debugpy.listen_stream()`. They are separate objects because the bytes + arriving at a `write()` are the only thing that distinguishes program + output from a DAP frame, so one object could not tell them apart. + + Everything either façade emits is appended to one outbound buffer in whole + frames and flushed from there, so a partial write to the underlying port + can never interleave two messages: the port sees frames in the order they + were produced, or it sees a prefix of that order. + """ + + def __init__(self, port=None): + self._port = None + self._poller = None + self._write_poller = None + self._out = bytearray() + self._demux = Demux() + self._consumed = 0 # payload bytes taken from `dap` since the last ack + self._byte = bytearray(1) + self.console = _PlainSide(self) + self.dap = _DapSide(self) + if port is not None: + self.attach(port) + + def attach(self, port): + """Take `port` as the real stream. + + Separate from `__init__` because on a port where the diversion is a + `dupterm` slot, the object being displaced *is* the real stream, and + it is only known once the swap has happened. + """ + self._port = port + self._poller = select.poll() + self._poller.register(port, select.POLLIN) + self._write_poller = select.poll() + self._write_poller.register(port, select.POLLOUT) + # The DAP façade inherits whatever the real stream has instead of EOF. + # A USB CDC interface has none - an idle one and one nobody is holding + # both read as no bytes - but it does have `isconnected()`, the DTR + # line the host raises on open and the kernel drops when the last + # opener goes, which is how a session notices its host disappeared. + self.dap.isconnected = getattr(port, "isconnected", None) + + # -- outbound --------------------------------------------------------- + + def _emit(self, data): + """Queue whole frames and push what the port will take. Never raises. + + An exception out of the console façade's `write()` would deactivate the + stdout diversion it sits in, which on a single-stream board silently + removes the console and the debug channel at once, so nothing here is + allowed to escape. A short write is not a failure: what did not go out + stays queued and leaves with the next call. + """ + self._out += data + self._flush() + + def _flush(self, timeout_ms=0): + while self._out: + if self._port is None: + return False + if not self._write_poller.poll(timeout_ms): + return False + try: + n = self._port.write(self._out) + except OSError: + return False # EAGAIN, or the port has gone; either way, later + if not n: + return False + _consume(self._out, n) + return True + + # -- inbound ---------------------------------------------------------- + + def pump(self, timeout_ms=0): + """Move whatever the port has into the demux, and credit it. + + Read one byte at a time behind a fresh zero-timeout poll, for the + reason `StreamTransport.recv` gives: a bulk `readinto` on a stream + whose read genuinely blocks (a plain file, as against a CDC driver + that always returns immediately) would wait for bytes that may never + come, because the request is looped internally until the buffer fills. + + Credit is counted here rather than where a payload is handed on, + because what the credit protects is the receive ring, and a byte read + out of it has already freed its place there whatever happens to it + next. Counting it later would credit a frame's overhead once per + partial read of it, which over-reports and defeats the limit. + """ + if self._port is None: + return False + if not self._poller.poll(timeout_ms): + self._send_ack(force=True) + return False + got = False + while True: + try: + r = self._port.readinto(self._byte) + except OSError: + break + if not r: + break + self._demux.feed(self._byte) + self._consumed += r + got = True + if self._consumed >= ACK_THRESHOLD: + self._send_ack() # mid-burst, so a long one does not stall the peer + if not self._poller.poll(0): + break + self._send_ack(force=True) + return got + + def _send_ack(self, force=False): + """Tell the peer how much of its traffic has left this end's buffer. + + Credit is what stops a sender overrunning a receive ring that drops + the tail of a packet on overflow. It is reported in wire bytes, the + unit the ring holds, so the sender does not have to model framing. + """ + if self._consumed and (force or self._consumed >= ACK_THRESHOLD): + n = self._consumed + self._consumed = 0 + self._emit(frame(CMD_DAP_ACK, bytes((n & 0xFF, (n >> 8) & 0xFF)))) + + @property + def unknown_code(self): + return self._demux.unknown_code + + def detach(self): + """Push out what is queued and give the real stream back. + + Returns the port so the caller can put it where it found it. The flush + is bounded rather than unconditional: a peer that has stopped reading + must not be able to keep the stream split forever, and by this point + whatever is still queued is the tail of a session that is over. + """ + if self._port is not None: + # Last framed message on the channel, and the peer's only notice + # that the session ended rather than went quiet. + self._emit(frame(CMD_DAP_EOF, b"")) + self._flush(_STALL_MS) + port = self._port + self._port = None + self._poller = None + self._write_poller = None + self.dap.isconnected = None + return port + + def close(self): + self.detach() + + +class _PlainSide(io.IOBase): + """The program's own stdout and stdin, escaped onto the shared stream. + + `io.IOBase` rather than a plain class because both places this object goes + demand the native stream protocol: `os.dupterm` raises `OSError: stream + operation not supported` without it, and `select.poll().register()` needs + the `ioctl`. + """ + + def __init__(self, mux): + self._mux = mux + + def write(self, buf): + n = len(buf) + mux = self._mux + mux._emit(escape(buf)) + if len(mux._out) > MAX_OUTBOX: + # Program output is never dropped, so a peer that has stopped + # reading slows the program down instead. Bounded: one flush + # attempt, not a wait for the whole queue to leave. + mux._flush(_STALL_MS) + return n # every byte was accepted; delivery is the outbound queue's job + + def readinto(self, buf, nbytes=None): + mux = self._mux + mux.pump() + want = len(buf) if nbytes is None else min(nbytes, len(buf)) + data = mux._demux.take_plain(want) + buf[: len(data)] = data + return len(data) + + def ioctl(self, op, arg): + if op == _IOCTL_POLL: + mux = self._mux + mux.pump() + ret = 0 + if arg & 1 and mux._demux.plain: # MP_STREAM_POLL_RD + ret |= 1 + if arg & 4: # MP_STREAM_POLL_WR - the outbound queue always accepts + ret |= 4 + return ret + if op == _IOCTL_GET_FILENO: + return _NOT_A_FILE + return 0 + + +class _DapSide(io.IOBase): + """The DAP channel, framed onto the shared stream. + + Handed to `debugpy.listen_stream()`, which registers it with `select.poll` + for both directions and calls `poll()` fresh on every `recv`/`send`. That + makes `ioctl` the place the inbound side is pumped: it runs on every wait + the DAP layer does, which under `settrace` is at every trace event. + """ + + def __init__(self, mux): + self._mux = mux + # Set by `ReplMux.attach` to the real stream's own liveness test, or + # left None where it has none. An instance attribute rather than a + # method so that `getattr(stream, "isconnected", None)` - what the + # boot script asks - can answer "this stream cannot tell you". + self.isconnected = None + + def write(self, buf): + mux = self._mux + data = bytes(buf) + for i in range(0, len(data), MAX_PAYLOAD): + mux._emit(frame(CMD_DAP, data[i : i + MAX_PAYLOAD])) + return len(data) + + def readinto(self, buf, nbytes=None): + mux = self._mux + mux.pump() + want = len(buf) if nbytes is None else min(nbytes, len(buf)) + data = mux._demux.take_dap(want) + buf[: len(data)] = data + return len(data) + + def ioctl(self, op, arg): + if op == _IOCTL_POLL: + mux = self._mux + mux.pump() + ret = 0 + if arg & 1 and mux._demux.dap: # MP_STREAM_POLL_RD + ret |= 1 + if arg & 4 and len(mux._out) <= MAX_OUTBOX: # MP_STREAM_POLL_WR + ret |= 4 + return ret + if op == _IOCTL_GET_FILENO: + return _NOT_A_FILE + return 0 From d669e3aed7b0caf4e45b16f1d8f63c06930e80ab Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Tue, 11 Aug 2026 21:52:01 +1000 Subject: [PATCH 72/74] debugpy: End a stream session when the host that held it lets go. `StreamTransport`'s host-has-gone signal could not end a wait that had not already carried traffic, and a `recv` given no timeout never re-read it. On a dedicated DAP interface that leaves a session nobody was using; on a stream shared with the REPL it takes away the console the board is reached by, because the framing wrapper stays in the runtime's `dupterm` slot until the session ends. Measured on a PYBD_SF6: a session killed before any client sent `initialize` left the board answering every later REPL in framed bytes until it was power-cycled. The signal now also arms when the channel was already held at the moment the transport was built. That is what separates the two cases. Nothing holds a dedicated interface between `listen_stream()` and the client's first connect, so its line being down says nothing until a byte has crossed; a shared stream's hold predates the channel, so only the host leaving can drop it, and waiting for traffic there means a client that never attaches holds the stream forever. A wait asked to block forever is served in slices, because the line is not something a poll can wait on and one unbounded poll never looks at it again. `_accept_and_initialize` retries a partial `initialize` rather than reading it as a client that sent some other command, and tears the session down when the channel closes rather than building one on a message that never arrived. Signed-off-by: Andrew Leech --- .../debugpy/common/stream_transport.py | 56 ++++++++++++++++--- python-ecosys/debugpy/debugpy/public_api.py | 16 +++++- 2 files changed, 61 insertions(+), 11 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/common/stream_transport.py b/python-ecosys/debugpy/debugpy/common/stream_transport.py index 8eda1342f..5d7ce5bc8 100644 --- a/python-ecosys/debugpy/debugpy/common/stream_transport.py +++ b/python-ecosys/debugpy/debugpy/common/stream_transport.py @@ -2,6 +2,13 @@ import select +# How long a wait that was asked to block forever may sit inside one poll +# before the liveness check runs again. The line `is_connected` reads is not +# something a poll can wait on, so a blocking wait would otherwise never +# notice the host letting go. A bound on how quickly that is noticed, not a +# timeout: a wait carrying a real one is still governed by it. +_LIVENESS_SLICE_MS = 100 + class StreamTransport: """Presents a reader/writer stream pair as a socket to `JsonMessageChannel`. @@ -34,6 +41,7 @@ def __init__(self, reader, writer=None, is_connected=None): self._eof = False self._is_connected = is_connected self._had_traffic = False + self._was_held = is_connected is not None and bool(is_connected()) self._poller = select.poll() self._poller.register(self._reader, select.POLLIN) self._write_poller = select.poll() @@ -54,17 +62,44 @@ def _peer_gone(self): DTR line, raised by the host when it opens the port and dropped by the kernel when the last opener goes away. - It counts only once the channel has carried a byte. The line goes up - and down for reasons that are not a session ending: nobody holds the - interface between `listen_stream()` and the client's first connect, - and a host may open it briefly beforehand just to check that it can. - Down on its own therefore says nothing; down after the two ends were - talking is the peer leaving. + It counts only once the channel has been held, because the line goes + up and down for reasons that are not a session ending: a host may open + an idle interface briefly just to check that it can. Down on its own + therefore says nothing; down after the channel was held is the peer + leaving. Held means either of two things, and which one applies is + decided by whether anyone was on the far end when the channel was + made: + + - Nobody was, so the channel has to earn the signal - a byte has to + cross it. This is the dedicated-DAP-interface case: nothing holds + that interface between `listen_stream()` and the client's first + connect, so its line being down at any point before then is the + ordinary state and not a peer. + - Somebody was, recorded at construction as `_was_held`. This is the + case where DAP shares the stream the host is already using to drive + the board: that hold predates the channel, so nothing but the host + leaving can drop it, and waiting for traffic would mean a client + that never sends anything holds the stream forever. """ - if self._is_connected is None or not self._had_traffic: + if self._is_connected is None or not (self._had_traffic or self._was_held): return False return not self._is_connected() + def _wait(self, poller, timeout_ms): + """Poll for readiness, giving up early once the peer has gone. + + Returns whatever `poll` returned, which is falsy both when the wait + timed out and when the peer left; callers separate the two with + `_peer_gone()`. A wait asked to block forever is served in slices so + that check gets to run - see `_LIVENESS_SLICE_MS`. + """ + if timeout_ms is not None: + return poller.poll(timeout_ms) + while True: + ready = poller.poll(_LIVENESS_SLICE_MS) + if ready or self._peer_gone(): + return ready + def recv(self, n): if self._eof: return b"" @@ -72,7 +107,10 @@ def recv(self, n): self._eof = True return b"" timeout_ms = None if self._timeout is None else max(0, int(self._timeout * 1000)) - if not self._poller.poll(timeout_ms): + if not self._wait(self._poller, timeout_ms): + if self._peer_gone(): + self._eof = True + return b"" raise OSError(11) # EAGAIN: no data within the timeout # `.read()`/`.readinto()` loop internally until the buffer is full @@ -115,7 +153,7 @@ def send(self, data): # the retry would then re-send that prefix and desynchronise the # Content-Length framing it was meant to protect. timeout_ms = None if self._timeout is None else max(0, int(self._timeout * 1000)) - if not self._write_poller.poll(timeout_ms): + if not self._wait(self._write_poller, timeout_ms): raise OSError(11) # EAGAIN: no room within the timeout # `write()` answers a non-blocking stream that took nothing with None # rather than 0, so both are treated as "came back not ready after diff --git a/python-ecosys/debugpy/debugpy/public_api.py b/python-ecosys/debugpy/debugpy/public_api.py index 2f11de548..079290845 100644 --- a/python-ecosys/debugpy/debugpy/public_api.py +++ b/python-ecosys/debugpy/debugpy/public_api.py @@ -129,8 +129,20 @@ def _accept_and_initialize(): _debug_session = DebugSession(client_sock, is_stream, _restart_supported) print("[DAP] Waiting for initialize request...") - init_message = _debug_session.channel.recv_message() - if init_message and init_message.get("command") == "initialize": + # `recv_message()` answers None both for "not a whole message yet" and + # for "the channel is gone", and against a blocking channel the first + # of those is what a message split across reads looks like. Retrying + # while the channel is open tells them apart, and stops a split + # `initialize` from being read as a client that sent something else. + init_message = None + while init_message is None and not _debug_session.channel.closed: + init_message = _debug_session.channel.recv_message() + if init_message is None: + print("[DAP] Connection closed before the initialize request") + _debug_session = None + client_sock.close() + return False + if init_message.get("command") == "initialize": _debug_session._handle_message(init_message) print("[DAP] Initialize request handled - returning control immediately") else: From abb03f6501ad25415b6850fe972a5549f4815a21 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Thu, 20 Aug 2026 17:11:44 +1000 Subject: [PATCH 73/74] debugpy: Drop the serial_dap capability. The key reported whether a session's DAP channel was a stream rather than a TCP socket, which existed to tell a dedicated serial interface apart from the network. That transport is gone; the one remaining stream channel reports itself as `repl_dap`, from the boot script that knows it split the REPL. `probe_capabilities()` therefore takes no argument and answers the same whenever it is called: every key in it is now a property of the firmware rather than of the run. Signed-off-by: Andrew Leech --- python-ecosys/debugpy/debugpy/public_api.py | 19 ++++-------- .../debugpy/debugpy/server/debug_session.py | 30 +++++++------------ 2 files changed, 17 insertions(+), 32 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/public_api.py b/python-ecosys/debugpy/debugpy/public_api.py index 079290845..65d119876 100644 --- a/python-ecosys/debugpy/debugpy/public_api.py +++ b/python-ecosys/debugpy/debugpy/public_api.py @@ -126,7 +126,7 @@ def _accept_and_initialize(): client_sock, client_addr = listener.accept() print(f"Debugger connected from {format_client_addr(client_addr)}") - _debug_session = DebugSession(client_sock, is_stream, _restart_supported) + _debug_session = DebugSession(client_sock, _restart_supported) print("[DAP] Waiting for initialize request...") # `recv_message()` answers None both for "not a whole message yet" and @@ -249,22 +249,15 @@ def get_capabilities(): """Return the firmware capability dict (settrace/save_names/set_local/f_back). Uses the active session's probe result if a session exists, otherwise - probes directly. Values always come from probing the running - interpreter, never from a build/variant name - except `serial_dap`, - which comes from whichever of `_debug_session`/`_listener` exists: the - boot script calls this between `listen()`/`listen_stream()` and - `wait_for_client()` (before a session exists), so `_listener` is the - only place the channel choice is recorded yet. - - Call this after `listen()`/`listen_stream()`; called before either, or - after `disconnect()`/a `wait_for_client()` timeout has cleared both - globals, `serial_dap` reports `False` regardless of which channel a - prior session used. + probes directly. Every value comes from probing the running interpreter, + never from a build or variant name. Which channel a session took is not + among them - that is a property of the run, and the boot script reports + it - so this answers the same whenever it is called. """ global _debug_session if _debug_session is not None: return _debug_session.capabilities - return DebugSession.probe_capabilities(isinstance(_listener, StreamTransport)) + return DebugSession.probe_capabilities() def breakpoint(): diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index 3a8f95dae..da010a22b 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -108,7 +108,7 @@ def _probe_local_names(frame): class DebugSession: """Manages a debugging session with a DAP client.""" - def __init__(self, client_socket, is_stream, restart_supported=False): + def __init__(self, client_socket, restart_supported=False): self.debug_logging = False # Initialize first self.channel = JsonMessageChannel(client_socket, self._debug_print) self.pdb = PdbAdapter() @@ -124,11 +124,8 @@ def __init__(self, client_socket, is_stream, restart_supported=False): # supportsRestartRequest not advertised, unless it was told otherwise. self.restart_supported = restart_supported self.restart_requested = False - # is_stream comes from the caller (_accept_and_initialize already - # knows which kind of channel client_socket is) rather than being - # re-derived here, so there is exactly one place that decides it. # Probed once at session start; never inferred from a build/variant name. - self.capabilities = self.probe_capabilities(is_stream) + self.capabilities = self.probe_capabilities() self.pdb.capabilities = self.capabilities def _debug_print(self, message): @@ -141,21 +138,17 @@ def _baremetal(self) -> bool: return sys.platform not in ("linux") # to be expanded @staticmethod - def probe_capabilities(is_stream): + def probe_capabilities(): """Probe what the running firmware actually supports. - Returns a dict with at least `settrace`, `save_names`, `set_local`, - `f_back` and `serial_dap`, each derived by exercising the real - interpreter - never by reading a build/variant name, which does not - reliably reflect what a given firmware image supports (see - BACKGROUND.md). `serial_dap` is the one exception to "probe, don't - ask": whether the DAP channel is a stream rather than a TCP socket - is a fact about *this session*, decided by whichever boot script - called `listen_stream()` vs `listen()`. `is_stream` is that fact - - required, not defaulted, so a caller cannot silently report "not a - stream" by forgetting the argument - passed in by the caller rather - than guessed here, so the two can never disagree. Safe to call on - both the unix port and bare-metal builds; never raises. + Returns a dict with at least `settrace`, `save_names`, `set_local` + and `f_back`, each derived by exercising the real interpreter - + never by reading a build/variant name, which does not reliably + reflect what a given firmware image supports (see BACKGROUND.md). + Which channel a session took is not in here: that is a fact about + the run rather than about the firmware, and the boot script reports + it (`caps["repl_dap"]`). Safe to call on both the unix port and + bare-metal builds; never raises. `save_names` is measured on code the firmware compiles here rather than on this module's own frame, so it reports the firmware and not @@ -166,7 +159,6 @@ def probe_capabilities(is_stream): "f_back": False, "save_names": False, "set_local": False, - "serial_dap": is_stream, } if not caps["settrace"]: return caps From b12d2f65f0fcd5cbd50a8bc9cd42d337ea054122 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Fri, 21 Aug 2026 08:50:51 +1000 Subject: [PATCH 74/74] debugpy: Probe the firmware's capabilities once per interpreter. Every value describes the firmware, which cannot change while it is running, but the probe ran on each call - once for the boot script's handshake and again when a client attaches - and it is not free: `_probe_local_names` compiles source on the device to measure the firmware's own compiler rather than whatever produced this module. Memoising was wrong while the function took a per-session argument. It no longer does. A copy is handed out so a caller adding its own key - the boot script adds `repl_dap` - cannot reach the cache. Signed-off-by: Andrew Leech --- .../debugpy/debugpy/server/debug_session.py | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py index da010a22b..f9873b04f 100644 --- a/python-ecosys/debugpy/debugpy/server/debug_session.py +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -105,6 +105,11 @@ def _probe_local_names(frame): return namespace["_probe"]() +# Filled by DebugSession.probe_capabilities() on its first call; the values +# describe the firmware, which does not change while it is running. +_CAPABILITIES = None + + class DebugSession: """Manages a debugging session with a DAP client.""" @@ -150,10 +155,19 @@ def probe_capabilities(): it (`caps["repl_dap"]`). Safe to call on both the unix port and bare-metal builds; never raises. + Probed once per interpreter and remembered: every value describes the + firmware, which cannot change while it is running, and the probe is + not free - `_probe_local_names` compiles source on the device. A copy + is handed out so a caller adding its own key (the boot script adds + `repl_dap`) cannot reach the cache. + `save_names` is measured on code the firmware compiles here rather than on this module's own frame, so it reports the firmware and not how debugpy itself was deployed (see _probe_local_names). """ + global _CAPABILITIES + if _CAPABILITIES is not None: + return dict(_CAPABILITIES) caps = { "settrace": hasattr(sys, "settrace"), "f_back": False, @@ -161,12 +175,14 @@ def probe_capabilities(): "set_local": False, } if not caps["settrace"]: - return caps + _CAPABILITIES = caps + return dict(caps) try: frame = sys._getframe() except Exception: - return caps + _CAPABILITIES = caps + return dict(caps) try: caps["f_back"] = hasattr(frame, "f_back") @@ -189,7 +205,8 @@ def probe_capabilities(): except Exception: pass - return caps + _CAPABILITIES = caps + return dict(caps) def start(self): """Start the debug session message loop."""