diff --git a/python-ecosys/debugpy/README.md b/python-ecosys/debugpy/README.md new file mode 100644 index 000000000..6cbbd45bd --- /dev/null +++ b/python-ecosys/debugpy/README.md @@ -0,0 +1,188 @@ +# MicroPython debugpy + +A minimal implementation of debugpy for MicroPython, enabling remote debugging +such as VS Code debugging support. + +## Features + +- Debug Adapter Protocol (DAP) support for VS Code integration +- Basic debugging operations: + - Breakpoints + - Step over/into/out + - Stack trace inspection + - Variable inspection: globals, and locals when the firmware reports + `save_names` (otherwise positional `local_NN` placeholders). Locals are + read-only; no MicroPython build implements local-variable write-back. + - Expression evaluation + - Continue + - Pause + +A pause takes effect at the next line the target executes, because the trace +hook is the only thing that can interrupt it. A target that is executing no +Python - blocked in `time.sleep`, or inside a long-running C function - stays +running until it does; the request is not lost, it is pending. + +## Requirements + +- MicroPython with `sys.settrace` support (enabled with `MICROPY_PY_SYS_SETTRACE`). + Real local-variable names additionally need + `MICROPY_PY_SYS_SETTRACE_LOCALNAMES`, where the build provides it; without it + locals are reported as positional placeholders. +- Socket support for network communication +- JSON support for DAP message parsing + +## Usage + +### Basic Usage + +```python +import debugpy + +# Start listening for debugger connections +host, port = debugpy.listen() # Default: 127.0.0.1:5678 +print(f"Debugger listening on {host}:{port}") + +# Enable debugging for current thread +debugpy.debug_this_thread() + +# Your code here... +def my_function(): + x = 10 + y = 20 + result = x + y # Set breakpoint here in VS Code + return result + +result = my_function() +print(f"Result: {result}") + +# Manual breakpoint +debugpy.breakpoint() +``` + +### VS Code Configuration + +Create a `.vscode/launch.json` file in your project: + +```json +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Attach to MicroPython", + "type": "python", + "request": "attach", + "connect": { + "host": "127.0.0.1", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "." + } + ], + "justMyCode": false + } + ] +} +``` + +### Testing + +1. Build the MicroPython Unix port with tracing enabled: + ```bash + cd ports/unix + make CFLAGS_EXTRA="-DMICROPY_PY_SYS_SETTRACE=1" + ``` + +2. Run the sample program under a debug server. The program itself contains + no debugpy calls, so the server is started around it: + ```bash + cd lib/micropython-lib/python-ecosys/debugpy + ../../../../ports/unix/build-standard/micropython -c "import debugpy; \ + debugpy.listen(); debugpy.wait_for_client(); \ + debugpy.debug_this_thread(); import test_vscode; test_vscode.main()" + ``` + It waits for a client to attach and finish configuring before running + anything, so breakpoints set now are in place for the first line. + +3. In VS Code, open the debugpy folder and press F5 to attach the debugger + +4. Set breakpoints in `test_vscode.py` and observe debugging functionality + +## API Reference + +### `debugpy.listen(port=5678, host="127.0.0.1")` + +Start listening for debugger connections. + +**Parameters:** +- `port`: Port number to listen on (default: 5678) +- `host`: Host address to bind to (default: "127.0.0.1") + +**Returns:** Tuple of (host, port) actually used + +### `debugpy.debug_this_thread()` + +Enable debugging for the current thread by installing the trace function. + +### `debugpy.breakpoint()` + +Trigger a manual breakpoint that will pause execution if a debugger is attached. + +### `debugpy.wait_for_client()` + +Wait for the debugger client to connect and initialize. + +### `debugpy.is_client_connected()` + +Check if a debugger client is currently connected. + +**Returns:** Boolean indicating connection status + +### `debugpy.disconnect()` + +Disconnect from the debugger client and clean up resources. + +## Architecture + +The implementation consists of several key components: + +1. **Public API** (`public_api.py`): Main entry points for users +2. **Debug Session** (`server/debug_session.py`): Handles DAP protocol communication +3. **PDB Adapter** (`server/pdb_adapter.py`): Bridges DAP and MicroPython's trace system +4. **Messaging** (`common/messaging.py`): JSON message handling for DAP +5. **Constants** (`common/constants.py`): DAP protocol constants + +## Limitations + +This is a minimal implementation with the following limitations: + +- Single-threaded debugging only +- No conditional breakpoints +- No function breakpoints +- Limited variable inspection (no nested object expansion) +- No step back functionality +- No hot code reloading +- Simplified stepping implementation + +## Compatibility + +Tested with: +- MicroPython Unix port +- VS Code with Python/debugpy extension +- CPython 3.x (for comparison) + +## Contributing + +This implementation provides a foundation for MicroPython debugging. Contributions are welcome to add: + +- Conditional breakpoint support +- Better variable inspection +- Multi-threading support +- Performance optimizations +- Additional DAP features + +## License + +MIT License - see the MicroPython project license for details. diff --git a/python-ecosys/debugpy/dap_monitor.py b/python-ecosys/debugpy/dap_monitor.py new file mode 100755 index 000000000..85455c9a6 --- /dev/null +++ b/python-ecosys/debugpy/dap_monitor.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""DAP protocol monitor - sits between VS Code and MicroPython debugpy.""" + +import socket +import threading +import json +import time +import sys +import argparse + + +class DAPMonitor: + def __init__(self, listen_port=5679, target_host="127.0.0.1", target_port=5678): + self.disconnect = False + self.listen_port = listen_port + self.target_host = target_host + self.target_port = target_port + self.client_sock = None + self.server_sock = None + + def start(self): + """Start the DAP monitor proxy.""" + print(f"DAP Monitor starting on port {self.listen_port}") + print(f"Will forward to {self.target_host}:{self.target_port}") + print("Start MicroPython debugpy server first, then connect VS Code to port 5679") + + # Create listening socket + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", self.listen_port)) + listener.listen(1) + + print(f"Listening for VS Code connection on port {self.listen_port}...") + + try: + # Wait for VS Code to connect + self.client_sock, client_addr = listener.accept() + print(f"VS Code connected from {client_addr}") + + # Connect to MicroPython debugpy server + self.server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.server_sock.connect((self.target_host, self.target_port)) + print(f"Connected to MicroPython debugpy at {self.target_host}:{self.target_port}") + + # Start forwarding threads + threading.Thread(target=self.forward_client_to_server, daemon=True).start() + threading.Thread(target=self.forward_server_to_client, daemon=True).start() + + print("DAP Monitor active - press Ctrl+C to stop") + while not self.disconnect: + time.sleep(1) + + except KeyboardInterrupt: + print("\nStopping DAP Monitor...") + except Exception as e: + print(f"Error: {e}") + finally: + self.cleanup() + + def forward_client_to_server(self): + """Forward messages from VS Code client to MicroPython server.""" + try: + while True: + data = self.receive_dap_message(self.client_sock, "VS Code") + if data is None: + break + self.send_raw_data(self.server_sock, data) + except Exception as e: + print(f"Client->Server forwarding error: {e}") + + def forward_server_to_client(self): + """Forward messages from MicroPython server to VS Code client.""" + try: + while True: + data = self.receive_dap_message(self.server_sock, "MicroPython") + if data is None: + break + self.send_raw_data(self.client_sock, data) + except Exception as e: + print(f"Server->Client forwarding error: {e}") + + def receive_dap_message(self, sock, source): + """Receive and log a DAP message.""" + try: + # Read headers + header = b"" + while b"\r\n\r\n" not in header: + byte = sock.recv(1) + if not byte: + return None + header += byte + + # Parse content length + header_str = header.decode("utf-8") + content_length = 0 + for line in header_str.split("\r\n"): + if line.startswith("Content-Length:"): + content_length = int(line.split(":", 1)[1].strip()) + break + + if content_length == 0: + return None + + # Read content + content = b"" + while len(content) < content_length: + chunk = sock.recv(content_length - len(content)) + if not chunk: + return None + content += chunk + + # Parse and Log the message + message = self.parse_dap(source, content) + self.log_dap_message(source, message) + # Check for disconnect command + if message: + if "disconnect" == message.get("command", message.get("event", "unknown")): + print(f"\n[{source}] Disconnect command received, stopping monitor.") + self.disconnect = True + return header + content + except Exception as e: + print(f"Error receiving from {source}: {e}") + return None + + def parse_dap(self, source, content): + """Parse DAP message and log it.""" + try: + message = json.loads(content.decode("utf-8")) + return message + except json.JSONDecodeError: + print(f"\n[{source}] Invalid JSON: {content}") + return None + + def log_dap_message(self, source, message): + """Log DAP message details.""" + msg_type = message.get("type", "unknown") + command = message.get("command", message.get("event", "unknown")) + seq = message.get("seq", 0) + + print(f"\n[{source}] {msg_type.upper()}: {command} (seq={seq})") + + if msg_type == "request": + args = message.get("arguments", {}) + if args: + print(f" Arguments: {json.dumps(args, indent=2)}") + elif msg_type == "response": + success = message.get("success", False) + req_seq = message.get("request_seq", 0) + print(f" Success: {success}, Request Seq: {req_seq}") + body = message.get("body") + if body: + print(f" Body: {json.dumps(body, indent=2)}") + msg = message.get("message") + if msg: + print(f" Message: {msg}") + elif msg_type == "event": + body = message.get("body", {}) + if body: + print(f" Body: {json.dumps(body, indent=2)}") + + def send_raw_data(self, sock, data): + """Send raw data to socket.""" + try: + sock.send(data) + except Exception as e: + print(f"Error sending data: {e}") + + def cleanup(self): + """Clean up sockets.""" + if self.client_sock: + self.client_sock.close() + if self.server_sock: + self.server_sock.close() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="DAP protocol monitor proxy") + parser.add_argument( + "--target-host", + "--th", + default="127.0.0.1", + help="Target debugpy host (default: 127.0.0.1)", + ) + parser.add_argument( + "--target-port", "--tp", type=int, default=5678, help="Target debugpy port (default: 5678)" + ) + parser.add_argument( + "--listen-port", + "--lp", + type=int, + default=5679, + help="Port to listen for VS Code (default: 5679)", + ) + args = parser.parse_args() + + monitor = DAPMonitor( + listen_port=args.listen_port, target_host=args.target_host, target_port=args.target_port + ) + monitor.start() diff --git a/python-ecosys/debugpy/debugpy/__init__.py b/python-ecosys/debugpy/debugpy/__init__.py new file mode 100644 index 000000000..ed2494e0e --- /dev/null +++ b/python-ecosys/debugpy/debugpy/__init__.py @@ -0,0 +1,41 @@ +"""MicroPython debugpy implementation. + +A minimal port of debugpy for MicroPython to enable VS Code debugging support. +This implementation focuses on the core DAP (Debug Adapter Protocol) functionality +needed for basic debugging operations like breakpoints, stepping, and variable inspection. +""" + +__version__ = "0.1.0" + +from .public_api import ( + RestartRequest, + breakpoint, + console, + debug_this_thread, + disconnect, + enable_restart, + get_capabilities, + is_client_connected, + listen, + listen_stream, + wait_for_client, + wait_for_restart, +) +from .common.constants import DEFAULT_HOST, DEFAULT_PORT + +__all__ = [ + "DEFAULT_HOST", + "DEFAULT_PORT", + "RestartRequest", + "breakpoint", + "console", + "debug_this_thread", + "disconnect", + "enable_restart", + "get_capabilities", + "is_client_connected", + "listen", + "listen_stream", + "wait_for_client", + "wait_for_restart", +] diff --git a/python-ecosys/debugpy/debugpy/common/__init__.py b/python-ecosys/debugpy/debugpy/common/__init__.py new file mode 100644 index 000000000..c53632010 --- /dev/null +++ b/python-ecosys/debugpy/debugpy/common/__init__.py @@ -0,0 +1 @@ +# Common utilities and constants for debugpy diff --git a/python-ecosys/debugpy/debugpy/common/constants.py b/python-ecosys/debugpy/debugpy/common/constants.py new file mode 100644 index 000000000..a0fd245c8 --- /dev/null +++ b/python-ecosys/debugpy/debugpy/common/constants.py @@ -0,0 +1,75 @@ +"""Constants used throughout debugpy.""" + +from micropython import const + +# Default networking settings +DEFAULT_HOST = "127.0.0.1" +DEFAULT_PORT = 5678 + +# DAP message types +MSG_TYPE_REQUEST = const("request") +MSG_TYPE_RESPONSE = const("response") +MSG_TYPE_EVENT = const("event") + +# DAP events +EVENT_INITIALIZED = const("initialized") +EVENT_STOPPED = const("stopped") +EVENT_CONTINUED = const("continued") +EVENT_THREAD = const("thread") +EVENT_BREAKPOINT = const("breakpoint") +EVENT_OUTPUT = const("output") +EVENT_TERMINATED = const("terminated") +EVENT_EXITED = const("exited") + +# DAP commands +CMD_INITIALIZE = const("initialize") +CMD_LAUNCH = const("launch") +CMD_ATTACH = const("attach") +CMD_SET_BREAKPOINTS = const("setBreakpoints") +CMD_CONTINUE = const("continue") +CMD_NEXT = const("next") +CMD_STEP_IN = const("stepIn") +CMD_STEP_OUT = const("stepOut") +CMD_PAUSE = const("pause") +CMD_STACK_TRACE = const("stackTrace") +CMD_SCOPES = const("scopes") +CMD_VARIABLES = const("variables") +CMD_SET_VARIABLE = const("setVariable") +CMD_EVALUATE = const("evaluate") +CMD_DISCONNECT = const("disconnect") +CMD_RESTART = const("restart") +CMD_CONFIGURATION_DONE = const("configurationDone") +CMD_THREADS = const("threads") +CMD_SOURCE = const("source") + +# Stop reasons +STOP_REASON_STEP = const("step") +STOP_REASON_BREAKPOINT = const("breakpoint") +STOP_REASON_EXCEPTION = const("exception") +STOP_REASON_PAUSE = const("pause") +STOP_REASON_ENTRY = const("entry") + +# Thread reasons +THREAD_REASON_STARTED = const("started") +THREAD_REASON_EXITED = const("exited") + +# Trace events +TRACE_CALL = const("call") +TRACE_LINE = const("line") +TRACE_RETURN = const("return") +TRACE_EXCEPTION = const("exception") + +# Step modes +STEP_INTO = const("into") +STEP_OVER = const("over") +STEP_OUT = const("out") + + +# Scope types +SCOPE_LOCALS = const("locals") +SCOPE_GLOBALS = const("globals") + +# Bounded wait for the DAP client to send configurationDone (seconds). There is +# no server thread, so a hang here would spin forever with no diagnostic; a +# timeout with a clear message replaces a silent guessed delay. +WAIT_FOR_CLIENT_TIMEOUT_S = const(30) diff --git a/python-ecosys/debugpy/debugpy/common/messaging.py b/python-ecosys/debugpy/debugpy/common/messaging.py new file mode 100644 index 000000000..cf2f94991 --- /dev/null +++ b/python-ecosys/debugpy/debugpy/common/messaging.py @@ -0,0 +1,203 @@ +"""JSON message handling for DAP protocol.""" + +import json +import time + +from .constants import MSG_TYPE_REQUEST, MSG_TYPE_RESPONSE, MSG_TYPE_EVENT + +# "Nothing to read / no room to write right now" on a socket carrying a +# timeout, which this channel always does (see DebugSession's settimeout +# calls). Which errno says that depends on the network stack, not on the +# situation: the unix port raises EAGAIN, and lwIP - every WiFi board - +# raises ETIMEDOUT once the timeout elapses. None of them means the peer went +# away, so none may close the channel. A stack whose errno is missing here +# drops the session on its first idle poll, which is immediately. +_WOULD_BLOCK = (11, 35, 110) # EAGAIN, EWOULDBLOCK, ETIMEDOUT + +# How long a single DAP frame may take to drain into the socket before the +# peer is treated as gone. Generous: it is a stall budget, not a latency +# target, and the loop returns as soon as the bytes are written. +_SEND_DEADLINE_MS = 5000 + + +class JsonMessageChannel: + """Handles JSON message communication over a socket using DAP format.""" + + def __init__(self, sock, debug_callback=None): + self.sock = sock + self.seq = 0 + self.closed = False + self._recv_buffer = b"" + self._debug_print = debug_callback or (lambda x: None) # Default to no-op + + def send_message(self, msg_type, command=None, **kwargs): + """Send a DAP message.""" + if self.closed: + return + + self.seq += 1 + message = { + "seq": self.seq, + "type": msg_type, + } + + if command: + if msg_type == MSG_TYPE_REQUEST: + message["command"] = command + if kwargs: + message["arguments"] = kwargs + elif msg_type == MSG_TYPE_RESPONSE: + message["command"] = command + message["request_seq"] = kwargs.get("request_seq", 0) + message["success"] = kwargs.get("success", True) + if "body" in kwargs: + message["body"] = kwargs["body"] + if "message" in kwargs: + message["message"] = kwargs["message"] + elif msg_type == MSG_TYPE_EVENT: + message["event"] = command + if kwargs: + message["body"] = kwargs + + json_str = json.dumps(message) + content = json_str.encode("utf-8") + header = f"Content-Length: {len(content)}\r\n\r\n".encode("utf-8") + + self._send_all(header + content) + + def _send_all(self, data): + """Write every byte of one frame, or close the channel trying. + + `sock.send()` is not `sendall()`: it may accept a prefix and return + the count, and on a socket carrying a timeout it reports a full + transmit buffer as an error instead of blocking. Either one truncates + a DAP frame mid-`Content-Length`, which desynchronises the stream for + the rest of the session rather than failing visibly. Both are ordinary + on a board - a `variables` response is easily larger than lwIP's + window - so the write is driven to completion here. + + `sendall()` is not the answer: MicroPython does not support it on a + socket with a timeout, which is the only kind this channel has. + """ + view = memoryview(data) + sent = 0 + start = time.ticks_ms() + while sent < len(view): + try: + sent += self.sock.send(view[sent:]) + except OSError as e: + if getattr(e, "errno", None) not in _WOULD_BLOCK: + self.closed = True + return + if time.ticks_diff(time.ticks_ms(), start) > _SEND_DEADLINE_MS: + self._debug_print( + "[DAP] send stalled with {} of {} bytes written; closing".format( + sent, len(view) + ) + ) + self.closed = True + return + time.sleep(0.001) + + def send_request(self, command, **kwargs): + """Send a request message.""" + self.send_message(MSG_TYPE_REQUEST, command, **kwargs) + + def send_response(self, command, request_seq, success=True, body=None, message=None): + """Send a response message.""" + kwargs = {"request_seq": request_seq, "success": success} + if body is not None: + kwargs["body"] = body + if message is not None: + kwargs["message"] = message + + self._debug_print( + f"[DAP] SEND: response {command} (req_seq={request_seq}, success={success})" + ) + if body: + self._debug_print(f"[DAP] body: {body}") + if message: + self._debug_print(f"[DAP] message: {message}") + + self.send_message(MSG_TYPE_RESPONSE, command, **kwargs) + + def send_event(self, event, **kwargs): + """Send an event message.""" + self._debug_print(f"[DAP] SEND: event {event}") + if kwargs: + self._debug_print(f"[DAP] body: {kwargs}") + self.send_message(MSG_TYPE_EVENT, event, **kwargs) + + def recv_message(self): + """Receive a DAP message, or None if a full one isn't available yet. + + Called repeatedly against a socket with a short recv timeout (see + `DebugSession.process_pending_messages`), so a single message's + header and body routinely arrive across several calls. Everything + read so far - including an already-located header - is kept in + `self._recv_buffer` verbatim until the *entire* message (header + + `Content-Length` body bytes) is available, and only then is it + parsed and sliced off. Parsing the header again on each call is + cheap and avoids having to separately persist "header already + parsed, N body bytes still outstanding" state between calls: a + prior version stripped the header out of the buffer as soon as it + was found, which discarded that state and desynchronised framing + for the rest of the connection whenever the body arrived in a + later read than the header. + """ + if self.closed: + return None + + # Non-blocking top-up: pull in whatever is available right now + # without blocking if there's nothing new yet. + try: + data = self.sock.recv(4096) + if not data: + # A truly empty read (as opposed to a _WOULD_BLOCK errno, + # handled below) means the peer closed the connection. + self.closed = True + return None + self._recv_buffer += data + except OSError as e: + if getattr(e, "errno", None) not in _WOULD_BLOCK: + self.closed = True + return None + # No new data available right now - fall through and try to + # parse a complete message out of whatever is already buffered. + + recv_buffer = self._recv_buffer + header_end = recv_buffer.find(b"\r\n\r\n") + if header_end < 0: + return None # Header not fully received yet. + + header_str = recv_buffer[:header_end].decode("utf-8") + content_length = 0 + for line in header_str.split("\r\n"): + if line.startswith("Content-Length:"): + content_length = int(line.split(":", 1)[1].strip()) + break + + body_start = header_end + 4 + if len(recv_buffer) < body_start + content_length: + return None # Body not fully received yet. + + body = recv_buffer[body_start : body_start + content_length] + self._recv_buffer = recv_buffer[body_start + content_length :] + + try: + message = json.loads(body.decode("utf-8")) + self._debug_print( + f"[DAP] Successfully received message: {message.get('type')} {message.get('command', message.get('event', 'unknown'))}" + ) + return message + except (ValueError, UnicodeDecodeError) as e: + print(f"[DAP] JSON parse error: {e}") + return None + + def close(self): + """Close the channel.""" + self.closed = True + try: + self.sock.close() + except OSError: + pass diff --git a/python-ecosys/debugpy/debugpy/common/repl_mux.py b/python-ecosys/debugpy/debugpy/common/repl_mux.py new file mode 100644 index 000000000..f39931669 --- /dev/null +++ b/python-ecosys/debugpy/debugpy/common/repl_mux.py @@ -0,0 +1,462 @@ +"""Framing that lets the DAP channel share the REPL's one byte stream. + +For a board with a single UART and no network, the stream carrying the REPL is +the only way in. `mpremote mount` already puts a second protocol on that stream +by marking its exchanges with `0x18`, and this reuses the marker and the code +namespace so one demux point can eventually serve both: codes 1..13 belong to +mount's filesystem RPC and are never emitted here. + +The wire is the same in both directions: + + 0x18 0x18 one literal 0x18 in the plain byte stream + 0x18 a framed message of `lo | hi << 8` bytes + anything else plain bytes - program stdout one way, + program stdin the other + +`Demux` is the reader for that wire and is used unmodified at both ends, so +"the program's output reaches the terminal" and "a program calling `input()` +gets what the user typed" are the same code path, tested once. The length is +explicit rather than reusing the DAP message's own `Content-Length` header so +that a reader never inspects a payload to find where it ends: the only content +assumption on the wire is about `0x18`, and escaping removes that one. + +Inbound bytes are credited (`CMD_DAP_ACK`), because the receive path this rides +on is a fixed ring that discards the tail of a packet when it overflows rather +than exerting back-pressure - the device drains it only when something polls, +and a target sitting in `time.sleep()` polls nothing. A sender that respects +the credit cannot overrun a receiver that has stopped listening; it blocks +instead, which is recoverable. +""" + +import errno +import io +import select + +# `mp_stream_p_t` request numbers, as `io.IOBase` hands them to `ioctl`. +_IOCTL_POLL = 3 +_IOCTL_GET_FILENO = 10 + +# What both façades answer to `MP_STREAM_GET_FILENO`: a negative return from a +# Python `ioctl` is `MP_STREAM_ERROR` with this as its errno. Neither of them +# is a file, and saying so matters rather than being pedantry - a port built +# with `MICROPY_PY_SELECT_POSIX_OPTIMISATIONS` (the unix one) asks every object +# registered with `select.poll` for a descriptor and, when it gets a number, +# polls *that* and never calls the object's own `ioctl` again. A façade that +# answered 0 would have the poller watching stdin while the demux that actually +# holds the bytes was never pumped. +_NOT_A_FILE = -errno.EINVAL + +MARKER = 0x18 + +# Shared with `mpremote`'s `fs_hook_cmds`, which owns 1..13. +CMD_DAP = 14 +CMD_DAP_ACK = 15 +# The DAP channel is over. Sent because the streams this rides on have no EOF +# of their own - an idle USB CDC interface and one nobody is holding both read +# as no bytes - so without it a host would wait out a finished session. Only +# the device sends it: a host that goes away drops DTR, which the device reads +# through `isconnected()`, and a client that merely detaches is the DAP +# protocol's own business. +CMD_DAP_EOF = 16 + +# Wire bytes a sender may have outstanding before the peer credits them. The +# smallest receive ring in the ports this runs on is 256 bytes +# (`MICROPY_HW_USB_CDC_RX_DATA_SIZE`), so this leaves room for a partly-drained +# one. Both ends must agree: `mpremote`'s copy is checked against this. +RX_CREDIT = 192 + +# Bytes of payload per frame. Bounds the reader's per-frame buffer, and is +# capped so that one whole frame always fits inside the credit window: a frame +# bigger than `RX_CREDIT` could never be sent at all, since the sender would +# wait for credit that only its own unsent bytes could earn. A DAP message +# longer than this is split across frames. +MAX_PAYLOAD = 128 + +# Consumed bytes a receiver may leave uncredited before it sends an ack. Small +# enough that a sender at the credit limit is released within one exchange, +# large enough that a steady stream does not ack every frame. A receiver that +# has drained everything acks regardless, so a sender is never left waiting on +# a threshold that will not be reached. +ACK_THRESHOLD = 64 + +# Outbound bytes queued before the DAP side reports itself unwritable. Reached +# only when the peer has stopped reading; the DAP layer then retries, which is +# back-pressure, where growing without limit would be a slow memory error on a +# device that has a few tens of kilobytes to spend. +MAX_OUTBOX = 4096 + +# Milliseconds a stalled console write waits for room before giving up and +# leaving the rest queued. +_STALL_MS = 100 + + +def frame(code, payload): + """One framed message: marker, code, two-byte length, payload.""" + n = len(payload) + return bytes((MARKER, code, n & 0xFF, (n >> 8) & 0xFF)) + bytes(payload) + + +def _consume(buf, n): + """Drop the first `n` bytes of `buf`, in place. + + `del buf[:n]` is the obvious spelling and raises on a MicroPython + bytearray, which implements slice assignment but not slice deletion. + Emptying it outright is the common case - a reader almost always takes + everything queued - and is the one path here that allocates nothing. + """ + if n >= len(buf): + buf[:] = b"" + else: + buf[:] = buf[n:] + + +def escape(data): + """Plain bytes with the marker doubled, so a reader never mistakes one. + + A program that prints `0x18` is the case this exists for; it costs one + extra byte per occurrence and nothing at all for output that has none. + """ + data = bytes(data) + if data.find(bytes((MARKER,))) < 0: + return data + return data.replace(bytes((MARKER,)), bytes((MARKER, MARKER))) + + +class Demux: + """Reads the wire above into plain bytes and framed payload. + + `plain` and `dap` are drained by the caller; `credited` accumulates what + the peer has reported consuming. `unknown_code` records the first code + this reader has no handler for, which on a single-UART link means the two + ends disagree about the protocol - worth reporting rather than skipping, + since every later byte is then suspect. + + Feeding is incremental: a marker, a length or a payload split across two + reads resumes where it left off, which is the normal case on a stream that + delivers whatever a USB packet happened to contain. + """ + + _PLAIN, _CODE, _LEN0, _LEN1, _BODY = 0, 1, 2, 3, 4 + + def __init__(self): + self.plain = bytearray() + self.dap = bytearray() + self.credited = 0 + self.eof = False + self.unknown_code = None + self._state = self._PLAIN + self._code = 0 + self._need = 0 + self._body = bytearray() + + def feed(self, data): + data = bytes(data) + i = 0 + end = len(data) + while i < end: + if self._state == self._PLAIN: + # Copy up to the next marker in one go: plain bytes are the + # bulk of the traffic in the stdout direction and scanning + # them one at a time is the whole cost of the layer. + j = data.find(bytes((MARKER,)), i) + if j < 0: + self.plain += data[i:] + return + self.plain += data[i:j] + i = j + 1 + self._state = self._CODE + continue + + c = data[i] + i += 1 + if self._state == self._CODE: + if c == MARKER: + self.plain.append(MARKER) # escaped literal + self._state = self._PLAIN + else: + self._code = c + self._state = self._LEN0 + elif self._state == self._LEN0: + self._need = c + self._state = self._LEN1 + elif self._state == self._LEN1: + self._need |= c << 8 + self._body = bytearray() + self._state = self._BODY if self._need else self._PLAIN + if not self._need: + self._deliver() + else: # _BODY + self._body.append(c) + if len(self._body) == self._need: + self._deliver() + self._state = self._PLAIN + + def _deliver(self): + if self._code == CMD_DAP: + self.dap += self._body + elif self._code == CMD_DAP_ACK: + if len(self._body) >= 2: + self.credited += self._body[0] | (self._body[1] << 8) + elif self._code == CMD_DAP_EOF: + self.eof = True + elif self.unknown_code is None: + self.unknown_code = self._code + + def take_plain(self, n): + out = bytes(self.plain[:n]) + _consume(self.plain, len(out)) + return out + + def take_dap(self, n): + out = bytes(self.dap[:n]) + _consume(self.dap, len(out)) + return out + + +class ReplMux: + """Owns the REPL stream and hands out the two façades that share it. + + `console` goes wherever the runtime's stdout is diverted - a `dupterm` + slot on the ports where that reaches every byte - and `dap` is handed to + `debugpy.listen_stream()`. They are separate objects because the bytes + arriving at a `write()` are the only thing that distinguishes program + output from a DAP frame, so one object could not tell them apart. + + Everything either façade emits is appended to one outbound buffer in whole + frames and flushed from there, so a partial write to the underlying port + can never interleave two messages: the port sees frames in the order they + were produced, or it sees a prefix of that order. + """ + + def __init__(self, port=None): + self._port = None + self._poller = None + self._write_poller = None + self._out = bytearray() + self._demux = Demux() + self._consumed = 0 # payload bytes taken from `dap` since the last ack + self._byte = bytearray(1) + self.console = _PlainSide(self) + self.dap = _DapSide(self) + if port is not None: + self.attach(port) + + def attach(self, port): + """Take `port` as the real stream. + + Separate from `__init__` because on a port where the diversion is a + `dupterm` slot, the object being displaced *is* the real stream, and + it is only known once the swap has happened. + """ + self._port = port + self._poller = select.poll() + self._poller.register(port, select.POLLIN) + self._write_poller = select.poll() + self._write_poller.register(port, select.POLLOUT) + # The DAP façade inherits whatever the real stream has instead of EOF. + # A USB CDC interface has none - an idle one and one nobody is holding + # both read as no bytes - but it does have `isconnected()`, the DTR + # line the host raises on open and the kernel drops when the last + # opener goes, which is how a session notices its host disappeared. + self.dap.isconnected = getattr(port, "isconnected", None) + + # -- outbound --------------------------------------------------------- + + def _emit(self, data): + """Queue whole frames and push what the port will take. Never raises. + + An exception out of the console façade's `write()` would deactivate the + stdout diversion it sits in, which on a single-stream board silently + removes the console and the debug channel at once, so nothing here is + allowed to escape. A short write is not a failure: what did not go out + stays queued and leaves with the next call. + """ + self._out += data + self._flush() + + def _flush(self, timeout_ms=0): + while self._out: + if self._port is None: + return False + if not self._write_poller.poll(timeout_ms): + return False + try: + n = self._port.write(self._out) + except OSError: + return False # EAGAIN, or the port has gone; either way, later + if not n: + return False + _consume(self._out, n) + return True + + # -- inbound ---------------------------------------------------------- + + def pump(self, timeout_ms=0): + """Move whatever the port has into the demux, and credit it. + + Read one byte at a time behind a fresh zero-timeout poll, for the + reason `StreamTransport.recv` gives: a bulk `readinto` on a stream + whose read genuinely blocks (a plain file, as against a CDC driver + that always returns immediately) would wait for bytes that may never + come, because the request is looped internally until the buffer fills. + + Credit is counted here rather than where a payload is handed on, + because what the credit protects is the receive ring, and a byte read + out of it has already freed its place there whatever happens to it + next. Counting it later would credit a frame's overhead once per + partial read of it, which over-reports and defeats the limit. + """ + if self._port is None: + return False + if not self._poller.poll(timeout_ms): + self._send_ack(force=True) + return False + got = False + while True: + try: + r = self._port.readinto(self._byte) + except OSError: + break + if not r: + break + self._demux.feed(self._byte) + self._consumed += r + got = True + if self._consumed >= ACK_THRESHOLD: + self._send_ack() # mid-burst, so a long one does not stall the peer + if not self._poller.poll(0): + break + self._send_ack(force=True) + return got + + def _send_ack(self, force=False): + """Tell the peer how much of its traffic has left this end's buffer. + + Credit is what stops a sender overrunning a receive ring that drops + the tail of a packet on overflow. It is reported in wire bytes, the + unit the ring holds, so the sender does not have to model framing. + """ + if self._consumed and (force or self._consumed >= ACK_THRESHOLD): + n = self._consumed + self._consumed = 0 + self._emit(frame(CMD_DAP_ACK, bytes((n & 0xFF, (n >> 8) & 0xFF)))) + + @property + def unknown_code(self): + return self._demux.unknown_code + + def detach(self): + """Push out what is queued and give the real stream back. + + Returns the port so the caller can put it where it found it. The flush + is bounded rather than unconditional: a peer that has stopped reading + must not be able to keep the stream split forever, and by this point + whatever is still queued is the tail of a session that is over. + """ + if self._port is not None: + # Last framed message on the channel, and the peer's only notice + # that the session ended rather than went quiet. + self._emit(frame(CMD_DAP_EOF, b"")) + self._flush(_STALL_MS) + port = self._port + self._port = None + self._poller = None + self._write_poller = None + self.dap.isconnected = None + return port + + def close(self): + self.detach() + + +class _PlainSide(io.IOBase): + """The program's own stdout and stdin, escaped onto the shared stream. + + `io.IOBase` rather than a plain class because both places this object goes + demand the native stream protocol: `os.dupterm` raises `OSError: stream + operation not supported` without it, and `select.poll().register()` needs + the `ioctl`. + """ + + def __init__(self, mux): + self._mux = mux + + def write(self, buf): + n = len(buf) + mux = self._mux + mux._emit(escape(buf)) + if len(mux._out) > MAX_OUTBOX: + # Program output is never dropped, so a peer that has stopped + # reading slows the program down instead. Bounded: one flush + # attempt, not a wait for the whole queue to leave. + mux._flush(_STALL_MS) + return n # every byte was accepted; delivery is the outbound queue's job + + def readinto(self, buf, nbytes=None): + mux = self._mux + mux.pump() + want = len(buf) if nbytes is None else min(nbytes, len(buf)) + data = mux._demux.take_plain(want) + buf[: len(data)] = data + return len(data) + + def ioctl(self, op, arg): + if op == _IOCTL_POLL: + mux = self._mux + mux.pump() + ret = 0 + if arg & 1 and mux._demux.plain: # MP_STREAM_POLL_RD + ret |= 1 + if arg & 4: # MP_STREAM_POLL_WR - the outbound queue always accepts + ret |= 4 + return ret + if op == _IOCTL_GET_FILENO: + return _NOT_A_FILE + return 0 + + +class _DapSide(io.IOBase): + """The DAP channel, framed onto the shared stream. + + Handed to `debugpy.listen_stream()`, which registers it with `select.poll` + for both directions and calls `poll()` fresh on every `recv`/`send`. That + makes `ioctl` the place the inbound side is pumped: it runs on every wait + the DAP layer does, which under `settrace` is at every trace event. + """ + + def __init__(self, mux): + self._mux = mux + # Set by `ReplMux.attach` to the real stream's own liveness test, or + # left None where it has none. An instance attribute rather than a + # method so that `getattr(stream, "isconnected", None)` - what the + # boot script asks - can answer "this stream cannot tell you". + self.isconnected = None + + def write(self, buf): + mux = self._mux + data = bytes(buf) + for i in range(0, len(data), MAX_PAYLOAD): + mux._emit(frame(CMD_DAP, data[i : i + MAX_PAYLOAD])) + return len(data) + + def readinto(self, buf, nbytes=None): + mux = self._mux + mux.pump() + want = len(buf) if nbytes is None else min(nbytes, len(buf)) + data = mux._demux.take_dap(want) + buf[: len(data)] = data + return len(data) + + def ioctl(self, op, arg): + if op == _IOCTL_POLL: + mux = self._mux + mux.pump() + ret = 0 + if arg & 1 and mux._demux.dap: # MP_STREAM_POLL_RD + ret |= 1 + if arg & 4 and len(mux._out) <= MAX_OUTBOX: # MP_STREAM_POLL_WR + ret |= 4 + return ret + if op == _IOCTL_GET_FILENO: + return _NOT_A_FILE + return 0 diff --git a/python-ecosys/debugpy/debugpy/common/stream_transport.py b/python-ecosys/debugpy/debugpy/common/stream_transport.py new file mode 100644 index 000000000..5d7ce5bc8 --- /dev/null +++ b/python-ecosys/debugpy/debugpy/common/stream_transport.py @@ -0,0 +1,175 @@ +"""Socket-shaped adapter for running the DAP channel over a stream, not a socket.""" + +import select + +# How long a wait that was asked to block forever may sit inside one poll +# before the liveness check runs again. The line `is_connected` reads is not +# something a poll can wait on, so a blocking wait would otherwise never +# notice the host letting go. A bound on how quickly that is noticed, not a +# timeout: a wait carrying a real one is still governed by it. +_LIVENESS_SLICE_MS = 100 + + +class StreamTransport: + """Presents a reader/writer stream pair as a socket to `JsonMessageChannel`. + + For boards with a dedicated CDC interface for DAP (no shared REPL/stdout + byte stream to demux) - the interface `messaging.py` and `debug_session.py` + need is exactly `recv`/`send`/`settimeout`/`close`, so this wraps the + stream to present that shape rather than changing anything downstream of + `self.sock`. + + `recv` raises `OSError(11)` (EAGAIN) when no data has arrived within the + current timeout, matching a non-blocking socket, and returns `b""` once + the stream is at EOF. `send` writes once and reports how much went out, + also matching a socket, raising EAGAIN only when nothing could be + written; driving a whole frame out is `messaging.py`'s `_send_all`, the + one place that knows where the frame boundary is. `settimeout` is mutated + repeatedly at runtime by `public_api.py`/`debug_session.py` and governs + both directions, matching a real socket; `poll(ms)` is called fresh on + every `recv`/`send` rather than cached, so each change takes effect + immediately. + + `is_connected`, when the runtime has something to offer, is a callable + reporting whether the host still holds the other end - see `_peer_gone`. + """ + + def __init__(self, reader, writer=None, is_connected=None): + self._reader = reader + self._writer = writer if writer is not None else reader + self._timeout = None # seconds, or None = block forever + self._eof = False + self._is_connected = is_connected + self._had_traffic = False + self._was_held = is_connected is not None and bool(is_connected()) + self._poller = select.poll() + self._poller.register(self._reader, select.POLLIN) + self._write_poller = select.poll() + self._write_poller.register(self._writer, select.POLLOUT) + self._buf = bytearray() # grown to the largest `n` seen, reused across recv() calls + + def settimeout(self, seconds): + self._timeout = seconds + + def _peer_gone(self): + """Has the host held this channel and then let go of it? + + A USB CDC interface never reaches EOF. An idle one and one whose host + has vanished read identically - no bytes - so a session stopped at a + breakpoint would wait for a `continue` that cannot come, and the board + would need a power cycle. `is_connected` is whatever the runtime has + instead: on stm32 it is `pyb.USB_VCP.isconnected()`, the interface's + DTR line, raised by the host when it opens the port and dropped by the + kernel when the last opener goes away. + + It counts only once the channel has been held, because the line goes + up and down for reasons that are not a session ending: a host may open + an idle interface briefly just to check that it can. Down on its own + therefore says nothing; down after the channel was held is the peer + leaving. Held means either of two things, and which one applies is + decided by whether anyone was on the far end when the channel was + made: + + - Nobody was, so the channel has to earn the signal - a byte has to + cross it. This is the dedicated-DAP-interface case: nothing holds + that interface between `listen_stream()` and the client's first + connect, so its line being down at any point before then is the + ordinary state and not a peer. + - Somebody was, recorded at construction as `_was_held`. This is the + case where DAP shares the stream the host is already using to drive + the board: that hold predates the channel, so nothing but the host + leaving can drop it, and waiting for traffic would mean a client + that never sends anything holds the stream forever. + """ + if self._is_connected is None or not (self._had_traffic or self._was_held): + return False + return not self._is_connected() + + def _wait(self, poller, timeout_ms): + """Poll for readiness, giving up early once the peer has gone. + + Returns whatever `poll` returned, which is falsy both when the wait + timed out and when the peer left; callers separate the two with + `_peer_gone()`. A wait asked to block forever is served in slices so + that check gets to run - see `_LIVENESS_SLICE_MS`. + """ + if timeout_ms is not None: + return poller.poll(timeout_ms) + while True: + ready = poller.poll(_LIVENESS_SLICE_MS) + if ready or self._peer_gone(): + return ready + + def recv(self, n): + if self._eof: + return b"" + if self._peer_gone(): + self._eof = True + return b"" + timeout_ms = None if self._timeout is None else max(0, int(self._timeout * 1000)) + if not self._wait(self._poller, timeout_ms): + if self._peer_gone(): + self._eof = True + return b"" + raise OSError(11) # EAGAIN: no data within the timeout + + # `.read()`/`.readinto()` loop internally until the buffer is full + # (py/stream.c's mp_stream_rw) - on a stream backed by a genuinely + # blocking read (a plain POSIX file, as opposed to a UART/CDC driver + # that always returns immediately), a second internal call would + # block waiting for bytes that may never come. Reading one byte at a + # time, re-checking readiness with a zero-timeout poll before every + # further byte, never makes a read the poller hasn't already + # confirmed data for. + if len(self._buf) < n: + self._buf = bytearray(n) + mv = memoryview(self._buf) + got = 0 + while got < n: + r = self._reader.readinto(mv[got : got + 1]) + if r is None: + break # raced the poll result - not actually ready + if not r: + self._eof = True + break + got += r + if got < n and not self._poller.poll(0): + break + + if got == 0: + if self._eof: + return b"" + raise OSError(11) + self._had_traffic = True + return bytes(mv[:got]) + + def send(self, data): + # A short write is normal, not exceptional: a CDC interface takes at + # most its transmit buffer per call (1024 bytes on stm32) and reports + # the count, exactly as a socket reports a short send. Reporting that + # count is what lets the caller resume from the right offset - a + # whole-buffer contract would have to raise once the timeout expired + # mid-frame, throwing away the count of what had already gone out, and + # the retry would then re-send that prefix and desynchronise the + # Content-Length framing it was meant to protect. + timeout_ms = None if self._timeout is None else max(0, int(self._timeout * 1000)) + if not self._wait(self._write_poller, timeout_ms): + raise OSError(11) # EAGAIN: no room within the timeout + # `write()` answers a non-blocking stream that took nothing with None + # rather than 0, so both are treated as "came back not ready after + # all" - the poll above can only promise room at the moment it ran. + written = self._writer.write(memoryview(data)) + if not written: + raise OSError(11) + return written + + def close(self): + try: + self._reader.close() + except OSError: + pass + if self._writer is not self._reader: + try: + self._writer.close() + except OSError: + pass diff --git a/python-ecosys/debugpy/debugpy/public_api.py b/python-ecosys/debugpy/debugpy/public_api.py new file mode 100644 index 000000000..65d119876 --- /dev/null +++ b/python-ecosys/debugpy/debugpy/public_api.py @@ -0,0 +1,304 @@ +"""Public API for debugpy.""" + +import socket +import struct +import sys +from .common.constants import DEFAULT_HOST, DEFAULT_PORT +from .common.stream_transport import StreamTransport +from .server.debug_session import DebugSession, RestartRequest + +_debug_session = None +# Bound-but-not-yet-accepted socket, held between listen() and the accept that +# wait_for_client() performs. +_listener = None +# Set by enable_restart() before a session exists, because the capability it +# controls is answered during the `initialize` that starts one. +_restart_supported = False + + +def listen(port=DEFAULT_PORT, host=DEFAULT_HOST): + """Bind a listening socket and return the address it is bound to. + + Returns as soon as the socket is bound, WITHOUT waiting for a client, so + the caller can publish the endpoint that a client then connects to. The + accept and the `initialize` handshake happen in `wait_for_client()`. This + matches CPython debugpy, where `listen()` reports the endpoint and + `wait_for_client()` blocks. + + Args: + port: Port number to listen on, or 0 to let the system choose + (default: 5678) + host: Host address to bind to (default: "127.0.0.1") + + Returns: + (host, port) tuple of the actual bound address + """ + global _listener + + if _listener is not None or _debug_session is not None: + raise RuntimeError("Already listening for debugger") + + # Create listening socket + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + except: + pass # Not supported in MicroPython + + # Use getaddrinfo for MicroPython compatibility + addr_info = socket.getaddrinfo(host, port) + addr = addr_info[0][-1] # Get the sockaddr + listener.bind(addr) + listener.listen(1) + + # Resolve the actual bound port (needed when the caller asked for port 0 / + # auto). Not every MicroPython port implements getsockname(). + requested_port = port + try: + bound_addr = listener.getsockname() + if isinstance(bound_addr, (tuple, list)) and len(bound_addr) >= 2: + port = bound_addr[1] + except Exception: + pass + if requested_port == 0 and port == 0: + # Callers act on the endpoint this returns, so reporting a port + # nothing can connect to would be worse than refusing: substituting + # DEFAULT_PORT here would advertise an address the socket is not + # bound to. Ask for an explicit port on a target whose getsockname() + # cannot report the OS-assigned one. + listener.close() + raise OSError( + "port=0 needs getsockname() to report the assigned port, which " + "this target does not implement; pass an explicit port" + ) + + _listener = listener + print(f"Debugpy listening on {host}:{port}") + return (host, port) + + +def listen_stream(reader, writer=None, is_connected=None): + """Start a debug session directly on an already-open stream, no TCP. + + For a board with a second CDC interface dedicated to DAP: `reader`/`writer` + are that interface's stream (the same object for both if it is one + read/write file, as `open(dev, "r+b")` gives on the unix port). Unlike + `listen()`, the stream is already connected - there is no bind/accept + step, so `wait_for_client()` goes straight to the initialize/ + configurationDone handshake. + + `is_connected` is how a stream that cannot reach EOF says the host has + gone, and belongs to the caller because it is port-specific; see + `StreamTransport._peer_gone`. A stream with a real EOF needs none. + """ + global _listener + + if _listener is not None or _debug_session is not None: + raise RuntimeError("Already listening for debugger") + + _listener = StreamTransport(reader, writer, is_connected) + print("Debugpy listening on stream") + return _listener + + +def _accept_and_initialize(): + """Accept the pending connection and handle the client's `initialize`. + + Split out of `listen()` so the endpoint can be published before a client + exists. Returns True once a session is ready. + """ + global _debug_session, _listener + + if _listener is None: + print("[DAP] no listening socket; call listen() first") + return False + + listener, _listener = _listener, None + # A stream transport (listen_stream()) is already connected - there is no + # separate client socket to accept, and no separate listener to close off + # once accepted (it IS the connection). + is_stream = isinstance(listener, StreamTransport) + client_sock = None + try: + if is_stream: + client_sock = listener + else: + client_sock, client_addr = listener.accept() + print(f"Debugger connected from {format_client_addr(client_addr)}") + + _debug_session = DebugSession(client_sock, _restart_supported) + + print("[DAP] Waiting for initialize request...") + # `recv_message()` answers None both for "not a whole message yet" and + # for "the channel is gone", and against a blocking channel the first + # of those is what a message split across reads looks like. Retrying + # while the channel is open tells them apart, and stops a split + # `initialize` from being read as a client that sent something else. + init_message = None + while init_message is None and not _debug_session.channel.closed: + init_message = _debug_session.channel.recv_message() + if init_message is None: + print("[DAP] Connection closed before the initialize request") + _debug_session = None + client_sock.close() + return False + if init_message.get("command") == "initialize": + _debug_session._handle_message(init_message) + print("[DAP] Initialize request handled - returning control immediately") + else: + print(f"[DAP] Warning: Expected initialize, got {init_message}") + + # Set socket to non-blocking for subsequent message processing + _debug_session.channel.sock.settimeout(0.001) + + print("[DAP] Debug session ready - all other messages will be handled in trace function") + return True + + except Exception as e: + print(f"[DAP] Connection error: {e}") + if client_sock is not None: + client_sock.close() + _debug_session = None + return False + finally: + # This is a single-connection server, so stop listening once the + # client is accepted. Not for a stream transport: it IS the client + # socket, still needed by the session this just started. + if not is_stream: + listener.close() + + +def format_client_addr(client_addr): + """Format client address using socket module methods""" + if isinstance(client_addr, (tuple, list)): + # Already in (ip, port) format + return f"{client_addr[0]}:{client_addr[1]}" + elif isinstance(client_addr, bytes) and len(client_addr) >= 8: + # Extract port (bytes 2-4, network byte order) + port = struct.unpack("!H", client_addr[2:4])[0] + # Extract IP address (bytes 4-8) using inet_ntoa + ip_packed = client_addr[4:8] + try: + # inet_ntoa expects 4-byte string in network byte order + ip_addr = socket.inet_ntoa(ip_packed) + return f"{ip_addr}:{port}" + except: + # Fallback if inet_ntoa not available (MicroPython) + ip_addr = ".".join(str(b) for b in ip_packed) + return f"{ip_addr}:{port}" + else: + return str(client_addr) + + +def wait_for_client(timeout_s=None): + """Block until a client has attached and finished configuring. + + Accepts the connection and handles `initialize` (both deferred by + `listen()` so the endpoint can be published first), then waits for + `configurationDone`. Breakpoints the client sets before then are honoured + because this drains the socket the whole time it waits. Returns True once + configurationDone arrives, False after a bounded timeout (logged, not + silent) or if nothing is listening. + """ + global _debug_session + if _debug_session is None and not _accept_and_initialize(): + print("[DAP] wait_for_client: nothing is listening, nothing to wait for") + return False + if timeout_s is None: + return _debug_session.wait_for_client() + return _debug_session.wait_for_client(timeout_s) + + +def enable_restart(): + """Declare that this process can re-run its target, so `restart` is offered. + + Must be called before `wait_for_client()`: the capability is answered in the + `initialize` request, which that accepts and handles. Only the code that + owns the run loop can honour a restart, so nothing else turns this on. + """ + global _restart_supported + _restart_supported = True + if _debug_session is not None: + _debug_session.restart_supported = True + + +def wait_for_restart(): + """Block between runs until a restart is requested, or the client leaves. + + Returns True if the target should be run again, False if there is no client + left to run it for. The caller must not be traced while it waits (see + DebugSession.wait_for_restart). + """ + if _debug_session is None: + return False + return _debug_session.wait_for_restart() + + +def console(text): + """Show `text` in the client's debug console, if a session is connected. + + A no-op with no session, so a target can report progress unconditionally + without caring whether it is being debugged. See DebugSession.console for + why anything worth saying goes here as well as to stdout. + """ + if _debug_session is not None: + _debug_session.console(text) + + +def get_capabilities(): + """Return the firmware capability dict (settrace/save_names/set_local/f_back). + + Uses the active session's probe result if a session exists, otherwise + probes directly. Every value comes from probing the running interpreter, + never from a build or variant name. Which channel a session took is not + among them - that is a property of the run, and the boot script reports + it - so this answers the same whenever it is called. + """ + global _debug_session + if _debug_session is not None: + return _debug_session.capabilities + return DebugSession.probe_capabilities() + + +def breakpoint(): + """Trigger a breakpoint in the debugger.""" + global _debug_session + if _debug_session: + _debug_session.trigger_breakpoint() + else: + # Fallback to built-in breakpoint if available + if hasattr(__builtins__, "breakpoint"): + __builtins__.breakpoint() + + +def debug_this_thread(): + """Enable debugging for the current thread.""" + global _debug_session + if _debug_session: + _debug_session.debug_this_thread() + else: + # Install trace function even if no session yet + if hasattr(sys, "settrace"): + sys.settrace(_default_trace_func) + else: + raise RuntimeError("MICROPY_PY_SYS_SETTRACE required") + + +def _default_trace_func(frame, event, arg): + """Default trace function when no debug session is active.""" + # Just return None to continue execution + return None + + +def is_client_connected(): + """Check if a debugger client is connected.""" + global _debug_session + return _debug_session is not None and _debug_session.is_connected() + + +def disconnect(): + """Disconnect from the debugger client.""" + global _debug_session + if _debug_session: + _debug_session.disconnect() + _debug_session = None diff --git a/python-ecosys/debugpy/debugpy/server/__init__.py b/python-ecosys/debugpy/debugpy/server/__init__.py new file mode 100644 index 000000000..1ab7a0ff5 --- /dev/null +++ b/python-ecosys/debugpy/debugpy/server/__init__.py @@ -0,0 +1 @@ +# Debug server components diff --git a/python-ecosys/debugpy/debugpy/server/debug_session.py b/python-ecosys/debugpy/debugpy/server/debug_session.py new file mode 100644 index 000000000..f9873b04f --- /dev/null +++ b/python-ecosys/debugpy/debugpy/server/debug_session.py @@ -0,0 +1,816 @@ +"""Main debug session handling DAP protocol communication.""" + +import sys +import time + +from ..common.constants import ( + CMD_ATTACH, + CMD_CONFIGURATION_DONE, + CMD_CONTINUE, + CMD_DISCONNECT, + CMD_EVALUATE, + CMD_INITIALIZE, + CMD_LAUNCH, + CMD_NEXT, + CMD_PAUSE, + CMD_RESTART, + CMD_SCOPES, + CMD_SET_BREAKPOINTS, + CMD_SET_VARIABLE, + CMD_SOURCE, + CMD_STACK_TRACE, + CMD_STEP_IN, + CMD_STEP_OUT, + CMD_THREADS, + CMD_VARIABLES, + EVENT_CONTINUED, + EVENT_INITIALIZED, + EVENT_OUTPUT, + EVENT_STOPPED, + EVENT_TERMINATED, + STOP_REASON_BREAKPOINT, + STOP_REASON_PAUSE, + STOP_REASON_STEP, + TRACE_CALL, + TRACE_EXCEPTION, + TRACE_LINE, + TRACE_RETURN, + WAIT_FOR_CLIENT_TIMEOUT_S, +) +from ..common.messaging import JsonMessageChannel +from .pdb_adapter import PdbAdapter + + +class RestartRequest(BaseException): + """Raised inside the debugged program to unwind it back to its launcher. + + A restart cannot wait for the target to return - the ordinary embedded + shape is a main loop that never does - and nothing else on the device can + interrupt it: there is no second thread, and the DAP message pump runs + inside the trace function, where raising would kill the program with an + error belonging to the debug channel. A deliberate raise from the trace + function is the one remaining mechanism, and it is how CPython debuggers + unwind a target too. + + Derived from BaseException so a target's own `except Exception` does not + swallow the unwind and leave the restart silently undone. A target that + catches BaseException (or uses a bare `except:`) can still swallow it; + there is no mechanism behind that, only documentation. + """ + + +def _is_placeholder_local_name(name): + """True if `name` is a positional `local_NN` placeholder, not a real name. + + Without MICROPY_PY_SYS_SETTRACE_LOCALNAMES, frame.f_locals synthesizes + names as `local_00`, `local_01`, ... (`local_%02d` in py/profile.c). The + digit test below stays width-agnostic so a build that numbers them + differently is still recognised; this is the only reliable signal that + separates the two cases at runtime. + """ + if not name.startswith("local_"): + return False + return name[len("local_") :].isdigit() + + +# Compiled by the running firmware at probe time; see _probe_local_names. +_LOCAL_NAMES_PROBE_SRC = """ +def _probe(): + _mpdbg_probe_local = 1 + return list(sys._getframe().f_locals.keys()) +""" + + +def _probe_local_names(frame): + """Local names as seen in a frame the *running firmware* compiled. + + Names are attached to a code object when that object is compiled, so + reading this module's own frame measures whichever compiler produced + this module, not the firmware. Those differ whenever debugpy is + installed cross-compiled: mpy-cross only persists names into .mpy with + MICROPY_PY_SYS_SETTRACE_LOCALNAMES_PERSIST, which is off by default (it + corrupts line numbers), so an .mpy install always reports placeholders + however the firmware was built. + + Compiling a throwaway function here measures the firmware's own + compiler, which is what the `save_names` capability claims. A firmware + without `exec` cannot compile source at all, so there `frame` - the + caller's own - is the only frame available and the honest answer. + """ + namespace = {"sys": sys} + try: + exec(_LOCAL_NAMES_PROBE_SRC, namespace) + except Exception: + return list(frame.f_locals.keys()) + return namespace["_probe"]() + + +# Filled by DebugSession.probe_capabilities() on its first call; the values +# describe the firmware, which does not change while it is running. +_CAPABILITIES = None + + +class DebugSession: + """Manages a debugging session with a DAP client.""" + + def __init__(self, client_socket, restart_supported=False): + self.debug_logging = False # Initialize first + self.channel = JsonMessageChannel(client_socket, self._debug_print) + self.pdb = PdbAdapter() + # Lets the adapter pump DAP messages while it waits. + self.pdb._debug_session = self # type: ignore[assignment] + self.initialized = False + self.connected = True + self.thread_id = 1 # Simple single-thread model + self.configuration_done = False + self._pumping = False + # Whether the launcher can actually re-run the target. Only it knows, + # so a session never assumes it: `restart` is refused, and + # supportsRestartRequest not advertised, unless it was told otherwise. + self.restart_supported = restart_supported + self.restart_requested = False + # Probed once at session start; never inferred from a build/variant name. + self.capabilities = self.probe_capabilities() + self.pdb.capabilities = self.capabilities + + def _debug_print(self, message): + """Print debug message only if debug logging is enabled.""" + if self.debug_logging: + print(message) + + @property + def _baremetal(self) -> bool: + return sys.platform not in ("linux") # to be expanded + + @staticmethod + def probe_capabilities(): + """Probe what the running firmware actually supports. + + Returns a dict with at least `settrace`, `save_names`, `set_local` + and `f_back`, each derived by exercising the real interpreter - + never by reading a build/variant name, which does not reliably + reflect what a given firmware image supports (see BACKGROUND.md). + Which channel a session took is not in here: that is a fact about + the run rather than about the firmware, and the boot script reports + it (`caps["repl_dap"]`). Safe to call on both the unix port and + bare-metal builds; never raises. + + Probed once per interpreter and remembered: every value describes the + firmware, which cannot change while it is running, and the probe is + not free - `_probe_local_names` compiles source on the device. A copy + is handed out so a caller adding its own key (the boot script adds + `repl_dap`) cannot reach the cache. + + `save_names` is measured on code the firmware compiles here rather + than on this module's own frame, so it reports the firmware and not + how debugpy itself was deployed (see _probe_local_names). + """ + global _CAPABILITIES + if _CAPABILITIES is not None: + return dict(_CAPABILITIES) + caps = { + "settrace": hasattr(sys, "settrace"), + "f_back": False, + "save_names": False, + "set_local": False, + } + if not caps["settrace"]: + _CAPABILITIES = caps + return dict(caps) + + try: + frame = sys._getframe() + except Exception: + _CAPABILITIES = caps + return dict(caps) + + try: + caps["f_back"] = hasattr(frame, "f_back") + except Exception: + pass + + try: + caps["set_local"] = hasattr(frame, "_set_local") + except Exception: + pass + + try: + local_names = _probe_local_names(frame) + # An empty locals dict (e.g. probing from module scope) proves + # nothing either way; only trust the signal when there is at + # least one local name to inspect for the placeholder pattern. + caps["save_names"] = bool(local_names) and not any( + _is_placeholder_local_name(n) for n in local_names + ) + except Exception: + pass + + _CAPABILITIES = caps + return dict(caps) + + def start(self): + """Start the debug session message loop.""" + try: + while self.connected and not self.channel.closed: + message = self.channel.recv_message() + if message is None: + break + + self._handle_message(message) + + except Exception as e: + print(f"Debug session error: {e}") + finally: + self.disconnect() + + def initialize_connection(self): + """Initialize the connection - handle just the essential initial messages then return.""" + # Note: debug_logging not available yet during init, so we always show these messages + print("[DAP] Processing initial DAP messages...") + + try: + # Process initial messages quickly and return control to main thread + # We'll handle ongoing messages in the trace function + attached = False + message_count = 0 + max_init_messages = 6 # Just handle the first few essential messages + + while message_count < max_init_messages and not attached: + try: + # Short timeout - don't block the main thread for long + self.channel.sock.settimeout(1.0) + message = self.channel.recv_message() + if message is None: + print("[DAP] No more messages in initial batch") + break + + print(f"[DAP] Initial message #{message_count + 1}: {message.get('command')}") + self._handle_message(message) + message_count += 1 + + # Just wait for attach, then we can return control + if message.get("command") == "attach": + attached = True + print("[DAP] ✅ Attach received - returning control to main thread") + break + + except Exception as e: + print(f"[DAP] Exception in initial processing: {e}") + break + finally: + self.channel.sock.settimeout(None) + + # After attach, continue processing a few more messages quickly + if attached: + self._debug_print("[DAP] Processing remaining setup messages...") + additional_count = 0 + while additional_count < 4: # Just a few more + try: + self.channel.sock.settimeout(0.5) # Short timeout + message = self.channel.recv_message() + if message is None: + break + self._debug_print(f"[DAP] Setup message: {message.get('command')}") + self._handle_message(message) + additional_count += 1 + except: + break + finally: + self.channel.sock.settimeout(None) + + print("[DAP] Initial setup complete - main thread can continue") + + except Exception as e: + print(f"[DAP] Initialization error: {e}") + + def process_pending_messages(self): + """Process any pending DAP messages without blocking. + + Not re-entered: the trace function calls this on entry to every new + frame, so handling a message here can call it again. A nested call + must not touch the socket timeout, because its `finally` would put the + socket back into blocking mode underneath the outer loop, whose next + recv() then waits for a message the client will not send until it has + seen an event this loop is what produces. MicroPython sockets have no + gettimeout(), so the nesting is tracked rather than the timeout saved. + + Nothing here may raise: every caller is `_trace_function`, so an + exception escaping this method lands in whichever line of the + debugged program was being traced and kills that program with an + errno belonging to the debug channel, not to anything the program + did. + """ + if self._pumping: + return + self._pumping = True + try: + # Set socket to non-blocking mode for message processing + self.channel.sock.settimeout(0.001) # Very short timeout + + while True: + message = self.channel.recv_message() + if message is None: + break + self._handle_message(message) + + except Exception: + # No messages available or socket error + pass + finally: + self._pumping = False + # Reset to blocking mode - but only against a channel that is + # still there. Restoring the timeout is a socket operation like + # any other and fails on a closed socket, and the loop above is + # exactly what closes it: a `disconnect` request handled there + # runs the whole session teardown, so on the way out of that + # request this socket is already gone. A client that vanishes + # without sending `disconnect` reaches the same place without + # setting the flag, hence the guard as well as the check. + # Either way the session is over, so end it here rather than + # leaving a trace function installed that pumps a dead channel. + if self.channel.closed: + return + try: + self.channel.sock.settimeout(None) + except Exception: + self.disconnect() + + def _handle_message(self, message): + """Handle incoming DAP messages.""" + msg_type = message.get("type") + command = message.get("command", message.get("event", "unknown")) + seq = message.get("seq", 0) + + self._debug_print(f"[DAP] RECV: {msg_type} {command} (seq={seq})") + if message.get("arguments"): + self._debug_print(f"[DAP] args: {message['arguments']}") + + if msg_type == "request": + self._handle_request(message) + elif msg_type == "response": + # We don't expect responses from client + self._debug_print(f"[DAP] Unexpected response from client: {message}") + elif msg_type == "event": + # We don't expect events from client + self._debug_print(f"[DAP] Unexpected event from client: {message}") + + def _handle_request(self, message): + """Handle DAP request messages.""" + command = message.get("command") + seq = message.get("seq", 0) + args = message.get("arguments", {}) + + try: + if command == CMD_INITIALIZE: + self._handle_initialize(seq, args) + elif command == CMD_LAUNCH: + self._handle_launch(seq, args) + elif command == CMD_ATTACH: + self._handle_attach(seq, args) + elif command == CMD_SET_BREAKPOINTS: + self._handle_set_breakpoints(seq, args) + elif command == CMD_CONTINUE: + self._handle_continue(seq, args) + elif command == CMD_NEXT: + self._handle_next(seq, args) + elif command == CMD_STEP_IN: + self._handle_step_in(seq, args) + elif command == CMD_STEP_OUT: + self._handle_step_out(seq, args) + elif command == CMD_PAUSE: + self._handle_pause(seq, args) + elif command == CMD_STACK_TRACE: + self._handle_stack_trace(seq, args) + elif command == CMD_SCOPES: + self._handle_scopes(seq, args) + elif command == CMD_VARIABLES: + self._handle_variables(seq, args) + elif command == CMD_SET_VARIABLE: + self._handle_set_variable(seq, args) + elif command == CMD_EVALUATE: + self._handle_evaluate(seq, args) + elif command == CMD_RESTART: + self._handle_restart(seq, args) + elif command == CMD_DISCONNECT: + self._handle_disconnect(seq, args) + elif command == CMD_CONFIGURATION_DONE: + self._handle_configuration_done(seq, args) + elif command == CMD_THREADS: + self._handle_threads(seq, args) + elif command == CMD_SOURCE: + self._handle_source(seq, args) + else: + self.channel.send_response( + command, seq, success=False, message=f"Unknown command: {command}" + ) + + except Exception as e: + self.channel.send_response(command, seq, success=False, message=str(e)) + + def _handle_initialize(self, seq, args): + """Handle initialize request.""" + capabilities = { + "supportsConfigurationDoneRequest": True, + "supportsEvaluateForHovers": True, + "supportTerminateDebuggee": True, + "supportSuspendDebuggee": True, + "supportsTerminateRequest": True, + "supportsSetVariable": True, + # "supportsFunctionBreakpoints": False, + # "supportsConditionalBreakpoints": False, + # "supportsHitConditionalBreakpoints": False, + # "supportsStepBack": False, + # "supportsRestartFrame": False, + # "supportsGotoTargetsRequest": False, + # "supportsStepInTargetsRequest": False, + # "supportsCompletionsRequest": False, + # "supportsModulesRequest": False, + # "additionalModuleColumns": [], + # "supportedChecksumAlgorithms": [], + # Advertised only when the launcher runs the target in a loop it + # can re-enter; a client that sees this offers a restart button + # and expects the debuggee to come back, not the session to end. + "supportsRestartRequest": self.restart_supported, + # "supportsExceptionOptions": False, + # "supportsValueFormattingOptions": False, + # "supportsExceptionInfoRequest": False, + # "supportsDelayedStackTraceLoading": False, + # "supportsLoadedSourcesRequest": False, + # "supportsLogPoints": False, + # "supportsTerminateThreadsRequest": False, + # "supportsSetExpression": False, + # "supportsDataBreakpoints": False, + # "supportsReadMemoryRequest": False, + # "supportsWriteMemoryRequest": False, + # "supportsDisassembleRequest": False, + # "supportsCancelRequest": False, + # "supportsBreakpointLocationsRequest": False, + # "supportsClipboardContext": False, + } + + self.channel.send_response(CMD_INITIALIZE, seq, body=capabilities) + self.channel.send_event(EVENT_INITIALIZED) + self.initialized = True + + def _handle_launch(self, seq, args): + """Handle launch request.""" + # For attach-mode debugging, we don't need to launch anything + self.channel.send_response(CMD_LAUNCH, seq) + + def _handle_attach(self, seq, args): + """Handle attach request.""" + # Check if debug logging should be enabled + self.debug_logging = args.get("logToFile", False) + + self._debug_print(f"[DAP] Processing attach request with args: {args}") + print( + f"[DAP] Debug logging {'enabled' if self.debug_logging else 'disabled'} (logToFile={self.debug_logging})" + ) + + # get debugger root and debuggee root from pathMappings + for pm in args.get("pathMappings", []): + # debuggee - debugger. Trailing slashes are stripped so "/remote" + # and "/remote/" name the same root: pdb_adapter's translation + # matches a mapping on a path-separator boundary it adds itself, + # and a root that already carries one would double it up. + remote_root = pm.get("remoteRoot", "./").rstrip("/") + local_root = pm.get("localRoot", "./").rstrip("/") + self.pdb.path_mappings.append((remote_root, local_root)) + # # TODO: justMyCode, debugOptions , + + # Enable trace function + self.pdb.set_trace_function(self._trace_function) + self.channel.send_response(CMD_ATTACH, seq) + + # After successful attach, we might need to send additional events + # Some debuggers expect a 'process' event or thread events + self._debug_print("[DAP] Attach completed, debugging is now active") + + def _handle_set_breakpoints(self, seq, args): + """Handle setBreakpoints request.""" + source = args.get("source", {}) + filename = source.get("path", "") + breakpoints = args.get("breakpoints", []) + + # Debug log the source information + self._debug_print(f"[DAP] setBreakpoints source info: {source}") + + # Set breakpoints in pdb adapter + actual_breakpoints = self.pdb.set_breakpoints(filename, breakpoints) + + self.channel.send_response( + CMD_SET_BREAKPOINTS, seq, body={"breakpoints": actual_breakpoints} + ) + + def _handle_continue(self, seq, args): + """Handle continue request.""" + self.pdb.continue_execution() + self.channel.send_response(CMD_CONTINUE, seq) + + def _handle_next(self, seq, args): + """Handle next (step over) request.""" + self.pdb.step_over() + self.channel.send_response(CMD_NEXT, seq) + + def _handle_step_in(self, seq, args): + """Handle stepIn request.""" + self.pdb.step_into() + self.channel.send_response(CMD_STEP_IN, seq) + + def _handle_step_out(self, seq, args): + """Handle stepOut request.""" + self.pdb.step_out() + self.channel.send_response(CMD_STEP_OUT, seq) + + def _handle_pause(self, seq, args): + """Handle pause request. + + The response says the request was accepted, not that the target has + stopped; `stopped` follows when the trace function consumes it. + """ + self.pdb.pause() + self.channel.send_response(CMD_PAUSE, seq) + + def _handle_stack_trace(self, seq, args): + """Handle stackTrace request.""" + stack_frames = self.pdb.get_stack_trace() + self.channel.send_response( + CMD_STACK_TRACE, + seq, + body={"stackFrames": stack_frames, "totalFrames": len(stack_frames)}, + ) + + def _handle_scopes(self, seq, args): + """Handle scopes request.""" + frame_id = args.get("frameId", 0) + self._debug_print(f"[DAP] Processing scopes request for frameId={frame_id}") + scopes = self.pdb.get_scopes(frame_id) + self._debug_print(f"[DAP] Generated scopes: {scopes}") + self.channel.send_response(CMD_SCOPES, seq, body={"scopes": scopes}) + + def _handle_variables(self, seq, args): + """Handle variables request.""" + variables_ref = args.get("variablesReference", 0) + variables = self.pdb.get_variables(variables_ref) + self.channel.send_response(CMD_VARIABLES, seq, body={"variables": variables}) + + def _handle_set_variable(self, seq, args): + """Handle setVariable request.""" + variables_ref = args.get("variablesReference", 0) + name = args.get("name", "") + value = args.get("value", "") + + if not name: + self.channel.send_response( + CMD_SET_VARIABLE, seq, success=False, message="No variable name provided" + ) + return + + self._debug_print( + f"[DAP] Processing setVariable request: name={name}, value={value}, ref={variables_ref}" + ) + + try: + updated_variable = self.pdb.set_variable(variables_ref, name, value) + self.channel.send_response(CMD_SET_VARIABLE, seq, body=updated_variable) + except Exception as e: + self.channel.send_response(CMD_SET_VARIABLE, seq, success=False, message=str(e)) + + def _handle_evaluate(self, seq, args): + """Handle evaluate request. + + `context` selects the contract PdbAdapter.evaluate_expression applies: + `repl`/`clipboard` (Debug Console, "Copy as Expression") may execute a + statement when `expression` isn't a valid expression; `watch`/`hover` + and any other or absent context stay read-only eval. + """ + expression = args.get("expression", "") + frame_id = args.get("frameId") + context = args.get("context", "watch") + if not expression: + self.channel.send_response( + CMD_EVALUATE, seq, success=False, message="No expression provided" + ) + return + try: + result = self.pdb.evaluate_expression(expression, frame_id, context) + self.channel.send_response( + CMD_EVALUATE, seq, body={"result": str(result), "variablesReference": 0} + ) + except Exception as e: + self.channel.send_response(CMD_EVALUATE, seq, success=False, message=str(e)) + + def _handle_restart(self, seq, args): + """Handle restart request: unwind the target so the launcher re-runs it. + + The session outlives the restart deliberately. Breakpoints live in + `self.pdb`, and a client that sent `restart` rather than reconnecting + does not re-send them, so keeping the one session alive is what makes + them still bind on the next run - and it costs no re-attach round trip. + + Only the flag is set here. Whatever the target is doing, it is doing it + somewhere below this call: this runs inside the trace function, so the + unwind happens where that returns to, not here. + """ + if not self.restart_supported: + self.channel.send_response( + CMD_RESTART, + seq, + success=False, + message="the target was not launched in a loop that can re-run it", + ) + return + + self.restart_requested = True + # A target stopped at a breakpoint is inside wait_for_continue(); it has + # to be let go before it can be unwound. A pending step or pause is + # cleared with it, so neither stops the target again on its way out. + self.pdb.step_mode = None + self.pdb.paused = False + self.pdb.continue_event = True + + self.channel.send_response(CMD_RESTART, seq) + # The client last heard `stopped`; without this its UI stays stopped on + # a frame that is about to cease to exist. + self.channel.send_event(EVENT_CONTINUED, threadId=self.thread_id, allThreadsContinued=True) + + def _raise_if_restarting(self): + """Unwind the target if a restart arrived, clearing the request. + + Cleared here, not by the launcher, so the exception itself is the whole + signal: one restart unwinds one run, and a second request during the + unwind is a fresh one rather than a repeat of this. + """ + if self.restart_requested: + self.restart_requested = False + raise RestartRequest + + def console(self, text): + """Show `text` in the client's debug console (a DAP `output` event). + + The only route a target's own notes have to the user on a transport + where device stdout never reaches the host: a mounted serial session's + filesystem pump discards everything the device prints. Run-boundary + markers go through here as well as to stdout so they are visible on + every transport, not just the ones with a readable console. + """ + self.channel.send_event(EVENT_OUTPUT, category="console", output=text) + + def wait_for_restart(self): + """Pump DAP messages between runs, until a restart or the client leaves. + + A target that returned normally is no longer generating trace events, so + nothing would service the socket and a restart request would sit unread + in it. Returns True if a restart arrived, False if the client went away, + in which case there is nothing left to restart for. + + `terminated` is deliberately not what marks the end of a run: a client + that sees it tears the session down, which is the opposite of what loop + mode exists for. An `output` event says the same thing without ending + anything, and is the only notice a client gets that the program has + finished and a restart is what comes next. + + Callers must not be traced while they wait here: a restart handled by + the pump below would otherwise unwind the caller itself. + """ + self.console("Target finished; waiting for a restart request.\n") + while True: + if self.restart_requested: + self.restart_requested = False + return True + if not self.connected or self.channel.closed: + return False + self.process_pending_messages() + time.sleep(0.01) + + def _handle_disconnect(self, seq, args): + """Handle disconnect request.""" + self.channel.send_response(CMD_DISCONNECT, seq) + self.disconnect() + + def _handle_configuration_done(self, seq, args): + """Handle configurationDone request.""" + # This indicates that the client has finished configuring breakpoints + # and is ready to start debugging + self.configuration_done = True + self.channel.send_response(CMD_CONFIGURATION_DONE, seq) + + def _handle_threads(self, seq, args): + """Handle threads request.""" + # MicroPython is single-threaded, so return one thread + threads = [{"id": self.thread_id, "name": "main"}] + self.channel.send_response(CMD_THREADS, seq, body={"threads": threads}) + + def _handle_source(self, seq, args): + """Handle source request.""" + source = args.get("source", {}) + source_path = source.get("path", "") + if self._baremetal or not source_path: + # BUG: unable to read the source on ESP32 + # Possible an effect of the import / initialization sequence ? + # Note that other source files ( other.py) do not seem to get requested in the same way + self.channel.send_response(CMD_SOURCE, seq, success=False) + return + self._debug_print(f"[DAP] Processing source request for path: {source}") + try: + # Try to read the source file + with open(source_path) as f: + content = f.read() + self.channel.send_response(CMD_SOURCE, seq, body={"content": content}) + except Exception: + self.channel.send_response( + CMD_SOURCE, + seq, + success=False, + message="cancelled", + # message=f"Could not read source: {e}" + ) + + def _trace_function(self, frame, event: str, arg): + """Trace function called by sys.settrace.""" + # https://docs.python.org/3/library/sys.html#sys.settrace + global _twiddel + # Process any pending DAP messages frequently + + self.process_pending_messages() + # Before any breakpoint work: a restart that arrived above wants this + # program gone, not stopped somewhere else on the way out. + self._raise_if_restarting() + # Handle breakpoints, pauses and stepping. The reason comes from the + # adapter because the adapter is what decided: a stop is a breakpoint, + # a consumed pause, or a landed step, and those are the only three. + if self.pdb.should_stop(frame, event, arg): + self._send_stopped_event( + STOP_REASON_BREAKPOINT + if self.pdb.hit_breakpoint + else STOP_REASON_PAUSE + if self.pdb.hit_pause + else STOP_REASON_STEP + ) + # Wait for continue command + self.pdb.wait_for_continue() + # A restart is one of the things that ends that wait. + self._raise_if_restarting() + + # The trace function is invoked (with event set to 'call') whenever a new local scope is entered; + # it should return a reference to a local trace function to be used for the new scope, + # or None if the scope shouldn't be traced. + + return self._trace_function + + def _send_stopped_event(self, reason): + """Send stopped event to client.""" + self.channel.send_event( + EVENT_STOPPED, reason=reason, threadId=self.thread_id, allThreadsStopped=True + ) + + def wait_for_client(self, timeout_s=WAIT_FOR_CLIENT_TIMEOUT_S): + """Block until the client has sent configurationDone, or time out. + + Same busy-poll shape as PdbAdapter.wait_for_continue(): there is no + server thread, so nothing services the socket unless this loop drains + it. Replaces a fixed sleep with a deterministic handshake - breakpoints + set before configurationDone are already applied by the time this + returns because process_pending_messages() has drained them too. + Returns True once configurationDone arrives, False if the bounded + timeout elapses first (a hard failure is worse than continuing with a + clear log message: a client that never configures is a client bug or + a dropped connection, not something to hang on forever). + """ + start = time.ticks_ms() + while not self.configuration_done: + self.process_pending_messages() + if not self.connected or self.channel.closed: + print("[DAP] wait_for_client: connection closed before configurationDone") + return False + if time.ticks_diff(time.ticks_ms(), start) > timeout_s * 1000: + print( + "[DAP] wait_for_client: timed out after {}s waiting for configurationDone".format( + timeout_s + ) + ) + return False + time.sleep(0.01) + return True + + def trigger_breakpoint(self): + """Trigger a manual breakpoint.""" + if self.initialized: + self._send_stopped_event(STOP_REASON_BREAKPOINT) + + def debug_this_thread(self): + """Enable debugging for current thread.""" + if hasattr(sys, "settrace"): + sys.settrace(self._trace_function) + + def is_connected(self): + """Check if client is connected.""" + return self.connected and not self.channel.closed + + def disconnect(self): + """Disconnect from client.""" + self.connected = False + if hasattr(sys, "settrace"): + sys.settrace(None) + self.pdb.cleanup() + self.channel.close() diff --git a/python-ecosys/debugpy/debugpy/server/pdb_adapter.py b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py new file mode 100644 index 000000000..da8118345 --- /dev/null +++ b/python-ecosys/debugpy/debugpy/server/pdb_adapter.py @@ -0,0 +1,968 @@ +"""PDB adapter for integrating with MicroPython's trace system.""" + +import os +import sys +import time + +from micropython import const # type: ignore[import-untyped] + +from ..common.constants import ( + SCOPE_GLOBALS, + SCOPE_LOCALS, + STEP_INTO, + STEP_OUT, + STEP_OVER, + TRACE_CALL, + TRACE_EXCEPTION, + TRACE_LINE, + TRACE_RETURN, +) + +Any = object + +VARREF_LOCALS = const(1) +VARREF_GLOBALS = const(2) +VARREF_LOCALS_SPECIAL = const(3) +VARREF_GLOBALS_SPECIAL = const(4) + +# New constants for complex variable references +VARREF_COMPLEX_BASE = const(10000) # Base for complex variable references +MAX_CACHE_SIZE = const(50) # Limit cache size for memory constraints + + +class VariableReferenceCache: + """Lightweight cache for complex variable references optimized for MicroPython.""" + + def __init__(self, max_size: int = MAX_CACHE_SIZE): + self.cache: dict[int, Any] = {} + self.insertion_order: list[int] = [] # Track insertion order for proper FIFO + self.next_ref: int = VARREF_COMPLEX_BASE + self.max_size: int = max_size + + def add_variable(self, value: Any) -> int: + """Add a complex variable and return its reference ID.""" + # Clean cache if approaching limit + if len(self.cache) >= self.max_size: + self._cleanup_oldest() + + ref_id = self.next_ref + self.cache[ref_id] = value + self.insertion_order.append(ref_id) + self.next_ref += 1 + return ref_id + + def get_variable(self, ref_id: int): # -> Optional[Any] + """Get variable by reference ID.""" + return self.cache.get(ref_id) + + def _cleanup_oldest(self) -> None: + """Remove oldest entries to free memory - optimized for MicroPython.""" + if not self.cache or not self.insertion_order: + return + to_remove = max(1, len(self.cache) // 3) + # Direct list slicing is more memory efficient than iteration + keys_to_remove = self.insertion_order[:to_remove] + # Batch delete for efficiency + for key in keys_to_remove: + self.cache.pop(key, None) # Use pop with default to avoid KeyError + # Update insertion order in one operation + self.insertion_order = self.insertion_order[to_remove:] + + def clear(self) -> None: + """Clear all cached variables.""" + self.cache.clear() + self.insertion_order.clear() + + +# Also try checking by basename for path mismatches +def basename(path: str): + return path.split("/")[-1] if "/" in path else path + + +# Check if this might be a relative path match +def ends_with_path(full_path: str, relative_path: str): + """Check if full_path ends with relative_path components.""" + full_parts = full_path.replace("\\", "/").split("/") + rel_parts = relative_path.replace("\\", "/").split("/") + if len(rel_parts) > len(full_parts): + return False + return full_parts[-len(rel_parts) :] == rel_parts + + +# Augmented-assignment operators checked longest-first so e.g. "**=" is not +# mistaken for "*=" followed by stray text. +_AUG_ASSIGN_OPS = ("**=", "//=", ">>=", "<<=", "+=", "-=", "*=", "/=", "%=", "&=", "|=", "^=", "=") + + +def _is_ident_char(ch: str) -> bool: + """True for `[A-Za-z0-9_]` - MicroPython's `str` has no `.isalnum()`.""" + return ch.isalpha() or ch.isdigit() or ch == "_" + + +def _assigned_name(statement: str): + """Return the target name of a simple top-level assignment, or None. + + Recognises only `...` where the identifier is the very + first token and `` is `=` or an augmented-assignment operator. This + is a deliberately narrow, best-effort check - it does NOT catch: + multi-target assignment (`a = b = 1`, only `a` is seen), tuple/list + unpacking (`a, b = 1, 2`), attribute/subscript targets (`obj.x = 1`, + `d[k] = 1`), `def`/`class` statements (which also bind a name), a + `for`/`with ... as` binding, or an assignment that is not the first + statement on the line (e.g. after `;`). Those forms pass through + undetected; callers must treat a `None` result as "not proven safe", + never as "proven no shadowing". + """ + stripped = statement.strip() + if not stripped or stripped[0].isdigit() or not _is_ident_char(stripped[0]): + return None + i = 1 + n = len(stripped) + while i < n and _is_ident_char(stripped[i]): + i += 1 + name = stripped[:i] + rest = stripped[i:].lstrip() + for op in _AUG_ASSIGN_OPS: + if rest.startswith(op): + if op == "=" and rest[1:2] == "=": + return None # `==`, a comparison, not an assignment + return name + return None + + +def _shadowed_local_warning(statement: str, locals_dict): + """Build the honesty-rule warning for `statement`, or None if it doesn't apply. + + Fires when `_assigned_name` recognises a top-level assignment whose + target name is also a key in `locals_dict` (the paused frame's + `f_locals` snapshot): that name is about to be rebound in `f_globals` + only, so the LOCAL of the same name stays exactly as it was. On + firmware without local-name capture (`save_names` capability False), + `locals_dict` keys are synthetic `local_N` placeholders rather than + real identifiers, so a real name can never match and this warning + silently cannot fire there - a known limitation, not a bug. + """ + name = _assigned_name(statement) + if name and name in locals_dict: + return ( + f"Warning: '{name}' also exists as a LOCAL in this frame; " + "the local is unchanged (statement ran against globals only)." + ) + return None + + +class PdbAdapter: + """Adapter between DAP protocol and MicroPython's sys.settrace functionality.""" + + def __init__(self): + self.breakpoints: dict[str, dict[int, dict]] = {} + # filename -> {line_no: breakpoint_info} # todo - simplify + self.current_frame = None + self.step_mode = None # None, 'over', 'into', 'out' + self.step_frame = None + self.step_depth = 0 + self.paused = False # a pause request waiting for the next line event + self.hit_breakpoint = False + self.hit_pause = False + self.continue_event = False + self.variables_cache = {} # frameId -> variables + self.var_cache = VariableReferenceCache() # Enhanced variable reference cache + self.frame_id_counter = 1 + self.path_mappings: list[tuple[str, str]] = [] + # list of [runtime_path -> vscode_path mapping] + self.file_mappings: dict[str, str] = {} + # runtime_path -> vscode_path mapping # todo : merge with .breakpoints + self.capabilities: dict = {} + # set by DebugSession at session start (see DebugSession.probe_capabilities); + # empty dict here means "not yet probed", treated as no set_local support + + def _debug_print(self, message): + """Print debug message only if debug logging is enabled.""" + if hasattr(self, "_debug_session") and self._debug_session.debug_logging: # type: ignore[attr-defined] + print(message) + + def _normalize_path(self, path: str): + """Normalize a file path for consistent comparisons.""" + # Convert to absolute path if possible + try: + if hasattr(os.path, "abspath"): + path = os.path.abspath(path) + elif hasattr(os.path, "realpath"): + path = os.path.realpath(path) + except: + pass + # Ensure consistent separators + path = path.replace("\\", "/") + return path + + def set_trace_function(self, trace_func): + """Install the trace function.""" + if hasattr(sys, "settrace"): + sys.settrace(trace_func) + else: + raise RuntimeError("sys.settrace not available") + + def _filename_as_debugee(self, path: str): + """Translate an IDE-side (vscode) path to the runtime's own path. + + The first mapping whose `vscode_path` names `path` itself or a + directory containing it wins - matched on a path-separator boundary, + not a bare string prefix, so a sibling directory that merely shares + the root's name (`/home/dev/src-old` against root `/home/dev/src`) + is left untranslated instead of being rewritten into a device path + that cannot exist. First-match-wins makes this the exact inverse of + `_filename_as_debugger` below. + """ + # check if we have a 1:1 file mapping for this path + if self.file_mappings.get(path): + return self.file_mappings[path] + for runtime_path, vscode_path in self.path_mappings: + if path == vscode_path or path.startswith(vscode_path + "/"): + path = runtime_path + path[len(vscode_path) :] + if path.startswith("//"): + path = path[1:] + return path + # If no mapping found, return the original path + return path + + def _filename_as_debugger(self, path: str): + """Translate a runtime path (`frame.f_code.co_filename`) to the IDE's path. + + Inverse of `_filename_as_debugee`: same first-match, boundary-aware + rule, applied in the other direction. + """ + path = path or "" + if not path: + return path + if path.startswith("<"): + # Special case for or similar + return path + for runtime_path, vscode_path in self.path_mappings: + if path == runtime_path or path.startswith(runtime_path + "/"): + path = vscode_path + path[len(runtime_path) :] + if path.startswith("//"): + path = path[1:] + return path + # If no mapping found, return the original path + return path + + def set_breakpoints(self, filename: str, breakpoints: list[dict]): + """Replace the breakpoint set for one file. + + DAP sends the whole set for a source on every request, so this + replaces rather than adds, and an empty list is how a client removes + every breakpoint in a file. + + The set is stored under both the path the client used and the name + the debuggee knows the same file by, because `should_stop` matches + whatever `frame.f_code.co_filename` reports. Both are replaced + together: clearing only one leaves the other still armed. + """ + local_name = self._filename_as_debugee(filename) + self.file_mappings[local_name] = filename + self.breakpoints[filename] = {} + self.breakpoints[local_name] = {} + + actual_breakpoints = [] + self._debug_print(f"[PDB] Setting breakpoints for file: {filename} (as {local_name})") + + for bp in breakpoints: + line = bp.get("line") + if line: + self.breakpoints[filename][line] = {} + self.breakpoints[local_name][line] = {} + actual_breakpoints.append( + {"line": line, "verified": True, "source": {"path": filename}} + ) + + self._debug_print(f"[PDB] Breakpoints set : {self.breakpoints}") + + return actual_breakpoints + + def should_stop(self, frame, event: str, arg): + """Determine if execution should stop at this point. + + Returns True for exactly three reasons - a breakpoint, a pending + pause, or a landed step - and records which one in `hit_breakpoint` / + `hit_pause` so the caller can name it without keeping its own copy of + the decision. + """ + # HOT path - no debug printing here + self.current_frame = frame + self.hit_breakpoint = False + self.hit_pause = False + + # Cache frame attributes to reduce lookup overhead + _frame_code = frame.f_code + _filename = _frame_code.co_filename + _lineno = frame.f_lineno + + # Optimize dictionary lookups - use .get() to avoid double lookup + file_breakpoints = self.breakpoints.get(_filename) + if file_breakpoints and _lineno in file_breakpoints: + # Only an event that is about to run this line counts as a hit. + # `return` reports the last line the frame executed, so a breakpoint + # on a function's final line would otherwise stop a second time on + # the way out, on a frame that has already produced its value. + # `call` reports the `def` line, which is the only event that can + # ever match a breakpoint placed there. + if event in (TRACE_CALL, TRACE_LINE): + self._debug_print(f"[PDB] HIT BREAKPOINT (exact match) at {_filename}:{_lineno}") + self.hit_breakpoint = True + return True + else: + # file not (yet) matched - this is slow so we do not want to do this often. + # TODO: use sys.path[] method to find the file, does not work for frozen .... + # if we have a path match , but no breakpoints - add it to the file_mappings dict simplify this check + if file_breakpoints is None: + self.breakpoints[_filename] = {} # Ensure the filename is in the breakpoints dict + if _filename not in self.file_mappings: + self.file_mappings[_filename] = self._filename_as_debugger(_filename) + + # A pause request that arrived while the target was running is consumed + # here. Only on `line`: `call` reports the `def` line before the body + # has run and `return` reports a frame that has already produced its + # value, so neither is a place the user asked to stop at. + if self.paused and event == TRACE_LINE: + self.paused = False + # A pause is a user interrupt; a step that outlived it would fire + # later at a point nobody asked for. + self.step_mode = None + self.hit_pause = True + return True + + # Check stepping + _step_mode = self.step_mode + if _step_mode == STEP_INTO: + if event in (TRACE_CALL, TRACE_LINE): + self.step_mode = None + return True + + elif _step_mode == STEP_OVER: + if event == TRACE_LINE and frame == self.step_frame: + self.step_mode = None + return True + elif event == TRACE_RETURN and frame == self.step_frame: + # Continue stepping in caller + if hasattr(frame, "f_back") and frame.f_back: + self.step_frame = frame.f_back + else: + self.step_mode = None + + elif _step_mode == STEP_OUT: + if event == TRACE_RETURN and frame == self.step_frame: + self.step_mode = None + return True + + return False + + def continue_execution(self): + """Continue execution.""" + self.step_mode = None + self.continue_event = True + + def step_over(self): + """Step over (next line).""" + self.step_mode = "over" + self.step_frame = self.current_frame + self.continue_event = True + + def step_into(self): + """Step into function calls.""" + self.step_mode = "into" + self.continue_event = True + + def step_out(self): + """Step out of current function.""" + self.step_mode = "out" + self.step_frame = self.current_frame + self.continue_event = True + + def pause(self): + """Request a stop at the next line event. + + Nothing stops here: the target is running, and the only code that can + interrupt it is the trace function. `should_stop` consumes this. A + target executing no traced bytecode - blocked in `time.sleep`, inside + a C-level loop, or between runs of a `--loop` session - produces no + trace event, so the request simply stays pending until one comes. + """ + self.paused = True + + def wait_for_continue(self): + """Busy-poll until a continue/step command arrives, or the client is gone. + + No server thread services the socket, so this loop is what drains it + while the target sits stopped. If the channel disappears (bridge + killed, board reset) while stopped, waiting forever would wedge the + target - `sys.settrace(None)` is dropped and the loop exits so the + target resumes and the process/session can end cleanly instead of + requiring a power cycle. + """ + self.continue_event = False + + self._debug_print("[PDB] Waiting for continue command...") + while not self.continue_event: + session = getattr(self, "_debug_session", None) + if session is None: + break + if not session.connected or session.channel.closed: + self._debug_print("[PDB] wait_for_continue: connection lost, resuming target") + if hasattr(sys, "settrace"): + sys.settrace(None) + self.continue_event = True + break + session.process_pending_messages() # type: ignore[arg-type] + time.sleep(0.01) + + # A pause that arrived during the wait is about a target that was + # already stopped. Dropping it here is what keeps it from stopping the + # target again one line after the user's next continue. + self.paused = False + + def get_stack_trace(self): + """Get the current stack trace.""" + if not self.current_frame: + return [] + + frames = [] + frame = self.current_frame + frame_id = 0 + + while frame: + filename = frame.f_code.co_filename + name = frame.f_code.co_name + line = frame.f_lineno + if "" in filename or filename.endswith("debugpy.py"): + hint = "subtle" + else: + hint = "normal" + + # Use the VS Code path if we have a mapping, otherwise use the original path + debugger_path = self._filename_as_debugger(filename) + # Create StackFrame info + frames.append( + { + "id": frame_id, + "name": name, + "source": {"path": debugger_path}, + "line": line, + "column": 1, + "endLine": line, + "endColumn": 1, + "presentationHint": hint, + } + ) + + # Cache frame for variable access + self.variables_cache[frame_id] = frame + + # MicroPython doesn't have f_back attribute + if hasattr(frame, "f_back"): + frame = frame.f_back + else: + # Only return the current frame for MicroPython + break + frame_id += 1 + + return frames + + def get_scopes(self, frame_id): + """Get variable scopes for a frame.""" + scopes = [ + { + "name": SCOPE_LOCALS, + "variablesReference": frame_id * 1000 + VARREF_LOCALS, + "expensive": False, + }, + { + "name": SCOPE_GLOBALS, + "variablesReference": frame_id * 1000 + VARREF_GLOBALS, + "expensive": False, + }, + ] + return scopes + + def _process_special_variables(self, var_dict, read_only=False): + """Process special variables (those starting and ending with __).""" + variables = [] + for name, value in var_dict.items(): + if name.startswith("__") and name.endswith("__"): + try: + # Use lightweight serialization instead of json.dumps + value_str = self._lightweight_serialize(value) + type_str = type(value).__name__ + info = { + "name": name, + "value": value_str, + "type": type_str, + "variablesReference": 0, + } + if read_only: + info["presentationHint"] = {"attributes": ["readOnly"]} + variables.append(info) + except Exception: + variables.append(self._var_error(name)) + return variables + + def _process_regular_variables(self, var_dict, read_only=False): + """Process regular variables (excluding special ones) - optimized.""" + variables = [] + for name, value in var_dict.items(): + # Skip private/internal variables + if name.startswith("__") and name.endswith("__"): + continue + # Use fast path for variable info generation + info = self._get_variable_info_fast(name, value) + if read_only: + info["presentationHint"] = {"attributes": ["readOnly"]} + variables.append(info) + return variables + + def _is_expandable(self, value: Any) -> bool: + """Check if a variable can be expanded (has child elements).""" + return isinstance(value, (dict, list, tuple, set)) + + def _get_preview(self, value: Any, fallback_text: str = "") -> str: + """Get a 30-char preview of a variable value with '...' if truncated - optimized for MicroPython.""" + try: + # Get repr and truncate to exactly 30 chars with "..." if needed + repr_val = repr(value) + if len(repr_val) <= 30: + return repr_val + else: + return repr_val[:30] + "..." + except (TypeError, ValueError, MemoryError): + # Memory-safe fallback + return fallback_text or f"<{type(value).__name__} object>"[:30] + + def _get_variable_info(self, name: str, value: Any) -> dict[str, str | int]: + """Get DAP-compliant variable information with proper type handling.""" + try: + # Handle expandable types + if self._is_expandable(value): + var_ref = self.var_cache.add_variable(value) + preview = self._get_preview(value) # Always use consistent preview + + if isinstance(value, dict): + return { + "name": name, + "value": preview, + "type": "dict", + "variablesReference": var_ref, + "namedVariables": len(value), + "indexedVariables": 0, + } + elif isinstance(value, list): + return { + "name": name, + "value": preview, + "type": "list", + "variablesReference": var_ref, + "indexedVariables": len(value), + "namedVariables": 0, + } + elif isinstance(value, tuple): + return { + "name": name, + "value": preview, + "type": "tuple", + "variablesReference": var_ref, + "indexedVariables": len(value), + "namedVariables": 0, + } + elif isinstance(value, set): + return { + "name": name, + "value": preview, + "type": "set", + "variablesReference": var_ref, + "indexedVariables": len(value), + "namedVariables": 0, + } + + # Simple types - use the preview helper + preview = self._get_preview(value) + + return { + "name": name, + "value": preview, + "type": type(value).__name__, + "variablesReference": 0, + } + except Exception: + return self._var_error(name) + + def _get_variable_info_fast(self, name: str, value: Any) -> dict[str, str | int]: + """Fast path for variable info generation with reduced allocations.""" + try: + # Handle expandable types + if self._is_expandable(value): + var_ref = self.var_cache.add_variable(value) + preview = self._get_preview(value) # Always use consistent preview + + # Use pre-calculated length for better performance + length = 0 + try: + length = len(value) # type: ignore[arg-type] + except: + pass + + # Return optimized structure based on type + if isinstance(value, dict): + return { + "name": name, + "value": preview, + "type": "dict", + "variablesReference": var_ref, + "namedVariables": length if length < 1000 else 1000, # Cap for performance + "indexedVariables": 0, + } + elif isinstance(value, list): + return { + "name": name, + "value": preview, + "type": "list", + "variablesReference": var_ref, + "indexedVariables": min(length, 1000), # Cap for performance + "namedVariables": 0, + } + else: # tuple, set, other + return { + "name": name, + "value": preview, + "type": type(value).__name__, + "variablesReference": var_ref, + "indexedVariables": min(length, 1000), + "namedVariables": 0, + } + + # Simple types - optimized path + preview = self._get_preview(value) + return { + "name": name, + "value": preview, + "type": type(value).__name__, + "variablesReference": 0, + } + except Exception: + return {"name": name, "value": "", "type": "unknown", "variablesReference": 0} + + def _expand_complex_variable(self, ref_id: int) -> list[dict[str, str | int]]: + """Expand a complex variable into its child elements - optimized for memory.""" + value = self.var_cache.get_variable(ref_id) + if value is None: + return [] + + variables = [] + try: + if isinstance(value, dict): + # Limit dictionary expansion to prevent memory exhaustion + items = list(value.items()) + max_items = min(len(items), 50) # Limit to 50 items max + for i in range(max_items): + key, val = items[i] + key_str = str(key)[:50] # Limit key string length + variables.append(self._get_variable_info(key_str, val)) + if len(items) > max_items: + variables.append( + { + "name": f"<{len(items) - max_items} more items>", + "value": "...", + "type": "info", + "variablesReference": 0, + } + ) + elif isinstance(value, (list, tuple)): + # Limit list/tuple expansion + max_items = min(len(value), 100) # Limit to 100 items max + for i in range(max_items): + variables.append(self._get_variable_info(f"[{i}]", value[i])) + if len(value) > max_items: + variables.append( + { + "name": f"<{len(value) - max_items} more items>", + "value": "...", + "type": "info", + "variablesReference": 0, + } + ) + elif isinstance(value, set): + # Handle set elements with size limit + items = list(value) # Convert once + max_items = min(len(items), 50) + for i in range(max_items): + variables.append(self._get_variable_info(f"<{i}>", items[i])) + if len(items) > max_items: + variables.append( + { + "name": f"<{len(items) - max_items} more items>", + "value": "...", + "type": "info", + "variablesReference": 0, + } + ) + except Exception as e: + # Return error info for debugging + variables.append( + { + "name": "error", + "value": f"Failed to expand: {str(e)[:50]}", # Limit error message length + "type": "error", + "variablesReference": 0, + } + ) + + return variables + + @staticmethod + def _var_error(name: str): + return {"name": name, "value": "", "type": "unknown", "variablesReference": 0} + + @staticmethod + def _special_vars(varref: int): + return {"name": "Special", "value": "", "variablesReference": varref} + + def get_variables(self, variables_ref): + """Get variables for a scope with enhanced complex variable support.""" + # Handle complex variable expansion + if variables_ref >= VARREF_COMPLEX_BASE: + return self._expand_complex_variable(variables_ref) + + frame_id = variables_ref // 1000 + scope_type = variables_ref % 1000 + + if frame_id not in self.variables_cache: + return [] + + frame = self.variables_cache[frame_id] + + # Locals are read-only in DAP when this firmware has no _set_local + # (STORY-1.3): the edit affordance is greyed out client-side instead + # of setVariable failing with an error after the fact. Globals always + # stay editable - global write-back works on every firmware. + locals_read_only = not self.capabilities.get("set_local", False) + + # Handle special scope types first + if scope_type == VARREF_LOCALS_SPECIAL: + var_dict = frame.f_locals if hasattr(frame, "f_locals") else {} + return self._process_special_variables(var_dict, read_only=locals_read_only) + elif scope_type == VARREF_GLOBALS_SPECIAL: + var_dict = frame.f_globals if hasattr(frame, "f_globals") else {} + return self._process_special_variables(var_dict) + + # Handle regular scope types with special folder + variables = [] + if scope_type == VARREF_LOCALS: + var_dict = frame.f_locals if hasattr(frame, "f_locals") else {} + variables.append(self._special_vars(frame_id * 1000 + VARREF_LOCALS_SPECIAL)) + elif scope_type == VARREF_GLOBALS: + var_dict = frame.f_globals if hasattr(frame, "f_globals") else {} + variables.append(self._special_vars(frame_id * 1000 + VARREF_GLOBALS_SPECIAL)) + else: + # Invalid reference, return empty + return [] + + # Add regular variables with enhanced processing + read_only = locals_read_only if scope_type == VARREF_LOCALS else False + variables.extend(self._process_regular_variables(var_dict, read_only=read_only)) + return variables + + def evaluate_expression(self, expression, frame_id=None, context="watch"): + """Evaluate a DAP `evaluate` request in the context of a frame. + + `watch`/`hover` (and any other/absent `context`) keep the original, + read-only contract: `eval()` only - a statement is a `SyntaxError`, + surfaced as an evaluation error, exactly as before this method + gained statement support. + + `repl`/`clipboard` add statement execution: `eval()` is tried first + (so a plain expression like `1 + 1` still returns a value); a + `SyntaxError` falls back to `exec(expression, globals_dict)` against + the frame's live `f_globals` only. The locals snapshot is + deliberately never passed to `exec` as a namespace - `exec(code, g, + l)` binds a top-level assignment into `l`, and `l` here is a + disposable copy handed back to the caller and then discarded, so + the assignment would silently vanish instead of taking effect. Only + `globals_dict` is live, so a statement's top-level assignments land + in the running module namespace and are visible to the target + program after `continue`. See `_shadowed_local_warning` for the + honesty-rule warning this implies when the assigned name also + exists as a frame LOCAL. + """ + if frame_id is not None and frame_id in self.variables_cache: + frame = self.variables_cache[frame_id] + globals_dict = frame.f_globals if hasattr(frame, "f_globals") else {} + locals_dict = frame.f_locals if hasattr(frame, "f_locals") else {} + else: + # Use current frame + frame = self.current_frame + if frame: + globals_dict = frame.f_globals if hasattr(frame, "f_globals") else {} + locals_dict = frame.f_locals if hasattr(frame, "f_locals") else {} + else: + globals_dict = globals() + locals_dict = {} + + try: + result = eval(expression, globals_dict, locals_dict) + return result + except SyntaxError as e: + if context not in ("repl", "clipboard"): + raise Exception(f"Evaluation error: {e}") + except Exception as e: + raise Exception(f"Evaluation error: {e}") + + # Only repl/clipboard reach here, and only after eval() raised a + # SyntaxError - try `expression` as a statement instead. + try: + exec(expression, globals_dict) + except Exception as e: + raise Exception(f"Evaluation error: {e}") + + warning = _shadowed_local_warning(expression, locals_dict) + return warning if warning else "" + + def cleanup(self): + """Clean up resources with enhanced cache management.""" + self.variables_cache.clear() + self.var_cache.clear() # Clear variable reference cache + self.breakpoints.clear() + if hasattr(sys, "settrace"): + sys.settrace(None) + + def _lightweight_serialize(self, value): # noqa: PLR0911 + """Lightweight serialization optimized for MicroPython memory constraints.""" + if value is None: + return "None" + elif isinstance(value, bool): + return "true" if value else "false" + elif isinstance(value, (int, float)): + return str(value) + elif isinstance(value, str): + # Simple escaping for strings - avoid full JSON complexity + if len(value) > 30: + escaped = value[:27].replace('"', '\\"').replace("\n", "\\n") + return f'"{escaped}..."' + else: + escaped = value.replace('"', '\\"').replace("\n", "\\n") + return f'"{escaped}"' + elif isinstance(value, (list, tuple)): + if len(value) == 0: + return "[]" if isinstance(value, list) else "()" + elif len(value) <= 3: + # Show small collections in full + items = [self._lightweight_serialize(item) for item in value] + brackets = "[]" if isinstance(value, list) else "()" + return f"{brackets[0]}{', '.join(items)}{brackets[1]}" + else: + # Show preview for large collections + preview = f"{type(value).__name__}({len(value)} items)" + return preview + elif isinstance(value, dict): + if len(value) == 0: + return "{}" + elif len(value) <= 2: + # Show small dicts in preview form + items = [] + for k, v in value.items(): + key_str = self._lightweight_serialize(k) + val_str = self._lightweight_serialize(v) + items.append(f"{key_str}: {val_str}") + return "{" + ", ".join(items) + "}" + else: + return f"dict({len(value)} items)" + else: + # Fallback for other types + type_name = type(value).__name__ + try: + repr_val = repr(value) + if len(repr_val) > 30: + return f"<{type_name} object>" + else: + return repr_val + except: + return f"<{type_name} object>" + + def set_variable(self, variables_ref: int, name: str, value: str) -> dict[str, str | int]: + """Set a variable to a new value and return the updated variable info. + + This function can modify both global and local variables when using a MicroPython + build with settrace and local variable modification support (sys._set_local_var). + + For global variables: Works reliably on all MicroPython builds. + For local variables: Requires MicroPython build with C-level local variable support. + """ + # Handle complex variable references (not supported for setting) + if variables_ref >= VARREF_COMPLEX_BASE: + raise Exception("Cannot set variables in complex object expansions") + + frame_id = variables_ref // 1000 + scope_type = variables_ref % 1000 + + # Only allow setting variables in the topmost frame (frame_id = 0) + if frame_id != 0: + raise Exception("Variable modification is only allowed in the topmost frame") + + # Use the current frame for modification + frame = self.current_frame + if frame is None: + raise Exception("No current frame available") + + # Get the appropriate variable contexts + globals_dict = frame.f_globals if hasattr(frame, "f_globals") else {} + locals_dict = frame.f_locals if hasattr(frame, "f_locals") else {} + + try: + # Try to evaluate the new value as a Python expression + try: + new_value = eval(value, globals_dict, locals_dict) + except: + # If evaluation fails, treat as string literal + new_value = value + + if scope_type == VARREF_GLOBALS or scope_type == VARREF_GLOBALS_SPECIAL: + # Check if variable exists in globals + if name not in globals_dict: + raise Exception(f"Global variable '{name}' not found") + + # For global variables, direct assignment works reliably + globals_dict[name] = new_value + self._debug_print(f"[PDB] Successfully set global variable '{name}' = {new_value}") + + elif scope_type == VARREF_LOCALS or scope_type == VARREF_LOCALS_SPECIAL: + # Check if variable exists in locals + if name not in locals_dict: + raise Exception(f"Local variable '{name}' not found") + + # Try to use the frame._set_local method to set local variables + try: + if hasattr(frame, "_set_local"): + # Use the frame._set_local method (CPython-compatible API) + frame._set_local(name, new_value) + self._debug_print( + f"[PDB] Successfully set local variable '{name}' = {new_value}" + ) + else: + # Fallback error if the method is not available + raise Exception( + f"Cannot modify local variable '{name}'. " + f"This MicroPython build doesn't support local variable modification. " + f"Please use a MicroPython build with settrace and local variable support." + ) + except Exception as inner_e: + # If frame.set_local fails, provide detailed error + raise Exception( + f"Failed to modify local variable '{name}': {inner_e}. " + f"Local variables in MicroPython are stored in internal code_state->state[] slots. " + f"Consider using global variables for reliable modification during debugging." + ) + + else: + raise Exception("Invalid scope reference") + + # Return the updated variable info + return self._get_variable_info(name, new_value) + + except Exception as e: + raise Exception(f"Failed to set variable '{name}': {e}") diff --git a/python-ecosys/debugpy/demo.py b/python-ecosys/debugpy/demo.py new file mode 100644 index 000000000..a1f94c2b7 --- /dev/null +++ b/python-ecosys/debugpy/demo.py @@ -0,0 +1,78 @@ +"""Simple demo of MicroPython debugpy functionality. + +Exercises the pieces a session is built from without starting one: the +firmware's trace hook, and the two package internals that sit on top of it. +Starting a server is the launcher's job, not a sample's. +""" + +import sys + +# The package is a sibling of this file, not installed. +sys.path.insert(0, ".") + + +def simple_function(a, b): + """A simple function to demonstrate debugging.""" + result = a + b + print(f"Computing {a} + {b} = {result}") + return result + + +def main(): + print("MicroPython debugpy Demo") + print("========================") + print() + + # Demonstrate trace functionality + print("1. Testing trace functionality:") + + def trace_function(frame, event, arg): + if event == "call": + print(f" -> Entering function: {frame.f_code.co_name}") + elif event == "line": + print(f" -> Executing line {frame.f_lineno} in {frame.f_code.co_name}") + elif event == "return": + print(f" -> Returning from {frame.f_code.co_name} with value: {arg}") + return trace_function + + # Enable tracing + sys.settrace(trace_function) + + # Execute traced function + result = simple_function(5, 3) + + # Disable tracing + sys.settrace(None) + + print(f"Result: {result}") + print() + + # Demonstrate debugpy components + print("2. Testing debugpy components:") + + # Test PDB adapter + from debugpy.server.pdb_adapter import PdbAdapter + + pdb = PdbAdapter() + + # Set some mock breakpoints + breakpoints = pdb.set_breakpoints("demo.py", [{"line": 10}, {"line": 15}]) + print(f" Set breakpoints: {len(breakpoints)} breakpoints") + + # Test messaging + from debugpy.common.messaging import JsonMessageChannel + + print(" JsonMessageChannel available") + + print() + print("3. debugpy is ready for VS Code integration!") + print(" To debug a program with VS Code:") + print(" - Start the server first: listen(), then wait_for_client()") + print(" - Import and run the program from there, so the client's") + print(" breakpoints are already set when it starts") + print(" - Attach with the 'Attach to MicroPython' configuration") + print(" - See development_guide.md for the command") + + +if __name__ == "__main__": + main() diff --git a/python-ecosys/debugpy/development_guide.md b/python-ecosys/debugpy/development_guide.md new file mode 100644 index 000000000..d28ae17ce --- /dev/null +++ b/python-ecosys/debugpy/development_guide.md @@ -0,0 +1,122 @@ +# Debugging MicroPython debugpy with VS Code + +For working on this module: how to see the DAP conversation, and what a healthy +one looks like. + +## Starting a session + +Nothing in the program being debugged talks to the debugger. The server is +started around it: `listen()`, then `wait_for_client()`, and only then is the +program imported and run, so the breakpoints the client sent are already in +place when its first line executes. `mpremote debug` does that with a boot +script; by hand, on the unix port, the same sequence is: + +```bash +/micropython -c "import debugpy; debugpy.listen(); \ + debugpy.wait_for_client(); debugpy.debug_this_thread(); \ + import test_vscode; test_vscode.main()" +``` + +`test_vscode.py` is a sample program with no debugpy import and no manual +breakpoint in it. `wait_for_client()` blocks until a client has attached and +sent `configurationDone`, so there is no race to connect and no sleep to tune. + +## Method 1: `--dap-log` (recommended) + +If you start sessions with `mpremote debug`, pass `--dap-log`. It interposes a +proxy between the client and the device, records every complete frame in both +directions as JSONL, and reports the proxy's endpoint in place of the device's +so the client cannot bypass it: + +```bash +mpremote debug --dap-log --dap-log-file dap.jsonl +``` + +Without `--dap-log-file` it writes a timestamped file in the current directory. +This works for every transport, including serial, and needs no second port or +alternate launch configuration. + +## Method 2: the server's own logging + +Start a session by hand, as above, and read the server's own trace from its +output. Connect to `127.0.0.1:5678` with `"logToFile": true` in the +configuration - that flag, read +from the `attach` request, is what turns on the per-message trace, so without it +you get only the handful of unconditional `[DAP]` progress lines +(`vscode_launch_example.json` sets it). From then on each message is printed as +it is handled: + +``` +[DAP] SEND: response attach (req_seq=2, success=True) +[DAP] RECV: request setBreakpoints (seq=3) +[DAP] SEND: response setBreakpoints (req_seq=3, success=True) +[DAP] RECV: request configurationDone (seq=4) +``` + +Because the flag arrives with `attach`, nothing up to and including that request +is logged - the attach *response* is the first line, as above. +This is the one method that needs nothing but the firmware, so it is what to +reach for when the question is whether the server received something at all. + +## Method 3: `dap_monitor.py` + +A standalone host-side proxy: it listens on 5679, forwards to 5678, and prints +the conversation. `--dap-log` supersedes it and is the supported route; this +module is kept for driving a session without mpremote. Using it means pointing +the client at 5679 instead of the port the server reports. + +```bash +python3 dap_monitor.py +``` + +## VS Code debug logging + +VS Code can log its own side: + +1. Open settings (Ctrl+,) +2. Search for `debug.console.verbosity` and set it to `verbose` +3. Set `debug.allowBreakpointsEverywhere` to `true` + +## Expected DAP sequence + +``` +1. initialize request -> response with capabilities, then an `initialized` event +2. attach request -> response +3. setBreakpoints request -> response with verified breakpoints +4. configurationDone request -> response + ... the debuggee now runs +5. stopped event when execution reaches a breakpoint +6. stackTrace request -> response with frames +7. scopes / variables requests -> response with the frame's scopes +8. continue request -> response, and the program resumes +``` + +What each step is actually responsible for: + +- **`configurationDone` is what releases the debuggee.** `wait_for_client()` + blocks until it arrives, draining requests while it waits, so breakpoints sent + before it are already applied when the program starts. Nothing is missed while + a client connects. This is the only ordering the server enforces. +- **`attach` carries `pathMappings`** and installs the trace function. It is not + what makes tracing happen when a boot script is in play: `debug_this_thread()` + installs the same function, so a client that skips `attach` still stops at + breakpoints - it just gets no path translation, so a breakpoint set under a + host path will not match what the debuggee reports. +- **`launch` is answered with success and does nothing.** This module is + attach-only. + +Breakpoints can be sent at any point, before or after `configurationDone`, and +`setBreakpoints` does not require `attach` first. + +## Common issues to look for + +1. **No `stopped` event at all** - check the capability probe first. Without + `sys.settrace` in the firmware there is no debugger; see the `caps` dict in + the launcher's handshake line. +2. **Missing DAP capabilities** - check the `initialize` response. +3. **Breakpoint verification failures** - look at the `setBreakpoints` exchange, + and at whether the path the client sent matches what the debuggee reports. +4. **Locals show as `local_00`, `local_01`, ...** - the firmware was built + without `MICROPY_PY_SYS_SETTRACE_LOCALNAMES`; the names are not recoverable + at runtime. +5. **Evaluation problems** - check the `evaluate` request/response pairs. diff --git a/python-ecosys/debugpy/manifest.py b/python-ecosys/debugpy/manifest.py new file mode 100644 index 000000000..6c4228298 --- /dev/null +++ b/python-ecosys/debugpy/manifest.py @@ -0,0 +1,6 @@ +metadata( + description="MicroPython implementation of debugpy for remote debugging", + version="0.1.0", +) + +package("debugpy") diff --git a/python-ecosys/debugpy/test_vscode.py b/python-ecosys/debugpy/test_vscode.py new file mode 100644 index 000000000..18dc20aed --- /dev/null +++ b/python-ecosys/debugpy/test_vscode.py @@ -0,0 +1,51 @@ +"""A program to debug: ordinary MicroPython that knows nothing about debugpy. + +Something else starts the session and then runs this. A launcher calls +`debugpy.listen()` and `debugpy.wait_for_client()`, imports this module and +calls `main()`; `development_guide.md` gives the command. That is the case +worth demonstrating - stopping in code with no debugger calls in it - and the +only one available when the program being debugged is on a device and the +client is not. +""" + +import sys + +foo = 42 +bar = "Hello, MicroPython!" + + +def fibonacci(n): + """Calculate fibonacci number (iterative for efficiency).""" + if n <= 1: + return n + a, b = 0, 1 + for _ in range(2, n + 1): + a, b = b, a + b + return b + + +def debuggable_code(): + """A call to step into, a global to watch, and a loop to break inside.""" + global foo + print("Starting debuggable code...") + + # Small numbers: fibonacci is here to be stepped through, not benchmarked. + numbers = [3, 4, 5] + for i, num in enumerate(numbers): + print(f"[{i}] Calculating fibonacci({num})...") + result = fibonacci(num) # <-- SET BREAKPOINT HERE + foo += result # Modify foo to see if it gets traced + print(f"fibonacci({num}) = {result}") + + print("Test completed successfully!") + + +def main(): + print("MicroPython VS Code Debugging Test") + print("==================================") + print(sys.implementation) + debuggable_code() + + +if __name__ == "__main__": + main() diff --git a/python-ecosys/debugpy/vscode_launch_example.json b/python-ecosys/debugpy/vscode_launch_example.json new file mode 100644 index 000000000..388e696bd --- /dev/null +++ b/python-ecosys/debugpy/vscode_launch_example.json @@ -0,0 +1,22 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Attach to MicroPython", + "type": "python", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "." + } + ], + "logToFile": true, + "justMyCode": false + } + ] +} \ No newline at end of file